diff --git a/app/services/spot_inspector.ts b/app/services/spot_inspector.ts index 4ee9471..f9f7f9b 100644 --- a/app/services/spot_inspector.ts +++ b/app/services/spot_inspector.ts @@ -16,6 +16,8 @@ const BATCH = 20 const IDLE_MS = 60_000 /** Unmatched spots are matched again this often (the header pass may have built their releases since). */ const REMATCH_MS = 60 * 60 * 1000 +/** Unmatched spots are matched again for this long after their inspection. */ +const REMATCH_DAYS = 30 /** A file's release posted further apart than this from the NZB's date is another upload of the same name. */ const MAX_SPREAD_MS = 48 * 3600 * 1000 @@ -62,9 +64,11 @@ export function nzbFileName(subject: string): { name: string; base: string } { * Each matched release is linked in `release_spots`, and gets the spot's * password in `release_passwords` (the NZB's own `` * first, else what the spot's title or text names) and `passworded`. A spot - * whose files match nothing is matched again every hour; after - * SPOT_STANDALONE_AFTER_HOURS it becomes a release of its own (source `spot`) - * with its NZB. + * whose files match nothing is matched again every hour, for 30 days; once + * SPOT_STANDALONE_AFTER_HOURS have passed since its inspection it becomes a + * release of its own (source `spot`) with its NZB, until the header pass + * builds the releases after all (a spot older than what the catalog has + * indexed): then it moves to those, and its own release goes. */ export class SpotInspector { private stopped = false @@ -199,6 +203,7 @@ export class SpotInspector { status: 'inspected', inspected_at: new Date(), files: files.length, + matched: 0, nzb: gzipSync(xml), ...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}), }) @@ -254,7 +259,17 @@ export class SpotInspector { } const perRelease = new Map() for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1) + if (perRelease.size === 0) return 0 await db.transaction(async (trx) => { + // A release of the spot's own goes now that the header pass built its files' releases. + if (spot.release_id) { + const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first() + if (own) { + await trx.from('release_spots').where('release_id', own.id).delete() + await trx.from('release_passwords').where('release_id', own.id).delete() + await trx.from('releases').where('id', own.id).delete() + } + } for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId }) for (const [releaseId, n] of perRelease) { await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n }) @@ -276,13 +291,19 @@ export class SpotInspector { /** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */ async rematch(): Promise<{ matched: number; standalone: number }> { const out = { matched: 0, standalone: 0 } - const spots = await db.from('spots').where('status', 'inspected').where('matched', 0).whereNull('release_id').orderBy('inspected_at').limit(500) + const { rows: spots } = await db.rawQuery( + `SELECT message_id, inspected_at, release_id FROM spots + WHERE status = 'inspected' AND matched = 0 AND inspected_at > now() - make_interval(days => ?) + ORDER BY coalesce(rematched_at, inspected_at) LIMIT 500`, + [REMATCH_DAYS] + ) const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000 for (const spot of spots) { if (this.stopped) break const n = await this.match(spot.message_id) + await db.from('spots').where('message_id', spot.message_id).update({ rematched_at: new Date() }) out.matched += n - if (n === 0 && spot.posted_at && new Date(spot.posted_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++ + if (n === 0 && !spot.release_id && new Date(spot.inspected_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++ } return out } diff --git a/database/migrations/1770000002800_spots.ts b/database/migrations/1770000002800_spots.ts index df681c1..05d07d9 100644 --- a/database/migrations/1770000002800_spots.ts +++ b/database/migrations/1770000002800_spots.ts @@ -9,7 +9,8 @@ import { BaseSchema } from '@adonisjs/lucid/schema' * (alt.binaries.ftd), keeps its files in `spot_files`, and matches them to * the releases the header pass built (the same release identity), linking * them in `release_spots`. A spot whose files match no release becomes a - * release of its own (source `spot`) once a day has passed. + * release of its own (source `spot`) a day after its inspection (until + * the header pass builds its releases after all: migration 2900). * The NZB itself is kept (gzip) for such a release's downloads. * Passwords are kept apart from the catalog in `release_passwords`, one row * per release and password, with where it came from (the NZB's meta, the diff --git a/database/migrations/1770000002900_spots_rematched.ts b/database/migrations/1770000002900_spots_rematched.ts new file mode 100644 index 0000000..9400cdc --- /dev/null +++ b/database/migrations/1770000002900_spots_rematched.ts @@ -0,0 +1,21 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Unmatched spots are matched again in turns (`rematched_at`), for 30 days after their inspection, also once they + * became a release of their own: the header pass may build their releases later (a spot's history before the + * catalog's), and then the spot moves to that release (LOG.md 2026-10-13 (45)). + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS rematched_at timestamptz`) + // Inspected before the inspector set it (2026-10-13). + this.schema.raw(`UPDATE spots SET matched = 0 WHERE status = 'inspected' AND matched IS NULL`) + this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`) + this.schema.raw(`CREATE INDEX IF NOT EXISTS spots_unmatched_idx ON spots (coalesce(rematched_at, inspected_at)) WHERE status = 'inspected' AND matched = 0`) + } + + async down() { + this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`) + this.schema.raw(`ALTER TABLE spots DROP COLUMN IF EXISTS rematched_at`) + } +} diff --git a/tests/functional/spots.spec.ts b/tests/functional/spots.spec.ts index a2b6126..0dbdf60 100644 --- a/tests/functional/spots.spec.ts +++ b/tests/functional/spots.spec.ts @@ -105,5 +105,17 @@ test.group('Spots', (group) => { assert.equal(release.files, 1) const { default: NzbService } = await import('#services/nzb_service') assert.include((await NzbService.build(Number(release.id)))!.xml, 'lonely.file.mkv') + // Not again while it has one; once the header pass builds the file's release, the spot moves there. + assert.deepEqual(await inspector.rematch(), { matched: 0, standalone: 0 }) + const group = await db.from('groups').where('name', 'alt.binaries.test.spots').firstOrFail() + const parsed = parseSubject('[1/1] - "lonely.file.mkv" yEnc (1/9)')! + const identity = releaseIdentity({ poster: 'p', subject: parsed.base, name: parsed.name, weak: false })! + const [h] = await db + .table('releases') + .insert({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster: 'p', groups: '[]', group_id: group.id, files: 1, posted_at: new Date(STAMP * 1000), readable: true }) + .returning('id') + assert.equal(await inspector.match(SPOT), 1) + assert.isNull(await db.from('releases').where('id', release.id).first()) + assert.equal(Number((await db.from('spots').where('message_id', SPOT).firstOrFail()).release_id), Number(h.id)) }) })