diff --git a/app/services/batch_store.ts b/app/services/batch_store.ts index 8b57d4f..a832c9b 100644 --- a/app/services/batch_store.ts +++ b/app/services/batch_store.ts @@ -3,6 +3,17 @@ import type { TransactionClientContract } from '@adonisjs/lucid/types/database' import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index' import ChunkPartitions from '#services/chunk_partitions' import { decodeChunk } from '#lib/chunks/index' +import { SIDECAR } from '#services/sidecar_folder' + +/** + * A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar + * fold's candidates (releases.sidecar_only; the settler queues only these). Its file list decoded here, small as it is. + */ +function sidecarOnly(r: ReleaseChunk): boolean | null { + if (r.files > 3 || r.bytes >= 1_000_000) return null + const files = decodeChunk(r.payload) + return files.length > 0 && files.every((f) => SIDECAR.test(f.name)) +} export class StaleBatch extends Error {} @@ -241,7 +252,7 @@ class BatchStore { const numbered = (r: ReleaseChunk) => { const n = (taken.get(r.key) ?? 0) + 1 taken.set(r.key, n) - return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) } + return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r), sidecar_only: sidecarOnly(r) } } // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000). const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))] @@ -252,10 +263,10 @@ class BatchStore { for (let attempt = 0; inserts.length > 0; attempt++) { if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`) const { rows } = await trx.rawQuery( - `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete) + `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete, sidecar_only) SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, - v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END - FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean) + v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END, v.sidecar_only + FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean, sidecar_only boolean) ON CONFLICT (release_key) DO NOTHING RETURNING id, release_key`, [groupId, JSON.stringify(inserts)] diff --git a/app/services/settler.ts b/app/services/settler.ts index 8463020..3bc78e2 100644 --- a/app/services/settler.ts +++ b/app/services/settler.ts @@ -28,10 +28,11 @@ class Settler { UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?) AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?) - RETURNING id, readable, files, bytes, poster), + RETURNING id, readable, files, bytes, poster, sidecar_only), marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id), - -- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them. - queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL + -- A small one of sidecars only (the batch store's sidecar_only) may be an upload's .nfo or .nzb that came apart + -- from it: the merger's sidecar fold looks at them. + queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE sidecar_only AND files <= 3 AND bytes < 1000000 AND poster IS NOT NULL ON CONFLICT DO NOTHING) SELECT count(*)::int AS n FROM settled`, [QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH] diff --git a/app/services/sidecar_folder.ts b/app/services/sidecar_folder.ts index c95951a..965b10e 100644 --- a/app/services/sidecar_folder.ts +++ b/app/services/sidecar_folder.ts @@ -6,7 +6,7 @@ const BATCH = 2000 /** An upload's sidecar is posted with it: its release is looked for this far from it. */ const WINDOW_MINUTES = 10 /** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */ -const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i +export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i /** The words of a name worth comparing: 4+ letters or digits. */ function words(text: string): Set { @@ -53,19 +53,17 @@ export class SidecarFolder { async round(): Promise { const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 } + // A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole + // queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)). const { rows: queued } = await db.rawQuery( - `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable - FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`, + `SELECT f.release_id AS queued_id, r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable + FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`, [BATCH] ) - const { rows: stale } = await db.rawQuery( - `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`, - [BATCH] - ) - const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id)) + const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id)) for (const c of queued) { + if (c.id === null) continue round.looked++ - done.push(Number(c.id)) const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id]) const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload)) if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue diff --git a/database/migrations/1770000004200_releases_sidecar_only.ts b/database/migrations/1770000004200_releases_sidecar_only.ts new file mode 100644 index 0000000..b6e1fc9 --- /dev/null +++ b/database/migrations/1770000004200_releases_sidecar_only.ts @@ -0,0 +1,16 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * releases.sidecar_only: set by the batch store when it inserts a small release (at most 3 files, under 1 MB) whose + * files are all sidecars (.nfo, .sfv, .nzb, …). The settler queues only those for the sidecar fold: it queued every + * small release, 13.5M (single images mostly), and the fold's queue scans held the merger back (LOG.md 2026-10-17 (72)). + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS sidecar_only boolean`) + } + + async down() { + this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS sidecar_only`) + } +} diff --git a/tests/functional/batch_store.spec.ts b/tests/functional/batch_store.spec.ts index 48f261d..34e3691 100644 --- a/tests/functional/batch_store.spec.ts +++ b/tests/functional/batch_store.spec.ts @@ -146,4 +146,17 @@ test.group('BatchStore', (group) => { const keys = (await releasesOf()).filter((r) => r.identity_key === first.identity_key).map((r) => r.release_key) assert.deepEqual(keys, [first.identity_key, `${first.identity_key}:3`]) }) + + test('a small release of sidecars only is marked for the sidecar fold', async ({ assert }) => { + await storeIn(GROUP, 1_000_000, [ + ...upload(1_000_000, 'Murdoch Mysteries S8D1.nzb', 1, '[00/80]', '2026-10-05T10:00:00Z'), + ...upload(1_000_010, 'Some.Tool.v1.zip', 1, '[01/01]', '2026-10-05T10:00:00Z'), + ...upload(1_000_020, 'Some.Show.S01E01.part01.rar', 2, '[02/30]', '2026-10-05T10:00:00Z'), + ]) + const flags = new Map((await db.from('releases').select('name', 'sidecar_only')).map((r) => [r.name, r.sidecar_only])) + assert.isTrue(flags.get('Murdoch Mysteries S8D1')) + assert.isFalse(flags.get('Some.Tool.v1')) + // Over 1 MB: not looked at. + assert.isNull(flags.get('Some.Show.S01E01')) + }) }) diff --git a/tests/functional/par2_merger.spec.ts b/tests/functional/par2_merger.spec.ts index 773b17e..22b10fa 100644 --- a/tests/functional/par2_merger.spec.ts +++ b/tests/functional/par2_merger.spec.ts @@ -220,6 +220,8 @@ test.group('Par2Merger', (group) => { // A picture isn't a sidecar; an nfo an hour later belongs to nothing here. const picture = await release('cover', 1, null, at, ['cover.jpg']) const late = await release('Murdoch Mysteries S8D1', 1, null, new Date(at.getTime() + 3600_000), ['Murdoch Mysteries S8D1.nfo']) + // Queued, its release gone meanwhile (merged): leaves the queue with its batch. + await db.table('fold_candidates').insert({ release_id: 987654321 }) const round = await new Par2Merger().round() assert.deepEqual(round.folded, { looked: 5, sidecars: 2, folded: 1 })