Fold only sidecar-only releases (sidecar_only at insert); no queue-wide scan for gone releases

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -3,6 +3,17 @@ import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
 import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
 import ChunkPartitions from '#services/chunk_partitions'
 import { decodeChunk } from '#lib/chunks/index'
+import { SIDECAR } from '#services/sidecar_folder'
+
+/**
+ * A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
+ * fold's candidates (releases.sidecar_only; the settler queues only these). Its file list decoded here, small as it is.
+ */
+function sidecarOnly(r: ReleaseChunk): boolean | null {
+  if (r.files > 3 || r.bytes >= 1_000_000) return null
+  const files = decodeChunk(r.payload)
+  return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
+}
 
 export class StaleBatch extends Error {}
 
@@ -241,7 +252,7 @@ class BatchStore {
         const numbered = (r: ReleaseChunk) => {
           const n = (taken.get(r.key) ?? 0) + 1
           taken.set(r.key, n)
-          return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
+          return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r), sidecar_only: sidecarOnly(r) }
         }
         // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
         const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
@@ -252,10 +263,10 @@ class BatchStore {
         for (let attempt = 0; inserts.length > 0; attempt++) {
           if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
           const { rows } = await trx.rawQuery(
-            `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
+            `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete, sidecar_only)
              SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
-                    v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
-             FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
+                    v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END, v.sidecar_only
+             FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean, sidecar_only boolean)
              ON CONFLICT (release_key) DO NOTHING
              RETURNING id, release_key`,
             [groupId, JSON.stringify(inserts)]

app/services/settler.ts

@@ -28,10 +28,11 @@ class Settler {
            UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
            WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
              AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
-           RETURNING id, readable, files, bytes, poster),
+           RETURNING id, readable, files, bytes, poster, sidecar_only),
          marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
-         -- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
-         queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
+         -- A small one of sidecars only (the batch store's sidecar_only) may be an upload's .nfo or .nzb that came apart
+         -- from it: the merger's sidecar fold looks at them.
+         queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE sidecar_only AND files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
            ON CONFLICT DO NOTHING)
          SELECT count(*)::int AS n FROM settled`,
         [QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]

app/services/sidecar_folder.ts

@@ -6,7 +6,7 @@ const BATCH = 2000
 /** An upload's sidecar is posted with it: its release is looked for this far from it. */
 const WINDOW_MINUTES = 10
 /** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
-const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
+export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
 
 /** The words of a name worth comparing: 4+ letters or digits. */
 function words(text: string): Set<string> {
@@ -53,19 +53,17 @@ export class SidecarFolder {
 
   async round(): Promise<FoldRound> {
     const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
+    // A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
+    // queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
     const { rows: queued } = await db.rawQuery(
-      `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
-       FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
+      `SELECT f.release_id AS queued_id, r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
+       FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
       [BATCH]
     )
-    const { rows: stale } = await db.rawQuery(
-      `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
-      [BATCH]
-    )
-    const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
+    const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
     for (const c of queued) {
+      if (c.id === null) continue
       round.looked++
-      done.push(Number(c.id))
       const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
       const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
       if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue

database/migrations/1770000004200_releases_sidecar_only.ts

@@ -0,0 +1,16 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * releases.sidecar_only: set by the batch store when it inserts a small release (at most 3 files, under 1 MB) whose
+ * files are all sidecars (.nfo, .sfv, .nzb, …). The settler queues only those for the sidecar fold: it queued every
+ * small release, 13.5M (single images mostly), and the fold's queue scans held the merger back (LOG.md 2026-10-17 (72)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS sidecar_only boolean`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS sidecar_only`)
+  }
+}

tests/functional/batch_store.spec.ts

@@ -146,4 +146,17 @@ test.group('BatchStore', (group) => {
     const keys = (await releasesOf()).filter((r) => r.identity_key === first.identity_key).map((r) => r.release_key)
     assert.deepEqual(keys, [first.identity_key, `${first.identity_key}:3`])
   })
+
+  test('a small release of sidecars only is marked for the sidecar fold', async ({ assert }) => {
+    await storeIn(GROUP, 1_000_000, [
+      ...upload(1_000_000, 'Murdoch Mysteries S8D1.nzb', 1, '[00/80]', '2026-10-05T10:00:00Z'),
+      ...upload(1_000_010, 'Some.Tool.v1.zip', 1, '[01/01]', '2026-10-05T10:00:00Z'),
+      ...upload(1_000_020, 'Some.Show.S01E01.part01.rar', 2, '[02/30]', '2026-10-05T10:00:00Z'),
+    ])
+    const flags = new Map((await db.from('releases').select('name', 'sidecar_only')).map((r) => [r.name, r.sidecar_only]))
+    assert.isTrue(flags.get('Murdoch Mysteries S8D1'))
+    assert.isFalse(flags.get('Some.Tool.v1'))
+    // Over 1 MB: not looked at.
+    assert.isNull(flags.get('Some.Show.S01E01'))
+  })
 })

tests/functional/par2_merger.spec.ts

@@ -220,6 +220,8 @@ test.group('Par2Merger', (group) => {
     // A picture isn't a sidecar; an nfo an hour later belongs to nothing here.
     const picture = await release('cover', 1, null, at, ['cover.jpg'])
     const late = await release('Murdoch Mysteries S8D1', 1, null, new Date(at.getTime() + 3600_000), ['Murdoch Mysteries S8D1.nfo'])
+    // Queued, its release gone meanwhile (merged): leaves the queue with its batch.
+    await db.table('fold_candidates').insert({ release_id: 987654321 })
 
     const round = await new Par2Merger().round()
     assert.deepEqual(round.folded, { looked: 5, sidecars: 2, folded: 1 })