Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 19:59:23 +02:00
1 parent bcb9be5da9
commit be8a3d4580
@@ -22,11 +22,16 @@ export default class extends BaseSchema {

  async up() {
    await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
+    // worker:spots claims recent spots first, then the history in posting order.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
    await this.swap(VISIBLE)
  }

  async down() {
    await this.swap('file_count > 0')
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_article_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_posted_idx`)
    await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
  }

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Spot details (CONCEPT.md D29): the description and image from the spot's
+ * full header. Stored whenever the header is read (recent spots, moderation,
+ * downloads); for other old spots on first view. `details_at` marks the read.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('spots', (table) => {
+      table.text('description')
+      table.jsonb('image')
+      table.timestamp('details_at', { useTz: true })
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('spots', (table) => {
+      table.dropColumn('description')
+      table.dropColumn('image')
+      table.dropColumn('details_at')
+    })
+  }
+}
@@ -0,0 +1,16 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Where a release's archive password came from: `nzb` (the NZB's
+ * `<meta type="password">`) or `spot` (the spot's title or description).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('releases', (table) => table.string('password_source', 8))
+    this.schema.raw(`UPDATE releases SET password_source = 'nzb' WHERE password IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => table.dropColumn('password_source'))
+  }
+}
@@ -0,0 +1,63 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import { packSegments, unpackSegments } from '#lib/nzb/segments'
+
+const BATCH = 5000
+
+/**
+ * Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
+ * ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
+ * plus `first_message_id` for the lookups the old expression index served.
+ * Run with the workers stopped; reclaim the space with VACUUM FULL files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await this.db.rawQuery(
+        `SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
+        [last, BATCH]
+      )
+      if (rows.length === 0) break
+      const ids: number[] = []
+      const packed: Buffer[] = []
+      const firsts: (string | null)[] = []
+      for (const row of rows) {
+        const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
+          .map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
+          .sort((a, b) => a.number - b.number)
+        ids.push(Number(row.id))
+        packed.push(packSegments(segments))
+        firsts.push(segments[0]?.id ?? null)
+      }
+      await this.db.rawQuery(
+        `UPDATE files f SET segments = v.segments, first_message_id = v.first
+         FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
+        [ids, packed, firsts] as any
+      )
+      last = ids[ids.length - 1]
+      done += rows.length
+      if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
+    const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
+    for (const row of rows) {
+      const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
+      await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
+    await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
+  }
+}
@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Header backfill (CONCEPT.md D32): a group walks backwards from
+ * `backfill_next_id` down to `backfill_until_id` in low-priority,
+ * header-only fetch ranges (`fetch_ranges.backfill`).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('groups', (table) => {
+      table.bigInteger('backfill_until_id')
+      table.bigInteger('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => {
+      table.boolean('backfill').notNullable().defaultTo(false)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('groups', (table) => {
+      table.dropColumn('backfill_until_id')
+      table.dropColumn('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => table.dropColumn('backfill'))
+  }
+}
@@ -0,0 +1,144 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const VISIBLE = `(file_count > 0 OR source = 'spot')`
+const BATCH = 20_000
+
+/**
+ * Search and index cleanup (CONCEPT.md D33).
+ * - `releases.search_text`: name, archive listing and readable file-name
+ *   stems, normalised (lowercase; dots, dashes, underscores and spaces as
+ *   one space), kept by triggers; one trigram index replaces the four the
+ *   old per-table OR search could hardly use.
+ * - Spot status indexes become partial indexes over the few rows the worker
+ *   and search look for.
+ * - Unused release indexes go (size: never scanned; complete: a boolean).
+ * Runs online: indexes are built and dropped concurrently.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
+    // For IndexMaintenance; needs a superuser (it is one in the compose setup).
+    await q(`CREATE EXTENSION IF NOT EXISTS pgstattuple`).catch((error) => console.warn(`pgstattuple not installed: ${error.message}`))
+
+    // Spots: the worker only looks for pending/processing and un-detailed imported spots, search for hidden ones.
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_todo_idx ON spots (article_number) WHERE status IN ('pending', 'processing')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_hidden_release_idx ON spots (release_id) WHERE status IN ('removed', 'dummy')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_details_todo_idx ON spots (posted_at DESC) WHERE status = 'imported' AND details_at IS NULL`)
+    for (const index of ['spots_status_posted_idx', 'spots_status_article_idx', 'spots_status_created_at_index', 'spots_status_release_id_index']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+    for (const index of ['releases_size_index', 'releases_complete_idx']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+
+    // Search text.
+    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS search_files text, ADD COLUMN IF NOT EXISTS search_text text`)
+    await q(`
+      CREATE OR REPLACE FUNCTION search_normalize(t text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT btrim(lower(regexp_replace(coalesce(t, ''), '[._[:space:]-]+', ' ', 'g')))
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_search_text(name text, contents jsonb, files text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT left(concat_ws(' ',
+          search_normalize(name),
+          nullif(search_normalize((SELECT string_agg(e->>'name', ' ') FROM (
+            SELECT e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) e LIMIT 200) x)), ''),
+          nullif(files, '')), 8000)
+      $$`)
+    // Visible releases only; computed when one becomes visible or its name, listing or file names change.
+    await q(`
+      CREATE OR REPLACE FUNCTION releases_search_text() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        IF NOT (NEW.file_count > 0 OR NEW.source = 'spot') THEN
+          NEW.search_text := NULL;
+        ELSIF TG_OP = 'INSERT' OR NEW.search_text IS NULL
+          OR NEW.name IS DISTINCT FROM OLD.name OR NEW.contents IS DISTINCT FROM OLD.contents
+          OR NEW.search_files IS DISTINCT FROM OLD.search_files THEN
+          NEW.search_text := release_search_text(NEW.name, NEW.contents, NEW.search_files);
+        END IF;
+        RETURN NEW;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`CREATE TRIGGER releases_search_text BEFORE INSERT OR UPDATE ON releases FOR EACH ROW EXECUTE FUNCTION releases_search_text()`)
+    // A file's readable stem joins its release's search text when it adds something.
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_search() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE
+        n text := coalesce(NEW.real_name, NEW.filename);
+        s text;
+      BEGIN
+        IF NEW.release_id IS NULL OR NOT is_readable_name(n) THEN RETURN NULL; END IF;
+        s := search_normalize(name_stem(n));
+        IF s = '' THEN RETURN NULL; END IF;
+        UPDATE releases SET search_files = left(concat_ws(' ', search_files, s), 4000)
+        WHERE id = NEW.release_id AND position(s IN coalesce(search_text, search_normalize(name))) = 0
+          AND length(coalesce(search_files, '')) < 4000;
+        RETURN NULL;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+
+    // Fill: readable file stems per release (one pass over files), then every visible release in batches.
+    await q(`DROP TABLE IF EXISTS search_stems_tmp`)
+    await q(`
+      CREATE UNLOGGED TABLE search_stems_tmp AS
+      SELECT release_id, left(string_agg(DISTINCT s, ' '), 4000) AS stems FROM (
+        SELECT f.release_id, search_normalize(name_stem(coalesce(f.real_name, f.filename))) AS s
+        FROM files f WHERE f.release_id IS NOT NULL AND is_readable_name(coalesce(f.real_name, f.filename))) x
+      WHERE s <> '' GROUP BY release_id`)
+    await q(`CREATE UNIQUE INDEX ON search_stems_tmp (release_id)`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
+      if (rows.length === 0) break
+      const to = Number(rows[rows.length - 1].id)
+      await q(
+        `UPDATE releases r SET search_files = (
+           SELECT nullif(array_to_string(array(
+             SELECT w FROM unnest(string_to_array(t.stems, ' ')) w
+             WHERE position(w IN search_normalize(r.name)) = 0), ' '), '')
+           FROM search_stems_tmp t WHERE t.release_id = r.id),
+           search_text = NULL
+         WHERE r.id > ? AND r.id <= ? AND ${VISIBLE.replaceAll('file_count', 'r.file_count').replace('source', 'r.source')}`,
+        [last, to]
+      )
+      last = to
+      done += rows.length
+      if (done % 200_000 < BATCH) console.log(`Search text for ${done} releases.`)
+    }
+    await q(`DROP TABLE search_stems_tmp`)
+
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_search_trgm ON releases USING gin (search_text gin_trgm_ops) WHERE ${VISIBLE}`)
+    for (const index of ['releases_name_trgm', 'releases_contents_trgm', 'files_filename_trgm', 'files_real_name_trgm']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_search_trgm`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`DROP FUNCTION IF EXISTS files_release_search()`)
+    await q(`DROP FUNCTION IF EXISTS releases_search_text()`)
+    await q(`DROP FUNCTION IF EXISTS release_search_text(text, jsonb, text)`)
+    await q(`DROP FUNCTION IF EXISTS search_normalize(text)`)
+    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS search_files, DROP COLUMN IF EXISTS search_text`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_size_index ON releases (size) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_complete_idx ON releases (complete) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_created_at_index ON spots (status, created_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_release_id_index ON spots (status, release_id)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_todo_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_hidden_release_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_details_todo_idx`)
+  }
+}
@@ -0,0 +1,43 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): also hex with a Unix time, UUIDs, digit-only names
+ * (and rclone's "83111763 n"), lowercase base32/36 tokens and short tokens
+ * with capitals all over. The same rules as looksRandom() in ArchiveService.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$'
+            OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+            OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+            OR stem ~ '^\\d{6,}( n){0,1}$'
+            OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+                AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+            OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+                AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3))
+          FROM s
+        $$`)
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$')
+          FROM s
+        $$`)
+    })
+  }
+}
@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * SFV listings (CONCEPT.md D34): the files an SFV names belong to its
+ * release; files arriving later are matched by name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.createTable('sfv_entries', (table) => {
+      table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
+      table.text('name').notNullable()
+      table.string('crc', 8).notNullable()
+      table.text('poster')
+      table.timestamp('posted_at', { useTz: true })
+      table.primary(['release_id', 'name'])
+      table.index(['name'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable('sfv_entries')
+  }
+}
@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** SFV grouping (CONCEPT.md D34) looks files up by their name. */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    // On the NVMe (default tablespace), like the files table's other indexes.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_name_idx ON files ((coalesce(real_name, filename)))`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_name_idx`)
+  }
+}
@@ -0,0 +1,37 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): generic folder/file names (VIDEO_TS, Sample, CD1) are
+ * no name either. Same list as GENERIC_NAMES in lib/release/release_key.ts.
+ */
+const FUNCTION = (generic: boolean) => `
+  CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+    WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+    SELECT name IS NOT NULL AND NOT (
+      (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+        AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+      OR stem ~* '^[0-9a-f]{16,}$'
+      OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+      OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+      OR stem ~ '^\\d{6,}( n){0,1}$'
+      OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+          AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+      OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+          AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3)
+      ${generic ? `OR btrim(stem) ~* '^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples{0,1}|subs{0,1}|subtitles{0,1}|proof|cover|covers|extras{0,1}|featurettes{0,1}|(cd|dvd|disc|disk)[ ._-]{0,1}\\d{1,2})$'` : ''})
+    FROM s
+  $$`
+
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(true))
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(false))
+    })
+  }
+}
@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Staged segments (CONCEPT.md D35): a header segment whose file is new and
+ * alone in its batch waits here until a second segment of the file shows
+ * up; ~97% never do (uploads with a random subject per article). Unlogged:
+ * no WAL, emptied by a crash, which only costs those waiting segments.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE UNLOGGED TABLE staged_segments (
+        file_key text PRIMARY KEY,
+        segments jsonb NOT NULL,
+        staged_at timestamptz NOT NULL
+      )`)
+    // Rows arrive in time order, so a BRIN index serves the expiry cheaply.
+    await this.db.rawQuery(`CREATE INDEX staged_segments_staged_at_brin ON staged_segments USING brin (staged_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE staged_segments`)
+  }
+}
@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Counter grouping (CONCEPT.md D36): staged segments also carry the key of
+ * the file they'd belong to by poster, file counter `[i/M]`, part count and
+ * posting day, so uploads with a random subject per article become files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE staged_segments ADD COLUMN IF NOT EXISTS counter_key text`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS staged_segments_counter_key_idx ON staged_segments (counter_key) WHERE counter_key IS NOT NULL`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS staged_segments_counter_key_idx`)
+    await this.db.rawQuery(`ALTER TABLE staged_segments DROP COLUMN IF EXISTS counter_key`)
+  }
+}
@@ -148,8 +148,10 @@ export class DeferredRangeSchema extends BaseModel {
 }

 export class FetchRangeSchema extends BaseModel {
-  static $columns = ['createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
+  static $columns = ['backfill', 'createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
  $columns = FetchRangeSchema.$columns
+  @column()
+  declare backfill: boolean
  @column.dateTime({ autoCreate: true })
  declare createdAt: DateTime
  @column()
@@ -171,7 +173,7 @@ export class FetchRangeSchema extends BaseModel {
 }

 export class FileSchema extends BaseModel {
-  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'realName', 'releaseId', 'size', 'updatedAt'] as const
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
  $columns = FileSchema.$columns
  @column()
  declare archive: any | null
@@ -186,12 +188,12 @@ export class FileSchema extends BaseModel {
  @column()
  declare filename: string
  @column()
+  declare firstMessageId: string | null
+  @column()
  declare groups: any | null
  @column({ isPrimary: true })
  declare id: number
  @column()
-  declare messageIds: any
-  @column()
  declare parts: number
  @column()
  declare poster: string
@@ -200,16 +202,22 @@ export class FileSchema extends BaseModel {
  @column()
  declare releaseId: bigint | number | null
  @column()
+  declare segments: Buffer
+  @column()
  declare size: bigint | number | null
  @column.dateTime({ autoCreate: true, autoUpdate: true })
  declare updatedAt: DateTime | null
 }

 export class GroupSchema extends BaseModel {
-  static $columns = ['active', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
+  static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
  $columns = GroupSchema.$columns
  @column()
  declare active: boolean
+  @column()
+  declare backfillNextId: bigint | number | null
+  @column()
+  declare backfillUntilId: bigint | number | null
  @column.dateTime({ autoCreate: true })
  declare createdAt: DateTime | null
  @column()
@@ -292,7 +300,7 @@ export class ReleaseFileSchema extends BaseModel {
 }

 export class ReleaseSchema extends BaseModel {
-  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'postedAt', 'poster', 'releaseKey', 'size', 'source'] as const
+  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
  $columns = ReleaseSchema.$columns
  @column()
  declare category: number | null
@@ -320,6 +328,8 @@ export class ReleaseSchema extends BaseModel {
  declare name: string
  @column({ serializeAs: null })
  declare password: string | null
+  @column()
+  declare passwordSource: string | null
  @column.dateTime()
  declare postedAt: DateTime | null
  @column()
@@ -327,13 +337,32 @@ export class ReleaseSchema extends BaseModel {
  @column()
  declare releaseKey: string
  @column()
+  declare searchFiles: string | null
+  @column()
+  declare searchText: string | null
+  @column()
  declare size: bigint | number
  @column()
  declare source: string | null
 }

+export class SfvEntrySchema extends BaseModel {
+  static $columns = ['crc', 'name', 'postedAt', 'poster', 'releaseId'] as const
+  $columns = SfvEntrySchema.$columns
+  @column()
+  declare crc: string
+  @column()
+  declare name: string
+  @column.dateTime()
+  declare postedAt: DateTime | null
+  @column()
+  declare poster: string | null
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+}
+
 export class SpotSchema extends BaseModel {
-  static $columns = ['articleNumber', 'category', 'createdAt', 'error', 'groupName', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'website'] as const
+  static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const
  $columns = SpotSchema.$columns
  @column()
  declare articleNumber: bigint | number | null
@@ -342,10 +371,16 @@ export class SpotSchema extends BaseModel {
  @column.dateTime({ autoCreate: true })
  declare createdAt: DateTime
  @column()
+  declare description: string | null
+  @column.dateTime()
+  declare detailsAt: DateTime | null
+  @column()
  declare error: string | null
  @column()
  declare groupName: string
  @column()
+  declare image: any | null
+  @column()
  declare keyId: number | null
  @column({ isPrimary: true })
  declare messageId: string
@@ -370,9 +405,24 @@ export class SpotSchema extends BaseModel {
  @column()
  declare title: string
  @column()
+  declare verified: boolean | null
+  @column()
  declare website: string | null
 }

+export class StagedSegmentSchema extends BaseModel {
+  static $columns = ['counterKey', 'fileKey', 'segments', 'stagedAt'] as const
+  $columns = StagedSegmentSchema.$columns
+  @column()
+  declare counterKey: string | null
+  @column({ isPrimary: true })
+  declare fileKey: string
+  @column()
+  declare segments: any
+  @column.dateTime()
+  declare stagedAt: DateTime
+}
+
 export class UserSchema extends BaseModel {
  static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
  $columns = UserSchema.$columns