Extension filter via a releases.extensions column

The first version matched extensions in search_text, which was wrong and
slow: file names enter search_text as stems without extension (so plainly
posted .mkv files were missed), and matching the TOASTed text took 3-4 s
for common types.

- Migration …130: releases.extensions text[] (two to four characters with
  a letter; .7z.001 and .r00 count as 7z and rar), kept current by triggers
  on files (name, release) and on releases (archive listing), filled with
  one sequential pass over files, with a GIN index.
- Search: ext matches with `extensions && ?`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/SearchService.ts

@@ -48,7 +48,7 @@ export interface SearchParams {
   /** Size bounds in bytes (releases without a known size never match). */
   minSize?: number
   maxSize?: number
-  /** File extensions (without the dot); a release matches if a file or archive entry has one of them. */
+  /** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */
   extensions?: string[]
   /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
   countLimit?: number
@@ -125,12 +125,11 @@ class SearchService {
       where.push('r.size <= ?')
       bindings.push(params.maxSize)
     }
-    // search_text holds file and entry names with dots as spaces, so an
-    // extension is a whole word there ("iso" but not "isolation").
+    // File types from file names and archive listings (migration …130), via their GIN index.
     const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8)
     if (extensions.length > 0) {
-      where.push(`r.search_text ~ ?`)
-      bindings.push(`(^| )(${extensions.join('|')})( |$)`)
+      where.push('r.extensions && ?::text[]')
+      bindings.push(extensions)
     }
     if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
     if (params.hideContainers) where.push('r.container IS NULL')

database/migrations/1768620765130_release_extensions.ts

@@ -0,0 +1,109 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const VISIBLE = `(file_count > 0 OR source = 'spot')`
+const BATCH = 20_000
+
+/**
+ * `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from
+ * its files' names and its archive listing, for the search's extension
+ * filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names
+ * enter it as stems without extension, and matching the long, TOASTed text
+ * took seconds per search.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
+
+    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`)
+    // Two to four characters with a letter (mkv, flac, 7z, par2); split
+    // archive volumes count as their archive type (.7z.001, .r00).
+    await q(`
+      CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT CASE
+          WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$')
+          WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar'
+          ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$')
+        END
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$
+        SELECT array(SELECT DISTINCT e FROM (
+          SELECT name_extension(c->>'name') AS e
+          FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x
+        WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e)
+      $$`)
+
+    // A new archive listing adds its entries' types.
+    await q(`
+      CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN
+          NEW.extensions := nullif(array(
+            SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}');
+        END IF;
+        RETURN NEW;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
+    await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`)
+
+    // A file adds its type to its release (once: most files of a release share it).
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE
+        e text := name_extension(coalesce(NEW.real_name, NEW.filename));
+      BEGIN
+        IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF;
+        UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x)
+        WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]);
+        RETURN NULL;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+
+    // Fill: file types per release in one sequential pass over files (random
+    // reads per release would take over an hour on the HDD), then the
+    // visible releases in batches by id, adding their listing's types.
+    await q(`DROP TABLE IF EXISTS file_extensions_tmp`)
+    await q(`
+      CREATE UNLOGGED TABLE file_extensions_tmp AS
+      SELECT release_id, array_agg(DISTINCT e) AS exts FROM (
+        SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x
+      WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`)
+    await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
+      if (rows.length === 0) break
+      const to = Number(rows[rows.length - 1].id)
+      await q(
+        `UPDATE releases r SET extensions = x.exts FROM (
+           SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts
+           FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id
+           WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x
+         WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`,
+        [last, to]
+      )
+      last = to
+      done += rows.length
+      if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`)
+    }
+    await q(`DROP TABLE file_extensions_tmp`)
+
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`)
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`)
+    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
+    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
+    await q(`DROP FUNCTION IF EXISTS files_release_extension()`)
+    await q(`DROP FUNCTION IF EXISTS releases_extensions()`)
+    await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`)
+    await q(`DROP FUNCTION IF EXISTS name_extension(text)`)
+    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`)
+  }
+}

database/schema.ts

@@ -300,7 +300,7 @@ export class ReleaseFileSchema extends BaseModel {
 }
 
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
+  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare category: number | null
@@ -313,6 +313,8 @@ export class ReleaseSchema extends BaseModel {
   @column()
   declare encrypted: boolean | null
   @column()
+  declare extensions: any | null
+  @column()
   declare fileCount: number
   @column()
   declare fileTotal: number | null

tests/functional/web_api.spec.ts

@@ -61,6 +61,16 @@ test.group('Web API', (group) => {
     assert.notInclude(await ids({ ext: 'iso' }), id)
     // "sho" is part of "show", not an extension.
     assert.notInclude(await ids({ ext: 'sho' }), id)
+
+    // A plainly posted file counts too, not only archive entries; volumes count as their archive.
+    const now = new Date()
+    await FileStore.insert([
+      { filename: 'Webtest.Show.S01E02.1080p.nfo', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
+      { filename: 'Webtest.Show.S01E02.1080p.7z.002', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
+    ])
+    assert.include(await ids({ ext: 'nfo' }), id)
+    assert.include(await ids({ ext: '7z' }), id)
+    assert.deepEqual((await db.from('releases').where('id', id).first()).extensions, ['7z', 'mkv', 'nfo'])
   })
 
   test('serves an NZB with the real file name', async ({ client, assert }) => {