diff --git a/app/services/SearchService.ts b/app/services/SearchService.ts index 11c26e7..c4359db 100644 --- a/app/services/SearchService.ts +++ b/app/services/SearchService.ts @@ -48,7 +48,7 @@ export interface SearchParams { /** Size bounds in bytes (releases without a known size never match). */ minSize?: number maxSize?: number - /** File extensions (without the dot); a release matches if a file or archive entry has one of them. */ + /** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */ extensions?: string[] /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */ countLimit?: number @@ -125,12 +125,11 @@ class SearchService { where.push('r.size <= ?') bindings.push(params.maxSize) } - // search_text holds file and entry names with dots as spaces, so an - // extension is a whole word there ("iso" but not "isolation"). + // File types from file names and archive listings (migration …130), via their GIN index. const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8) if (extensions.length > 0) { - where.push(`r.search_text ~ ?`) - bindings.push(`(^| )(${extensions.join('|')})( |$)`) + where.push('r.extensions && ?::text[]') + bindings.push(extensions) } if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)') if (params.hideContainers) where.push('r.container IS NULL') diff --git a/database/migrations/1768620765130_release_extensions.ts b/database/migrations/1768620765130_release_extensions.ts new file mode 100644 index 0000000..6cec80b --- /dev/null +++ b/database/migrations/1768620765130_release_extensions.ts @@ -0,0 +1,109 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +const VISIBLE = `(file_count > 0 OR source = 'spot')` +const BATCH = 20_000 + +/** + * `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from + * its files' names and its archive listing, for the search's extension + * filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names + * enter it as stems without extension, and matching the long, TOASTed text + * took seconds per search. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any) + + await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`) + // Two to four characters with a letter (mkv, flac, 7z, par2); split + // archive volumes count as their archive type (.7z.001, .r00). + await q(` + CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ + SELECT CASE + WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$') + WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar' + ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$') + END + $$`) + await q(` + CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$ + SELECT array(SELECT DISTINCT e FROM ( + SELECT name_extension(c->>'name') AS e + FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x + WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e) + $$`) + + // A new archive listing adds its entries' types. + await q(` + CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN + NEW.extensions := nullif(array( + SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}'); + END IF; + RETURN NEW; + END $$`) + await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`) + await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`) + + // A file adds its type to its release (once: most files of a release share it). + await q(` + CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$ + DECLARE + e text := name_extension(coalesce(NEW.real_name, NEW.filename)); + BEGIN + IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF; + UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x) + WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]); + RETURN NULL; + END $$`) + await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`) + await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`) + + // Fill: file types per release in one sequential pass over files (random + // reads per release would take over an hour on the HDD), then the + // visible releases in batches by id, adding their listing's types. + await q(`DROP TABLE IF EXISTS file_extensions_tmp`) + await q(` + CREATE UNLOGGED TABLE file_extensions_tmp AS + SELECT release_id, array_agg(DISTINCT e) AS exts FROM ( + SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x + WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`) + await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`) + let last = 0 + let done = 0 + while (true) { + const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH]) + if (rows.length === 0) break + const to = Number(rows[rows.length - 1].id) + await q( + `UPDATE releases r SET extensions = x.exts FROM ( + SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts + FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id + WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x + WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`, + [last, to] + ) + last = to + done += rows.length + if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`) + } + await q(`DROP TABLE file_extensions_tmp`) + + await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`) + } + + async down() { + const q = (sql: string) => this.db.rawQuery(sql) + await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`) + await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`) + await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`) + await q(`DROP FUNCTION IF EXISTS files_release_extension()`) + await q(`DROP FUNCTION IF EXISTS releases_extensions()`) + await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`) + await q(`DROP FUNCTION IF EXISTS name_extension(text)`) + await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`) + } +} diff --git a/database/schema.ts b/database/schema.ts index 272bcdb..3b77b29 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -300,7 +300,7 @@ export class ReleaseFileSchema extends BaseModel { } export class ReleaseSchema extends BaseModel { - static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const + static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const $columns = ReleaseSchema.$columns @column() declare category: number | null @@ -313,6 +313,8 @@ export class ReleaseSchema extends BaseModel { @column() declare encrypted: boolean | null @column() + declare extensions: any | null + @column() declare fileCount: number @column() declare fileTotal: number | null diff --git a/tests/functional/web_api.spec.ts b/tests/functional/web_api.spec.ts index ab330ea..65f823a 100644 --- a/tests/functional/web_api.spec.ts +++ b/tests/functional/web_api.spec.ts @@ -61,6 +61,16 @@ test.group('Web API', (group) => { assert.notInclude(await ids({ ext: 'iso' }), id) // "sho" is part of "show", not an extension. assert.notInclude(await ids({ ext: 'sho' }), id) + + // A plainly posted file counts too, not only archive entries; volumes count as their archive. + const now = new Date() + await FileStore.insert([ + { filename: 'Webtest.Show.S01E02.1080p.nfo', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now }, + { filename: 'Webtest.Show.S01E02.1080p.7z.002', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now }, + ]) + assert.include(await ids({ ext: 'nfo' }), id) + assert.include(await ids({ ext: '7z' }), id) + assert.deepEqual((await db.from('releases').where('id', id).first()).extensions, ['7z', 'mkv', 'nfo']) }) test('serves an NZB with the real file name', async ({ client, assert }) => {