diff --git a/database/migrations/1768620765250_release_completeness_narrower.ts b/database/migrations/1768620765250_release_completeness_narrower.ts new file mode 100644 index 0000000..1456931 --- /dev/null +++ b/database/migrations/1768620765250_release_completeness_narrower.ts @@ -0,0 +1,94 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Release completeness (D24) for fewer releases, and without binaries + * (LOG.md 2026-10-07 (7)). + * + * - Only releases whose completeness can change are noted for the sweep: a + * par2 listing, a file_total, or complete now (it can become incomplete). + * For any other release release_complete() is false by definition; noting + * every release a file went into made a sweep check 430,000 releases in + * 51 minutes. + * - release_complete() matches listed files by name against the files + * themselves (coalesce(real_name, filename)); the size match still goes + * through a binary when there is one. Seeding no longer creates binaries + * for most files and the GC deletes them, so a name only reachable through + * a binary was lost. + * - Notes that don't qualify are dropped. + */ +export default class extends BaseSchema { + async up() { + const q = (sql: string) => this.db.rawQuery(sql) + await q(` + CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$ + SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete)) + OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) + $$`) + await q(` + CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$ + INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid) + $$`) + await q(` + CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$ + WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid), + -- The release's files, once, by index (read only when there's a par2 listing to check). + have AS MATERIALIZED ( + SELECT b.size, coalesce(f.real_name, f.filename) AS name + FROM files f LEFT JOIN binaries b ON b.file_id = f.id + WHERE f.release_id = rid) + SELECT coalesce(( + SELECT CASE + WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total + WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS ( + SELECT 1 FROM release_files rf + WHERE rf.release_id = rid AND rf.size >= 65536 + AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size) + AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name)) + ELSE false END + FROM r), false) + $$`) + await q(` + CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + WITH added AS ( + SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes, + array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts, + array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words + FROM ( + SELECT release_id, size, name_extension(nm) AS e, + CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w + FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f + ) x + GROUP BY release_id + ) + UPDATE releases r SET + file_count = r.file_count + a.n, + size = r.size + a.bytes, + extensions = CASE + WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions + ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END, + search_files = CASE + WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files + ELSE nullif(left(concat_ws(' ', r.search_files, ( + SELECT string_agg(w, ' ') FROM unnest(a.words) w + WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END + FROM added a + WHERE r.id = a.release_id; + + -- Only releases whose completeness can change (LOG.md 2026-10-07 (7)). + INSERT INTO release_dirty (release_id) + SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n + WHERE release_completeness_matters(n.release_id); + INSERT INTO file_keys (key_hash) + SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL + ON CONFLICT DO NOTHING; + RETURN NULL; + END $$`) + await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`) + } + + async down() { + // The previous definitions are in migrations …180, …190 and …220. + throw new Error('Not reversible: re-run the earlier definitions by hand if needed.') + } +} diff --git a/tests/functional/release_complete.spec.ts b/tests/functional/release_complete.spec.ts index eb2a882..52dd3d5 100644 --- a/tests/functional/release_complete.spec.ts +++ b/tests/functional/release_complete.spec.ts @@ -45,9 +45,34 @@ test.group('release completeness', (group) => { const row = await db.from('releases').where('id', id).first() assert.equal(Number(row.file_count), 3) assert.deepEqual(row.extensions, ['bin']) - // Noted for the completeness sweep once, not per file. + // No par2 listing, no file total, not complete: its completeness can't change, so it isn't noted. + const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count') + assert.equal(Number(count), 0) + }) + + test('a release with a par2 listing is noted once per insert, and complete by file names alone', async ({ assert }) => { + const now = new Date() + const [row] = await db + .table('releases') + .insert({ release_key: 'names-only'.padEnd(40, '0'), name: 'Names.Only', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }) + .returning('id') + const id = Number(row.id) + await db.rawQuery( + `INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id) + SELECT ?::bigint, md5(i::text), 'n' || i || '.rar', 100000 + i, md5(i::text), 'set' FROM generate_series(1, 3) i`, + [id] + ) + await ReleaseCompleteness.sweep() + // Seeded files: no binaries, matched by name only. + await db.rawQuery( + `INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at) + SELECT 'n' || i || '.rar', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'names-only-' || i, now(), now() FROM generate_series(1, 3) i`, + [id] + ) const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count') assert.equal(Number(count), 1) + await ReleaseCompleteness.sweep() + assert.isTrue(Boolean((await db.from('releases').where('id', id).first()).complete)) }) test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {