Release completeness: note only releases where it can change; names from files
A sweep checked 430k noted releases in 51 minutes, nearly all without a par2 listing or file total, where completeness can't change. Listed names are now matched against the files themselves rather than through binaries, which seeding mostly no longer creates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
06de0f21c5
commit
e7e338213a
2 files changed
+120
-1
No files matched your search
@@ -0,0 +1,94 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Release completeness (D24) for fewer releases, and without binaries
|
||||
* (LOG.md 2026-10-07 (7)).
|
||||
*
|
||||
* - Only releases whose completeness can change are noted for the sweep: a
|
||||
* par2 listing, a file_total, or complete now (it can become incomplete).
|
||||
* For any other release release_complete() is false by definition; noting
|
||||
* every release a file went into made a sweep check 430,000 releases in
|
||||
* 51 minutes.
|
||||
* - release_complete() matches listed files by name against the files
|
||||
* themselves (coalesce(real_name, filename)); the size match still goes
|
||||
* through a binary when there is one. Seeding no longer creates binaries
|
||||
* for most files and the GC deletes them, so a name only reachable through
|
||||
* a binary was lost.
|
||||
* - Notes that don't qualify are dropped.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete))
|
||||
OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
|
||||
-- The release's files, once, by index (read only when there's a par2 listing to check).
|
||||
have AS MATERIALIZED (
|
||||
SELECT b.size, coalesce(f.real_name, f.filename) AS name
|
||||
FROM files f LEFT JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = rid)
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = rid AND rf.size >= 65536
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name))
|
||||
ELSE false END
|
||||
FROM r), false)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
WITH added AS (
|
||||
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
||||
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
||||
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
||||
FROM (
|
||||
SELECT release_id, size, name_extension(nm) AS e,
|
||||
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
||||
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
||||
) x
|
||||
GROUP BY release_id
|
||||
)
|
||||
UPDATE releases r SET
|
||||
file_count = r.file_count + a.n,
|
||||
size = r.size + a.bytes,
|
||||
extensions = CASE
|
||||
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
||||
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
||||
search_files = CASE
|
||||
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
||||
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
||||
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
||||
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
||||
FROM added a
|
||||
WHERE r.id = a.release_id;
|
||||
|
||||
-- Only releases whose completeness can change (LOG.md 2026-10-07 (7)).
|
||||
INSERT INTO release_dirty (release_id)
|
||||
SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n
|
||||
WHERE release_completeness_matters(n.release_id);
|
||||
INSERT INTO file_keys (key_hash)
|
||||
SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
// The previous definitions are in migrations …180, …190 and …220.
|
||||
throw new Error('Not reversible: re-run the earlier definitions by hand if needed.')
|
||||
}
|
||||
}
|
||||
@@ -45,9 +45,34 @@ test.group('release completeness', (group) => {
|
||||
const row = await db.from('releases').where('id', id).first()
|
||||
assert.equal(Number(row.file_count), 3)
|
||||
assert.deepEqual(row.extensions, ['bin'])
|
||||
// Noted for the completeness sweep once, not per file.
|
||||
// No par2 listing, no file total, not complete: its completeness can't change, so it isn't noted.
|
||||
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
|
||||
assert.equal(Number(count), 0)
|
||||
})
|
||||
|
||||
test('a release with a par2 listing is noted once per insert, and complete by file names alone', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: 'names-only'.padEnd(40, '0'), name: 'Names.Only', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
const id = Number(row.id)
|
||||
await db.rawQuery(
|
||||
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
|
||||
SELECT ?::bigint, md5(i::text), 'n' || i || '.rar', 100000 + i, md5(i::text), 'set' FROM generate_series(1, 3) i`,
|
||||
[id]
|
||||
)
|
||||
await ReleaseCompleteness.sweep()
|
||||
// Seeded files: no binaries, matched by name only.
|
||||
await db.rawQuery(
|
||||
`INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
|
||||
SELECT 'n' || i || '.rar', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'names-only-' || i, now(), now() FROM generate_series(1, 3) i`,
|
||||
[id]
|
||||
)
|
||||
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
|
||||
assert.equal(Number(count), 1)
|
||||
await ReleaseCompleteness.sweep()
|
||||
assert.isTrue(Boolean((await db.from('releases').where('id', id).first()).complete))
|
||||
})
|
||||
|
||||
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
|
||||
|
||||
Reference in new issue
Block a user