diff --git a/app/services/release_judge.ts b/app/services/release_judge.ts index 035bf5c..c77537f 100644 --- a/app/services/release_judge.ts +++ b/app/services/release_judge.ts @@ -64,12 +64,15 @@ export const RULES: Rule[] = [ // "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …: // a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other // titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name. - sql: `SELECT r.id FROM releases r - WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`, + // By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the + // regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)). + sql: `SELECT r.id FROM posters p JOIN releases r ON left(r.poster, 200) = p.poster + WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`, }, { name: 'locked', says: 'passworded, and no spot, NZB or title gives the password', + // releases_passworded_idx: a scan of all of releases was 30 s a round. sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`, }, { @@ -141,12 +144,16 @@ export class ReleaseJudge { `DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`, [rule.name] ) + // Only the releases whose verdict changes: each row through the insert's conflict check took 60–150 s a rule + // for a few thousand changes in a million matches (LOG.md 2026-10-19 (80)). + await trx.rawQuery(`ANALYZE judged`) const added = await trx.rawQuery( `INSERT INTO release_verdicts (release_id, rule, rule_version) - SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id + SELECT j.release_id, ?, ? FROM judged j LEFT JOIN release_verdicts v ON v.release_id = j.release_id + WHERE v.rule IS DISTINCT FROM ? AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id) ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version WHERE release_verdicts.rule <> excluded.rule`, - [rule.name, JUDGE_VERSION] + [rule.name, JUDGE_VERSION, rule.name] ) stats.removed = removed.rowCount ?? 0 stats.added = added.rowCount ?? 0 diff --git a/app/services/spot_inspector.ts b/app/services/spot_inspector.ts index 60d18bc..9ed5a83 100644 --- a/app/services/spot_inspector.ts +++ b/app/services/spot_inspector.ts @@ -349,6 +349,8 @@ export class SpotInspector { const releaseId = Number(row.id) await db.transaction(async (trx) => { await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore() + // Its poster for poster search, as the batch store adds its releases' (migration 4000). + if (spot.poster) await trx.rawQuery(`INSERT INTO posters (poster) VALUES (left(?, 200)) ON CONFLICT DO NOTHING`, [spot.poster]) if (spot.password) { await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore() } diff --git a/database/migrations/1770000004700_judge_indexes.ts b/database/migrations/1770000004700_judge_indexes.ts new file mode 100644 index 0000000..1b8c958 --- /dev/null +++ b/database/migrations/1770000004700_judge_indexes.ts @@ -0,0 +1,22 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The junk judge's locked rule by an index of the passworded releases (1.2M of 74M), not a scan of all of releases + * every round (30 s; LOG.md 2026-10-19 (80)). And the posters of the spot inspector's own releases (source `spot`), + * which it didn't add to posters: poster search, and the judge's scraped rule by posters, missed them. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_passworded_idx ON releases (id) WHERE passworded`) + this.schema.raw( + `INSERT INTO posters (poster) SELECT DISTINCT left(s.poster, 200) FROM spots s + JOIN releases r ON r.id = s.release_id AND r.source = 'spot' WHERE s.poster IS NOT NULL ON CONFLICT DO NOTHING` + ) + } + + async down() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_passworded_idx`) + } +} diff --git a/tests/functional/release_judge.spec.ts b/tests/functional/release_judge.spec.ts index 988dc2f..abbafc5 100644 --- a/tests/functional/release_judge.spec.ts +++ b/tests/functional/release_judge.spec.ts @@ -59,6 +59,8 @@ test.group('ReleaseJudge', (group) => { const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', { poster: '@_CelebMovie_@', source: 'subject' }) const celeborn = await release('Some.Album.2010', { poster: 'celeborn@not.here (Celeborn)', source: 'subject' }) + // Scraped posters are found through posters (the batch store adds each release's). + await db.table('posters').multiInsert([{ poster: '@_CelebMovie_@' }, { poster: 'celeborn@not.here (Celeborn)' }]) const locked = await release('Locked.Thing', { passworded: true }) const unlocked = await release('Known.Password', { passworded: true }) @@ -107,6 +109,9 @@ test.group('ReleaseJudge', (group) => { await db.table('release_passwords').insert({ release_id: locked, password: 'found', source: 'nzb' }) const again = await new ReleaseJudge().round() assert.equal(again.rules.locked.removed, 1) + // Verdicts that stay are not written again. + assert.equal(again.rules.repeated.added, 0) + assert.equal(again.rules.fake.added, 0) assert.isNull(await db.from('release_verdicts').where('release_id', locked).first()) }) }) diff --git a/tests/functional/spots.spec.ts b/tests/functional/spots.spec.ts index 0dbdf60..01df678 100644 --- a/tests/functional/spots.spec.ts +++ b/tests/functional/spots.spec.ts @@ -103,6 +103,8 @@ test.group('Spots', (group) => { const release = await db.from('releases').where('source', 'spot').firstOrFail() assert.equal(release.name, 'Lonely File') assert.equal(release.files, 1) + assert.isNotNull(release.poster) + assert.isNotNull(await db.from('posters').where('poster', release.poster).first()) const { default: NzbService } = await import('#services/nzb_service') assert.include((await NzbService.build(Number(release.id)))!.xml, 'lonely.file.mkv') // Not again while it has one; once the header pass builds the file's release, the spot moves there.