Judge: write only changed verdicts; scraped and locked by index
A round took 22-29 min an hour: every match went through the insert's conflict check (60-150 s a rule), scraped and locked each scanned all of releases. The spot inspector's releases now add their poster to posters. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aa179f908a
commit
a983cad509
5 files changed
+42
-4
No files matched your search
@@ -64,12 +64,15 @@ export const RULES: Rule[] = [
|
||||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||||
sql: `SELECT r.id FROM releases r
|
||||
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||||
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
|
||||
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
|
||||
sql: `SELECT r.id FROM posters p JOIN releases r ON left(r.poster, 200) = p.poster
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||||
},
|
||||
{
|
||||
name: 'locked',
|
||||
says: 'passworded, and no spot, NZB or title gives the password',
|
||||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||||
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
@@ -141,12 +144,16 @@ export class ReleaseJudge {
|
||||
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
|
||||
[rule.name]
|
||||
)
|
||||
// Only the releases whose verdict changes: each row through the insert's conflict check took 60–150 s a rule
|
||||
// for a few thousand changes in a million matches (LOG.md 2026-10-19 (80)).
|
||||
await trx.rawQuery(`ANALYZE judged`)
|
||||
const added = await trx.rawQuery(
|
||||
`INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
|
||||
SELECT j.release_id, ?, ? FROM judged j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
|
||||
WHERE v.rule IS DISTINCT FROM ? AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule`,
|
||||
[rule.name, JUDGE_VERSION]
|
||||
[rule.name, JUDGE_VERSION, rule.name]
|
||||
)
|
||||
stats.removed = removed.rowCount ?? 0
|
||||
stats.added = added.rowCount ?? 0
|
||||
|
||||
@@ -349,6 +349,8 @@ export class SpotInspector {
|
||||
const releaseId = Number(row.id)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
|
||||
// Its poster for poster search, as the batch store adds its releases' (migration 4000).
|
||||
if (spot.poster) await trx.rawQuery(`INSERT INTO posters (poster) VALUES (left(?, 200)) ON CONFLICT DO NOTHING`, [spot.poster])
|
||||
if (spot.password) {
|
||||
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user