Daan MeijerandClaude Fable 5.1
a71de56a7c
One file of one article, 2–100 kB, under a readable release-like name outside the music, e-book and comic groups, without an e-book, subtitle, crack or sidecar word: the 2015–17 flood's 1.5M rows that the edge indexed before its rule. The same tests as isTinyArchive in Postgres's regex dialect; no join on the candidates (98% of a sample have an archive candidate, the rest lone par2 files and scrambled names). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
314 lines
17 KiB
TypeScript
314 lines
17 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
import { SOURCES } from '#lib/release/source'
|
||
import { TINY_MIN_BYTES, NONSENSE_MIN_BYTES } from '#lib/rules/index'
|
||
|
||
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
|
||
export const JUDGE_VERSION = 1
|
||
/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
|
||
const ROUND_MS = 2 * 60 * 1000
|
||
/**
|
||
* A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
|
||
* (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
|
||
* after a merge (repeated).
|
||
*/
|
||
const FULL_MS = 24 * 60 * 60 * 1000
|
||
/** Changed releases judged at a time. */
|
||
const CHUNK = 50_000
|
||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||
const REPEATED_RELEASES = 20
|
||
|
||
/**
|
||
* A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
|
||
* these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
|
||
* 2026-10-16 (64)), so {0,1}.
|
||
*/
|
||
const MEDIA =
|
||
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
||
/** The posters of scraped celebrity pictures. */
|
||
const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
|
||
/** An executable's name. */
|
||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||
/**
|
||
* The tiny rule's name tests, lib/rules/rule_set.ts's RELEASE_LIKE_SPACED, EBOOK, TINY_CONTENT and
|
||
* STUB_EXEMPT_GROUPS in Postgres's dialect (and no "?", see MEDIA).
|
||
*/
|
||
const TINY_RELEASE_LIKE = `(\\y(19|20)[0-9][0-9]\\y|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-{0,1}dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\\.[0-9]+)+)\\y|-[a-z0-9]{2,}$|\\.[a-z0-9]+\\.[a-z0-9]+\\.)`
|
||
const TINY_EBOOK = `\\y(epub|mobi|azw3{0,1}|pdf|djvu|cbr|cbz|e-{0,1}books{0,1})\\y`
|
||
const TINY_CONTENT = `\\y(subs{0,1}|subtitles{0,1}|subpack|srt|untertitel|ondertitels{0,1}|ondertiteling|legendas{0,1}|napisy|sottotitoli|cracks{0,1}|cracked|keygen|keymaker|trainer|patch|patched|serials{0,1}|activator|loader|nocd|no-cd|fix|covers{0,1}|artwork|booklet|scans{0,1}|proof|sample|info|readme)\\y`
|
||
const TINY_EXEMPT_GROUPS = `(sound|mp3|karaoke|music|audio|flac|lossless|e-{0,1}book|epub|comic|guitar|font|magazine)`
|
||
|
||
export interface Rule {
|
||
name: string
|
||
/** Why, in a few words (the log, the web interface). */
|
||
says: string
|
||
/** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
|
||
sql: (scoped: boolean) => string
|
||
}
|
||
|
||
/** A join to the releases in scope (an incremental round), on the given release id column. */
|
||
const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
|
||
|
||
|
||
|
||
/**
|
||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||
* the header pass, the prober, the merges and the spots know by now.
|
||
*/
|
||
export const RULES: Rule[] = [
|
||
{
|
||
name: 'dead',
|
||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||
sql: (scoped) =>
|
||
`SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
|
||
WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
|
||
},
|
||
{
|
||
name: 'fake',
|
||
says: 'named as a film, episode or music, but the content is a program',
|
||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||
// an executable by name or by its first bytes, under a media name.
|
||
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
|
||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||
UNION
|
||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||
},
|
||
{
|
||
name: 'repeated',
|
||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||
// Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
|
||
// (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
|
||
sql: (scoped) =>
|
||
scoped
|
||
? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
|
||
: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||
},
|
||
{
|
||
name: 'scraped',
|
||
says: 'celebrity pictures scraped from the web, reposted by one spam family',
|
||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
|
||
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
|
||
sql: (scoped) =>
|
||
scoped
|
||
? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
|
||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
|
||
: `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
|
||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
|
||
},
|
||
{
|
||
name: 'locked',
|
||
says: 'passworded, and no spot, NZB or title gives the password',
|
||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||
},
|
||
{
|
||
name: 'tiny',
|
||
says: 'one file of one article, 2–100 kB, under a release-like name: the exe-stub flood in miniature',
|
||
// The catalog's side of the edge's tiny-archive rule (lib/rules/rule_set.ts isTinyArchive; LOG.md (131),
|
||
// (135)): the same name tests in Postgres's regex dialect ("\y" for "\b", "{0,1}" for "?"), the readable flag
|
||
// for its scrambled and random tests, the group set for its groups. The 2015–17 flood that the edge indexed
|
||
// before the rule: 1.5M rows, 98% of them with an archive candidate, the rest lone par2 files and scrambled
|
||
// names, so no join on the candidates. A scan of releases (9 GB) a full round.
|
||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||
WHERE r.files = 1 AND r.segments = 1 AND r.bytes >= ${TINY_MIN_BYTES} AND r.bytes < ${NONSENSE_MIN_BYTES}
|
||
AND r.readable AND NOT coalesce(r.sidecar_only, false)
|
||
AND r.name ~* '${TINY_RELEASE_LIKE}' AND r.name !~* '${TINY_EBOOK}' AND r.name !~* '${TINY_CONTENT}'
|
||
AND NOT EXISTS (SELECT 1 FROM group_sets gs WHERE gs.id = r.group_set_id AND gs.groups::text ~* '${TINY_EXEMPT_GROUPS}')`,
|
||
},
|
||
{
|
||
name: 'unusable',
|
||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
|
||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||
AND r.settled_at IS NOT NULL
|
||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||
},
|
||
]
|
||
|
||
export interface JudgeRound {
|
||
/** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
|
||
releases?: number
|
||
/** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
|
||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||
}
|
||
|
||
/**
|
||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
|
||
* minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
|
||
* daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
|
||
* verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||
*/
|
||
export class ReleaseJudge {
|
||
private stopped = false
|
||
|
||
stop() {
|
||
this.stopped = true
|
||
}
|
||
|
||
async run() {
|
||
while (!this.stopped) {
|
||
const started = Date.now()
|
||
try {
|
||
if (await this.fullDue()) {
|
||
const round = await this.full()
|
||
if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
|
||
} else {
|
||
const round = await this.changed()
|
||
if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
|
||
}
|
||
} catch (error: any) {
|
||
logger.error(`Judge round failed: ${error.message}`)
|
||
}
|
||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||
}
|
||
}
|
||
|
||
/** A full round is due: a day after the last, or the last was by another rule set. */
|
||
async fullDue(): Promise<boolean> {
|
||
const state = await db.from('judge_state').first()
|
||
return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
|
||
}
|
||
|
||
/**
|
||
* Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
|
||
* half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
|
||
*/
|
||
async full(dry = false): Promise<JudgeRound> {
|
||
const started = new Date()
|
||
const round: JudgeRound = { rules: {} }
|
||
// Verdicts of an older rule set are judged again from scratch.
|
||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||
await db.rawQuery(`TRUNCATE judge_full`)
|
||
for (const rule of RULES) {
|
||
if (this.stopped) return round
|
||
const { rowCount } = await db.rawQuery(
|
||
`INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||
[rule.name]
|
||
)
|
||
round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
|
||
}
|
||
if (dry || this.stopped) return round
|
||
await db.rawQuery(`ANALYZE judge_full`)
|
||
await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
|
||
await db.rawQuery(
|
||
`INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
|
||
[started, JUDGE_VERSION]
|
||
)
|
||
return round
|
||
}
|
||
|
||
/**
|
||
* The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
|
||
* A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
|
||
* already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
|
||
* file in that many without them had its releases judged before.
|
||
*/
|
||
async changed(): Promise<JudgeRound> {
|
||
const round: JudgeRound = { releases: 0, rules: {} }
|
||
for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
|
||
let more = true
|
||
while (more && !this.stopped) {
|
||
await db.transaction(async (trx) => {
|
||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
|
||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||
await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||
const { rows } = await trx.rawQuery(
|
||
`WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
|
||
scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
|
||
SELECT (SELECT count(*) FROM taken) AS taken`
|
||
)
|
||
const taken = Number(rows[0].taken)
|
||
more = taken === CHUNK
|
||
if (taken === 0) return
|
||
// The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
|
||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
|
||
await trx.rawQuery(
|
||
`CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
|
||
)
|
||
await trx.rawQuery(
|
||
`INSERT INTO judge_tipped
|
||
SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
|
||
FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
|
||
CROSS JOIN LATERAL (
|
||
SELECT count(DISTINCT f.release_id) AS total,
|
||
count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
|
||
FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
|
||
WHERE c.total >= ${REPEATED_RELEASES}`
|
||
)
|
||
await trx.rawQuery(
|
||
`INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
|
||
WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
|
||
ON CONFLICT DO NOTHING`
|
||
)
|
||
await trx.rawQuery(`ANALYZE judge_scope`)
|
||
const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
|
||
round.releases! += Number(scope[0].n)
|
||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
|
||
for (const rule of RULES) {
|
||
const { rowCount } = await trx.rawQuery(
|
||
`INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||
[rule.name]
|
||
)
|
||
round.rules[rule.name].matched += rowCount ?? 0
|
||
}
|
||
await trx.rawQuery(`ANALYZE judged`)
|
||
await this.apply(trx, 'judged', true, round)
|
||
})
|
||
}
|
||
return round
|
||
}
|
||
|
||
/**
|
||
* The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
|
||
* more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
|
||
* insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
|
||
*/
|
||
private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
|
||
const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
|
||
for (const { rule, n } of rows) {
|
||
round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
|
||
round.rules[rule][field] += Number(n)
|
||
}
|
||
}
|
||
const { rows: removed } = await trx.rawQuery(
|
||
`WITH gone AS (DELETE FROM release_verdicts v
|
||
WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
|
||
NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
|
||
RETURNING v.rule)
|
||
SELECT rule, count(*) AS n FROM gone GROUP BY 1`
|
||
)
|
||
tally(removed, 'removed')
|
||
const { rows: added } = await trx.rawQuery(
|
||
`WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||
SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
|
||
WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
|
||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||
WHERE release_verdicts.rule <> excluded.rule
|
||
RETURNING rule)
|
||
SELECT rule, count(*) AS n FROM written GROUP BY 1`,
|
||
[JUDGE_VERSION]
|
||
)
|
||
tally(added, 'added')
|
||
}
|
||
}
|
||
|
||
/** "dead 1,157,018 (+1,136 −712), …" for the log. */
|
||
function describe(round: JudgeRound): string {
|
||
return Object.entries(round.rules)
|
||
.map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
|
||
.join(', ')
|
||
}
|