The junk judge: releases judged by what their probes showed; verdicts hide them
Voyager 1's second filter (D42) for voyager 2. Hourly, in order (the first rule wins): dead (every probed first article gone), fake (a film, episode or music name whose content is a program: archive entries all executables, or the file itself by name or sniffed exe), repeated (the same first 16 KiB and size in 20+ releases), locked (passworded, no password known), unusable (random names, no inner name, no merge, no spot, a day after settling). Verdicts in release_verdicts (rule, version) hide releases from search (judged shows them); a verdict whose rule no longer matches is taken back. Nothing dropped yet. Service judge; releases:judge --once --dry counts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
99b72ddd8f
commit
5897d595cf
9 files changed
+324
-3
No files matched your search
@@ -0,0 +1,142 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again. */
|
||||
export const JUDGE_VERSION = 1
|
||||
/** Between rounds: each rule is a scan of its table. */
|
||||
const ROUND_MS = 60 * 60 * 1000
|
||||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||||
const REPEATED_RELEASES = 20
|
||||
|
||||
/** A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. */
|
||||
const MEDIA =
|
||||
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]?26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]?dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
||||
/** An executable's name. */
|
||||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||||
|
||||
export interface Rule {
|
||||
name: string
|
||||
/** Why, in a few words (the log, the web interface). */
|
||||
says: string
|
||||
/** SELECT of the release ids the rule judges. */
|
||||
sql: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||||
* the header pass, the prober, the merges and the spots know by now.
|
||||
*/
|
||||
export const RULES: Rule[] = [
|
||||
{
|
||||
name: 'dead',
|
||||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||||
sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
|
||||
},
|
||||
{
|
||||
name: 'fake',
|
||||
says: 'named as a film, episode or music, but the content is a program',
|
||||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||||
// an executable by name or by its first bytes, under a media name.
|
||||
sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||||
UNION
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||||
},
|
||||
{
|
||||
name: 'repeated',
|
||||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||||
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||||
},
|
||||
{
|
||||
name: 'locked',
|
||||
says: 'passworded, and no spot, NZB or title gives the password',
|
||||
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling',
|
||||
sql: `SELECT r.id FROM releases r
|
||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
]
|
||||
|
||||
export interface JudgeRound {
|
||||
/** Per rule: releases it judges now, verdicts added, verdicts taken back. */
|
||||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||||
}
|
||||
|
||||
/**
|
||||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
|
||||
* round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
|
||||
* matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
*/
|
||||
export class ReleaseJudge {
|
||||
private stopped = false
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
async run() {
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
const round = await this.round()
|
||||
const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
|
||||
logger.info(`Judged: ${parts.join(', ')}.`)
|
||||
} catch (error: any) {
|
||||
logger.error(`Judge round failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
/** One round over every rule; with dry, only counts. */
|
||||
async round(dry = false): Promise<JudgeRound> {
|
||||
const round: JudgeRound = { rules: {} }
|
||||
// Verdicts of an older rule set are judged again from scratch.
|
||||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||||
const judged = new Set<string>()
|
||||
for (const rule of RULES) {
|
||||
if (this.stopped) break
|
||||
const { rows } = await db.rawQuery(rule.sql)
|
||||
const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
|
||||
for (const id of ids) judged.add(id)
|
||||
const stats = { matched: ids.length, added: 0, removed: 0 }
|
||||
if (!dry) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
for (let i = 0; i < ids.length; i += 50_000) {
|
||||
await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
|
||||
}
|
||||
const removed = await trx.rawQuery(
|
||||
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
|
||||
[rule.name]
|
||||
)
|
||||
const added = await trx.rawQuery(
|
||||
`INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule`,
|
||||
[rule.name, JUDGE_VERSION]
|
||||
)
|
||||
stats.removed = removed.rowCount ?? 0
|
||||
stats.added = added.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
round.rules[rule.name] = stats
|
||||
}
|
||||
return round
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
|
||||
export interface SearchParams {
|
||||
q?: string
|
||||
/** Also releases the junk judge hid (release_verdicts). */
|
||||
judged?: boolean
|
||||
/** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
|
||||
readable?: boolean
|
||||
/** Only releases known to have all their files. */
|
||||
@@ -80,6 +82,8 @@ function contains(word: string): string {
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
const where: string[] = ['r.files > 0']
|
||||
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
||||
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
||||
const bindings: unknown[] = []
|
||||
// Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
|
||||
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
||||
|
||||
Reference in new issue
Block a user