158 lines
7.7 KiB
TypeScript
158 lines
7.7 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
|
||
/** Raised when a rule changes: verdicts of an older version are judged again. */
|
||
export const JUDGE_VERSION = 1
|
||
/** Between rounds: each rule is a scan of its table. */
|
||
const ROUND_MS = 60 * 60 * 1000
|
||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||
const REPEATED_RELEASES = 20
|
||
|
||
/**
|
||
* A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
|
||
* these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
|
||
* 2026-10-16 (64)), so {0,1}.
|
||
*/
|
||
const MEDIA =
|
||
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
||
/** The posters of scraped celebrity pictures. */
|
||
const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
|
||
/** An executable's name. */
|
||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||
|
||
export interface Rule {
|
||
name: string
|
||
/** Why, in a few words (the log, the web interface). */
|
||
says: string
|
||
/** SELECT of the release ids the rule judges. */
|
||
sql: string
|
||
}
|
||
|
||
/**
|
||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||
* the header pass, the prober, the merges and the spots know by now.
|
||
*/
|
||
export const RULES: Rule[] = [
|
||
{
|
||
name: 'dead',
|
||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||
sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
|
||
},
|
||
{
|
||
name: 'fake',
|
||
says: 'named as a film, episode or music, but the content is a program',
|
||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||
// an executable by name or by its first bytes, under a media name.
|
||
sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
|
||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||
UNION
|
||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||
},
|
||
{
|
||
name: 'repeated',
|
||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||
},
|
||
{
|
||
name: 'scraped',
|
||
says: 'celebrity pictures scraped from the web, reposted by one spam family',
|
||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||
sql: `SELECT r.id FROM releases r
|
||
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||
},
|
||
{
|
||
name: 'locked',
|
||
says: 'passworded, and no spot, NZB or title gives the password',
|
||
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||
},
|
||
{
|
||
name: 'unusable',
|
||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling',
|
||
sql: `SELECT r.id FROM releases r
|
||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||
},
|
||
]
|
||
|
||
export interface JudgeRound {
|
||
/** Per rule: releases it judges now, verdicts added, verdicts taken back. */
|
||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||
}
|
||
|
||
/**
|
||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
|
||
* round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
|
||
* matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||
*/
|
||
export class ReleaseJudge {
|
||
private stopped = false
|
||
|
||
stop() {
|
||
this.stopped = true
|
||
}
|
||
|
||
async run() {
|
||
while (!this.stopped) {
|
||
const started = Date.now()
|
||
try {
|
||
const round = await this.round()
|
||
const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
|
||
logger.info(`Judged: ${parts.join(', ')}.`)
|
||
} catch (error: any) {
|
||
logger.error(`Judge round failed: ${error.message}`)
|
||
}
|
||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||
}
|
||
}
|
||
|
||
/** One round over every rule; with dry, only counts. */
|
||
async round(dry = false): Promise<JudgeRound> {
|
||
const round: JudgeRound = { rules: {} }
|
||
// Verdicts of an older rule set are judged again from scratch.
|
||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||
const judged = new Set<string>()
|
||
for (const rule of RULES) {
|
||
if (this.stopped) break
|
||
const { rows } = await db.rawQuery(rule.sql)
|
||
const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
|
||
for (const id of ids) judged.add(id)
|
||
const stats = { matched: ids.length, added: 0, removed: 0 }
|
||
if (!dry) {
|
||
await db.transaction(async (trx) => {
|
||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||
for (let i = 0; i < ids.length; i += 50_000) {
|
||
await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
|
||
}
|
||
const removed = await trx.rawQuery(
|
||
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
|
||
[rule.name]
|
||
)
|
||
const added = await trx.rawQuery(
|
||
`INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
|
||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||
WHERE release_verdicts.rule <> excluded.rule`,
|
||
[rule.name, JUDGE_VERSION]
|
||
)
|
||
stats.removed = removed.rowCount ?? 0
|
||
stats.added = added.rowCount ?? 0
|
||
})
|
||
}
|
||
round.rules[rule.name] = stats
|
||
}
|
||
return round
|
||
}
|
||
}
|