diff --git a/REBUILD.md b/REBUILD.md index 3327149..57afb44 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -341,8 +341,13 @@ or music name whose content is a program: archive entries all executables, or the file itself one by name or by its first bytes, sniffed `exe`), *repeated* (the same first 16 KiB and size in 20+ releases), *locked* (passworded, no password from a spot, its NZB or the title), -*unusable* (random names, no inner name, no par2 or inner-name merge, no -spot, a day after settling). Verdicts (`release_verdicts`, with the rule +*tiny* (one file of one article, 2–100 kB, under a readable release-like +name outside the music, e-book and comic groups, without an e-book, +subtitle, crack or sidecar word: the catalog's side of the edge's +tiny-archive rule, 1.5M rows of the 2015–17 flood that the edge indexed +before the rule; LOG.md (131), (135)), *unusable* (random names, no inner +name, no par2 or inner-name merge, no spot, a day after settling). +Verdicts (`release_verdicts`, with the rule set's version) hide releases from search (`judged` shows them); a verdict whose rule no longer matches is taken back. Nothing is dropped yet. diff --git a/app/services/release_judge.ts b/app/services/release_judge.ts index 79e7901..938c961 100644 --- a/app/services/release_judge.ts +++ b/app/services/release_judge.ts @@ -1,6 +1,7 @@ import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import { SOURCES } from '#lib/release/source' +import { TINY_MIN_BYTES, NONSENSE_MIN_BYTES } from '#lib/rules/index' /** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */ export const JUDGE_VERSION = 1 @@ -28,6 +29,14 @@ const MEDIA = const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes` /** An executable's name. */ const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$` +/** + * The tiny rule's name tests, lib/rules/rule_set.ts's RELEASE_LIKE_SPACED, EBOOK, TINY_CONTENT and + * STUB_EXEMPT_GROUPS in Postgres's dialect (and no "?", see MEDIA). + */ +const TINY_RELEASE_LIKE = `(\\y(19|20)[0-9][0-9]\\y|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-{0,1}dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\\.[0-9]+)+)\\y|-[a-z0-9]{2,}$|\\.[a-z0-9]+\\.[a-z0-9]+\\.)` +const TINY_EBOOK = `\\y(epub|mobi|azw3{0,1}|pdf|djvu|cbr|cbz|e-{0,1}books{0,1})\\y` +const TINY_CONTENT = `\\y(subs{0,1}|subtitles{0,1}|subpack|srt|untertitel|ondertitels{0,1}|ondertiteling|legendas{0,1}|napisy|sottotitoli|cracks{0,1}|cracked|keygen|keymaker|trainer|patch|patched|serials{0,1}|activator|loader|nocd|no-cd|fix|covers{0,1}|artwork|booklet|scans{0,1}|proof|sample|info|readme)\\y` +const TINY_EXEMPT_GROUPS = `(sound|mp3|karaoke|music|audio|flac|lossless|e-{0,1}book|epub|comic|guitar|font|magazine)` export interface Rule { name: string @@ -100,6 +109,20 @@ export const RULES: Rule[] = [ // releases_passworded_idx: a scan of all of releases was 30 s a round. sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`, }, + { + name: 'tiny', + says: 'one file of one article, 2–100 kB, under a release-like name: the exe-stub flood in miniature', + // The catalog's side of the edge's tiny-archive rule (lib/rules/rule_set.ts isTinyArchive; LOG.md (131), + // (135)): the same name tests in Postgres's regex dialect ("\y" for "\b", "{0,1}" for "?"), the readable flag + // for its scrambled and random tests, the group set for its groups. The 2015–17 flood that the edge indexed + // before the rule: 1.5M rows, 98% of them with an archive candidate, the rest lone par2 files and scrambled + // names, so no join on the candidates. A scan of releases (9 GB) a full round. + sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} + WHERE r.files = 1 AND r.segments = 1 AND r.bytes >= ${TINY_MIN_BYTES} AND r.bytes < ${NONSENSE_MIN_BYTES} + AND r.readable AND NOT coalesce(r.sidecar_only, false) + AND r.name ~* '${TINY_RELEASE_LIKE}' AND r.name !~* '${TINY_EBOOK}' AND r.name !~* '${TINY_CONTENT}' + AND NOT EXISTS (SELECT 1 FROM group_sets gs WHERE gs.id = r.group_set_id AND gs.groups::text ~* '${TINY_EXEMPT_GROUPS}')`, + }, { name: 'unusable', says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final', diff --git a/tests/functional/release_judge.spec.ts b/tests/functional/release_judge.spec.ts index 9aa5da5..152e51c 100644 --- a/tests/functional/release_judge.spec.ts +++ b/tests/functional/release_judge.spec.ts @@ -163,6 +163,55 @@ test.group('ReleaseJudge', (group) => { assert.isNull(await db.from('release_verdicts').where('release_id', locked).first()) }) + test('judges tiny single-article files under release-like names', async ({ assert }) => { + let n = 0 + const settled = new Date(Date.now() - 3 * 86400_000) + const release = async (name: string, extra: Record = {}) => { + n++ + const key = `jt${String(n).padStart(6, '0')}`.padEnd(40, '0') + return Number( + await insertRelease({ + release_key: key, + identity_key: key, + name, + source: 'stem', + groups: ['alt.binaries.boneless'], + files: 1, + segments: 1, + bytes: 15_000, + posted_at: settled, + readable: true, + settled_at: settled, + ...extra, + }) + ) + } + const dotted = await release('Velvet.S02E20.GERMAN.720p.HDTV.x264-ACED') + const spaced = await release('Pacific Rim (2013) BDRip DVDr-sailo1') + const subs = await release('Some.Film.2019.German.Subs-XYZ') + const ebook = await release('Some.Book.2019.epub-GRP') + const music = await release('Some.Album.2019.FLAC-GRP', { groups: ['alt.binaries.sounds.flac'] }) + const big = await release('Big.Film.2019.1080p-GRP', { bytes: 500_000 }) + const twoFiles = await release('Two.Films.2019.1080p-GRP', { files: 2, segments: 2 }) + const plain = await release('Holiday photo') + const sidecar = await release('Side.Car.2019.1080p-GRP', { sidecar_only: true }) + const unreadable = await release('Xk9v.2019.Qpz-ab', { readable: false }) + + const round = await new ReleaseJudge().full() + const verdicts = new Map( + (await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [ + Number(v.release_id), + v.rule, + ]) + ) + assert.equal(verdicts.get(dotted), 'tiny') + assert.equal(verdicts.get(spaced), 'tiny') + for (const id of [subs, ebook, music, big, twoFiles, plain, sidecar, unreadable]) { + assert.notEqual(verdicts.get(id), 'tiny') + } + assert.equal(round.rules.tiny.added, 2) + }) + test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({ assert, }) => {