import db from '@adonisjs/lucid/services/db' import { decodeChunk, type ChunkFile } from '#lib/chunks/index' /** Candidates looked at per round. */ const BATCH = 2000 /** An upload's sidecar is posted with it: its release is looked for this far from it. */ const WINDOW_MINUTES = 10 /** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */ const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i /** The words of a name worth comparing: 4+ letters or digits. */ function words(text: string): Set { return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4)) } export interface Sibling { id: number name: string files: number file_total: number | null } /** * Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject * ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one * under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that * isn't one release. */ export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null { const said = words(files.map((f) => f.subject).join(' ')) const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w))) if (named.length === 1) return named[0] if (named.length > 1) return null const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : [] return counted.length === 1 ? counted[0] : null } export interface FoldRound { looked: number sidecars: number folded: number } /** * The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another * counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small * settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in * the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via * 'sidecar'). The rest leave the queue untouched. */ export class SidecarFolder { constructor(private merge: (trx: any, found: Map, key: string, via: 'sidecar') => Promise) {} async round(): Promise { const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 } const { rows: queued } = await db.rawQuery( `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`, [BATCH] ) const { rows: stale } = await db.rawQuery( `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`, [BATCH] ) const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id)) for (const c of queued) { round.looked++ done.push(Number(c.id)) const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id]) const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload)) if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue round.sidecars++ if (!c.posted_at) continue const { rows: siblings } = await db.rawQuery( `SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?) AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`, [c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files] ) const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) }))) if (!sibling) continue const full = siblings.find((s: any) => Number(s.id) === sibling.id) const merged = await db.transaction((trx) => this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar') ) if (merged > 0) round.folded++ } if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete() return round } }