From d773c91ad01815de9bdf93a55f9627f30afefbad Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 00:08:44 +0200 Subject: [PATCH] Rebuild: phase 1 modules from the current app, ordered by pipeline stage lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 --- lib/chunks/codec.ts | 45 +++++ lib/chunks/index.ts | 1 + lib/files/aggregator.ts | 86 ++++++++++ lib/files/index.ts | 2 + lib/files/volumes.ts | 45 +++++ lib/headers/file_key.ts | 47 ++++++ lib/headers/index.ts | 3 + lib/headers/segment.ts | 72 ++++++++ lib/headers/subject.ts | 59 +++++++ lib/nntp/group.ts | 23 +++ lib/nntp/index.ts | 3 + lib/nntp/xover_client.ts | 238 +++++++++++++++++++++++++++ lib/nntp/xref.ts | 13 ++ lib/release/identity.ts | 223 +++++++++++++++++++++++++ lib/release/index.ts | 5 + lib/release/random.ts | 27 +++ lib/rules/index.ts | 2 + lib/rules/rule_set.ts | 69 ++++++++ lib/rules/upload_tools.ts | 52 ++++++ tests/unit/chunks/codec.spec.ts | 18 ++ tests/unit/files/aggregator.spec.ts | 39 +++++ tests/unit/headers/subject.spec.ts | 65 ++++++++ tests/unit/nntp/group.spec.ts | 18 ++ tests/unit/nntp/xover_client.spec.ts | 106 ++++++++++++ tests/unit/release/identity.spec.ts | 169 +++++++++++++++++++ tests/unit/rules/rule_set.spec.ts | 56 +++++++ 26 files changed, 1486 insertions(+) create mode 100644 lib/chunks/codec.ts create mode 100644 lib/chunks/index.ts create mode 100644 lib/files/aggregator.ts create mode 100644 lib/files/index.ts create mode 100644 lib/files/volumes.ts create mode 100644 lib/headers/file_key.ts create mode 100644 lib/headers/index.ts create mode 100644 lib/headers/segment.ts create mode 100644 lib/headers/subject.ts create mode 100644 lib/nntp/group.ts create mode 100644 lib/nntp/index.ts create mode 100644 lib/nntp/xover_client.ts create mode 100644 lib/nntp/xref.ts create mode 100644 lib/release/identity.ts create mode 100644 lib/release/index.ts create mode 100644 lib/release/random.ts create mode 100644 lib/rules/index.ts create mode 100644 lib/rules/rule_set.ts create mode 100644 lib/rules/upload_tools.ts create mode 100644 tests/unit/chunks/codec.spec.ts create mode 100644 tests/unit/files/aggregator.spec.ts create mode 100644 tests/unit/headers/subject.spec.ts create mode 100644 tests/unit/nntp/group.spec.ts create mode 100644 tests/unit/nntp/xover_client.spec.ts create mode 100644 tests/unit/release/identity.spec.ts create mode 100644 tests/unit/rules/rule_set.spec.ts diff --git a/lib/chunks/codec.ts b/lib/chunks/codec.ts new file mode 100644 index 0000000..f51afdf --- /dev/null +++ b/lib/chunks/codec.ts @@ -0,0 +1,45 @@ +import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib' + +/** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */ +export interface ChunkFile { + name: string + poster: string + subject: string + /** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */ + postedAt: number | null + groups: string[] + segments: [number, string, number][] +} + +const VERSION = 1 + +/** + * A release chunk's payload (REBUILD.md, chunk store): a version byte, then + * zstd of a JSON line with the files (name, poster, subject, date, groups, + * sizes, and part numbers unless they are 1..N) followed by one message-id + * per line, in file order. Message-ids are mostly random, so this is close + * to what plain compression gets (LOG.md 2026-10-08 (3)). + */ +export function encodeChunk(files: ChunkFile[]): Buffer { + const head = files.map((f) => { + const sequential = f.segments.every(([n], i) => n === i + 1) + return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) } + }) + const ids = files.flatMap((f) => f.segments.map(([, id]) => id)) + const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8') + return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })]) +} + +export function decodeChunk(data: Buffer): ChunkFile[] { + if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`) + const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n') + let at = 0 + return (JSON.parse(head) as any[]).map((f) => ({ + name: f.n, + poster: f.p, + subject: f.j, + postedAt: f.d, + groups: f.g, + segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]), + })) +} diff --git a/lib/chunks/index.ts b/lib/chunks/index.ts new file mode 100644 index 0000000..d46b6f1 --- /dev/null +++ b/lib/chunks/index.ts @@ -0,0 +1 @@ +export { encodeChunk, decodeChunk, type ChunkFile } from './codec.js' diff --git a/lib/files/aggregator.ts b/lib/files/aggregator.ts new file mode 100644 index 0000000..e25b56b --- /dev/null +++ b/lib/files/aggregator.ts @@ -0,0 +1,86 @@ +/** + * Edge aggregation (CONCEPT.md D37): groups header segments into files in + * memory, so a seeding node writes one row per file instead of one per + * article. A file is done once every part number from 1 to its total has + * an article; what never completes within the node's window (an upload cut + * by the range edges, a random subject per article) leaves as fragments + * for the regular ingest path, where it can meet its other parts. + */ + +/** What the aggregator needs of a segment (IngestSegment has all of it). */ +export interface AggregatableSegment { + fileKey: string + number: number + totalParts: number + messageId: string + bytes: number +} + +export interface AggregatedFile { + fileKey: string + /** One segment per part number, in part order; the first article seen wins (a later one is a repost). */ + segments: S[] +} + +interface OpenFile { + parts: Map + totalParts: number + /** Batch in which a segment last arrived. */ + touched: number +} + +export class FileAggregator { + private open = new Map>() + private batch = 0 + /** Segments with a part number outside 1..total: never part of a complete file. */ + private stray: S[] = [] + + /** Adds one batch of segments; returns the files they completed. */ + add(segments: S[]): AggregatedFile[] { + this.batch++ + const touched = new Set() + for (const segment of segments) { + if (segment.number < 1 || segment.number > segment.totalParts) { + this.stray.push(segment) + continue + } + let file = this.open.get(segment.fileKey) + if (!file) { + file = { parts: new Map(), totalParts: segment.totalParts, touched: this.batch } + this.open.set(segment.fileKey, file) + } + if (!file.parts.has(segment.number)) file.parts.set(segment.number, segment) + file.touched = this.batch + touched.add(segment.fileKey) + } + + const complete: AggregatedFile[] = [] + for (const key of touched) { + const file = this.open.get(key)! + if (file.parts.size < file.totalParts) continue + this.open.delete(key) + complete.push({ fileKey: key, segments: [...file.parts.values()].sort((a, b) => a.number - b.number) }) + } + return complete + } + + /** + * Gives up on files that got no segment in the last `idleBatches` batches + * (all open files when 0) and returns their segments, plus stray ones. + */ + evict(idleBatches = 0): S[] { + const out = this.stray + this.stray = [] + for (const [key, file] of this.open) { + if (idleBatches > 0 && this.batch - file.touched < idleBatches) continue + this.open.delete(key) + out.push(...file.parts.values()) + } + return out + } + + /** Files still waiting for parts. */ + get openFiles(): number { + return this.open.size + } +} diff --git a/lib/files/index.ts b/lib/files/index.ts new file mode 100644 index 0000000..9956f25 --- /dev/null +++ b/lib/files/index.ts @@ -0,0 +1,2 @@ +export { FileAggregator, type AggregatableSegment, type AggregatedFile } from './aggregator.js' +export { isArchiveStart, isLaterVolume, rarVolume, sevenZipVolume } from './volumes.js' diff --git a/lib/files/volumes.ts b/lib/files/volumes.ts new file mode 100644 index 0000000..7cf1947 --- /dev/null +++ b/lib/files/volumes.ts @@ -0,0 +1,45 @@ +/** + * Files whose first segment says what an archive holds: named first RAR + * volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension + * uploaders use for archive volumes too (sniffed by content). + */ +export function isArchiveStart(name: string): boolean { + if (/\.bin$/i.test(name)) return true + const volume = sevenZipVolume(name) + if (volume === 0 || volume === 1) return true + return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) +} + +/** + * A later RAR volume (.part02.rar, .r00) or a par2 recovery volume + * (.vol03+04.par2): the volume number is readable even under a random stem. + * The set is listed from its first volume and named from its index .par2, + * so the first segment of these says nothing new (LOG.md 2026-10-04). + */ +export function isLaterVolume(name: string): boolean { + const part = /\.part(\d+)\.rar$/i.exec(name) + if (part) return Number(part[1]) > 1 + return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name) +} + +/** + * The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of + * `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for + * other names. + */ +export function rarVolume(name: string): { base: string; number: number } | null { + const part = /^(.*)\.part(\d+)\.rar$/i.exec(name) + if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) } + const old = /^(.*)\.r(\d{2,3})$/i.exec(name) + if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 } + const first = /^(.*)\.rar$/i.exec(name) + if (first) return { base: first[1].toLowerCase(), number: 1 } + return null +} + +/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */ +export function sevenZipVolume(name: string): number | null { + const split = /\.7z\.(\d{1,4})$/i.exec(name) + if (split) return Number(split[1]) + return /\.7z$/i.test(name) ? 0 : null +} diff --git a/lib/headers/file_key.ts b/lib/headers/file_key.ts new file mode 100644 index 0000000..56f2061 --- /dev/null +++ b/lib/headers/file_key.ts @@ -0,0 +1,47 @@ +import { createHash } from 'node:crypto' + +/** + * Identity of one uploaded file: every segment of it has the same poster, the + * same subject apart from the segment counter, and the same segment count. + * The group is deliberately left out so crossposts resolve to one file. + * See CONCEPT.md (D12). + */ +export function fileKey(poster: string, base: string, totalSegments: number): string { + return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex') +} + +/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */ +export function looksLikeFilename(name: string): boolean { + return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name) +} + +/** + * Identity of a file known only from its yEnc header (obfuscated posts). + * The poster is left out: obfuscating posters randomise it per article. + * - A real filename is stable across the articles of a file, and needed to + * tell apart the equal-sized volumes of one archive. + * - A random, extensionless name can change per article too; the exact + * byte size and part count are what stays the same. + * Collisions are caught by the conflict check in IngestService. + */ +export function weakFileKey(name: string, size: number, totalParts: number): string { + const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape' + return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex') +} + +/** + * Identity of a file by its counters, for uploads that give every article + * its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`, + * part count and posting day stay the same across the articles of one file. + * The day keeps different uploads with the same shape apart; collisions that + * remain are caught by the conflict check (these files are weak identities). + * Null when the subject has no file counter. + */ +export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null { + const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject) + if (!counter || segment.totalParts < 2) return null + const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : '' + return createHash('sha1') + .update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`) + .digest('hex') +} diff --git a/lib/headers/index.ts b/lib/headers/index.ts new file mode 100644 index 0000000..64392ac --- /dev/null +++ b/lib/headers/index.ts @@ -0,0 +1,3 @@ +export { parseSubject, type ParsedSubject } from './subject.js' +export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js' +export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js' diff --git a/lib/headers/segment.ts b/lib/headers/segment.ts new file mode 100644 index 0000000..db97f15 --- /dev/null +++ b/lib/headers/segment.ts @@ -0,0 +1,72 @@ +import { parseXrefGroups } from '../nntp/xref.js' +import { fileKey } from './file_key.js' +import { parseSubject } from './subject.js' +import { releaseIdentity, type ReleaseIdentity } from '../release/index.js' + +/** One segment, with the identity of the file it belongs to already resolved. */ +export interface IngestSegment { + fileKey: string + poster: string + /** Subject without segment counter. */ + subject: string + name: string + totalParts: number + number: number + /** Without angle brackets. */ + messageId: string + bytes: number + articleNumber?: bigint | number + postedAt?: Date + groups: string[] + /** + * The identity comes from the yEnc name only (obfuscated posts). Such names + * can be shared by several equal-sized files of one post, so a second + * article for the same part number marks the binary as conflicted. + */ + weakIdentity?: boolean + /** Release (download) this file belongs to, when it can be determined. */ + release?: ReleaseIdentity | null + /** Exact file size from =ybegin (body-derived segments only). */ + size?: number + /** yEnc CRCs (body-derived segments only), to verify split volumes. */ + pcrc32?: string + partSize?: number + fileCrc32?: string + /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */ + md5_16k?: string +} + +/** An XOVER overview entry as returned by nntp-js. */ +export type OverviewHeader = Record + +function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) { + const xrefGroups = parseXrefGroups(header.xref) + return { + poster: header.from ?? '', + messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), + bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0, + articleNumber, + postedAt: header.date ? new Date(header.date) : undefined, + groups: xrefGroups.length > 0 ? xrefGroups : [groupName], + } +} + +/** Builds a segment from the subject line; null when the subject is obfuscated. */ +export function segmentFromHeader( + header: OverviewHeader, + groupName: string, + articleNumber?: bigint | number +): IngestSegment | null { + const parsed = header.subject ? parseSubject(header.subject) : null + if (!parsed || !header['message-id']) return null + const base = common(header, groupName, articleNumber) + return { + ...base, + fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments), + subject: parsed.base, + name: parsed.name, + totalParts: parsed.totalSegments, + number: parsed.segment, + release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }), + } +} diff --git a/lib/headers/subject.ts b/lib/headers/subject.ts new file mode 100644 index 0000000..8304476 --- /dev/null +++ b/lib/headers/subject.ts @@ -0,0 +1,59 @@ +export interface ParsedSubject { + /** Filename from the quoted part of the subject. */ + name: string + /** Segment number `k` from `(k/N)`. */ + segment: number + /** Segment count `N` from `(k/N)`. */ + totalSegments: number + /** File counter `[i/M]`, when present. */ + fileIndex?: number + fileTotal?: number + /** + * The subject with the segment counter removed. Identical for every segment + * of one uploaded file, so it is part of the file identity (see fileKey). + */ + base: string +} + +const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g +const FILE_COUNTER = /\[(\d+)\/(\d+)\]/ + +/** + * Parses a binary post subject such as + * `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`. + * + * Returns null for subjects without a quoted filename and segment counter + * (obfuscated posts), which need their yEnc header read from the body instead. + */ +export function parseSubject(subject: string): ParsedSubject | null { + const nameEnd = subject.lastIndexOf('"') + const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1 + if (nameStart === -1) return null + const name = subject.slice(nameStart + 1, nameEnd) + if (!name) return null + + // The segment counter is the last "(k/N)" after the filename; a file size may follow it. + let counter: RegExpExecArray | undefined + for (const match of subject.matchAll(SEGMENT_COUNTER)) { + if (match.index > nameEnd) counter = match + } + if (!counter) return null + + const segment = Number.parseInt(counter[1], 10) + const totalSegments = Number.parseInt(counter[2], 10) + if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null + + const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)) + .replace(/\s+/g, ' ') + .trim() + + const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart)) + return { + name, + segment, + totalSegments, + fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined, + fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined, + base, + } +} diff --git a/lib/nntp/group.ts b/lib/nntp/group.ts new file mode 100644 index 0000000..24785c8 --- /dev/null +++ b/lib/nntp/group.ts @@ -0,0 +1,23 @@ +export interface GroupInfo { + count: bigint + first: bigint + last: bigint + name: string +} + +/** + * Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1). + * + * nntp-js's own `group()` destructures this reply without skipping the status + * code, so its `first`/`last` are off by one field. Pass it `result.resp`. + */ +export function parseGroupResponse(resp: string): GroupInfo { + const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim()) + if (!match) throw new Error(`Unexpected GROUP response: ${resp}`) + return { + count: BigInt(match[1]), + first: BigInt(match[2]), + last: BigInt(match[3]), + name: match[4], + } +} diff --git a/lib/nntp/index.ts b/lib/nntp/index.ts new file mode 100644 index 0000000..c80c36b --- /dev/null +++ b/lib/nntp/index.ts @@ -0,0 +1,3 @@ +export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js' +export { parseXrefGroups } from './xref.js' +export { parseGroupResponse, type GroupInfo } from './group.js' diff --git a/lib/nntp/xover_client.ts b/lib/nntp/xover_client.ts new file mode 100644 index 0000000..e592cee --- /dev/null +++ b/lib/nntp/xover_client.ts @@ -0,0 +1,238 @@ +import { promisify } from 'node:util' +import net from 'node:net' +import tls from 'node:tls' +import { deflate, deflateSync, unzipSync } from 'node:zlib' + +const deflateAsync = promisify(deflate) + +export interface XoverClientOptions { + host: string + port: number + user?: string + password?: string + secure?: boolean + /** Hostname the TLS certificate must match, when connecting through a tunnel (default: host). */ + servername?: string + /** A reply with no data for this long fails, and the connection with it (inactivity, not duration). */ + timeoutMs?: number + /** Ask for gzip-compressed overview data (XFEATURE COMPRESS GZIP), when the server has it. */ + compress?: boolean +} + +export type Overview = [number, Record] + +/** The overview fields in RFC 3977 order, named as nntp-js names them (segmentFromHeader reads these). */ +const FIELDS = ['subject', 'from', 'date', 'message-id', 'references', ':bytes', ':lines', 'xref'] +const TERMINATOR = Buffer.from('\r\n.\r\n') +const DOT_LINE = Buffer.from('.\r\n') + +/** + * A minimal NNTP client for seeding nodes (CONCEPT.md D37): one connection, + * GROUP and XOVER only, with gzip-compressed overviews. nntp-js can't read + * those: after "224 … [COMPRESS=GZIP]" the lines come as one zlib stream, + * followed by a plain "." line; an empty range is a 420 without body. + * About six times less traffic per header (LOG.md 2026-10-05). + */ +export class XoverClient { + private chunks: Buffer[] = [] + private length = 0 + private wake?: () => void + private failure?: Error + compressed = false + + private constructor( + private sock: net.Socket, + private timeoutMs: number + ) { + sock.on('data', (chunk: Buffer) => { + this.chunks.push(chunk) + this.length += chunk.length + this.wake?.() + }) + const fail = (error: Error) => { + this.failure ??= error + this.wake?.() + } + sock.on('error', fail) + sock.on('close', () => fail(new Error('NNTP connection closed'))) + sock.on('timeout', () => { + fail(new Error('NNTP reply timed out')) + sock.destroy() + }) + } + + static async connect(options: XoverClientOptions): Promise { + const sock = options.secure + ? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host }) + : net.connect({ host: options.host, port: options.port }) + const client = new XoverClient(sock, options.timeoutMs ?? 120_000) + sock.setTimeout(client.timeoutMs) + try { + await client.expect(await client.line(), ['200', '201']) + if (options.user) { + const reply = await client.command(`AUTHINFO USER ${options.user}`) + if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281']) + else await client.expect(reply, ['281']) + } + if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290') + } catch (error) { + client.close() + throw error + } + // Idle between commands is fine (a node waiting for the seed API); only replies are timed. + sock.setTimeout(0) + return client + } + + async group(name: string): Promise<{ count: number; low: number; high: number }> { + return this.timed(() => this.selectGroup(name)) + } + + /** Overviews of articles `from`..`to` in the current group; an empty range gives []. */ + async xover(from: number, to: number): Promise { + return this.timed(() => this.overviews(from, to)) + } + + /** + * Overviews of `from`..`to` as one zlib/gzip stream, as the server sent it when it + * compresses (plain replies are deflated here): the seed API passes these to nodes + * without unpacking them (CONCEPT.md D38). `parseOverviews(unzipSync(raw).toString())`. + */ + async xoverRaw(from: number, to: number): Promise { + return this.timed(async () => { + const reply = await this.command(`XOVER ${from}-${to}`) + if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0)) + await this.expect(reply, ['224']) + if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw + // Plain replies (the relay's default): compressed off the event loop, at the fastest level. + return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 }) + }) + } + + /** A reply that stalls for timeoutMs fails the connection; idle time between commands doesn't count. */ + private async timed(exchange: () => Promise): Promise { + this.sock.setTimeout(this.timeoutMs) + try { + return await exchange() + } finally { + this.sock.setTimeout(0) + } + } + + private async selectGroup(name: string): Promise<{ count: number; low: number; high: number }> { + const reply = await this.command(`GROUP ${name}`) + await this.expect(reply, ['211']) + const [, count, low, high] = reply.split(' ') + return { count: Number(count), low: Number(low), high: Number(high) } + } + + private async overviews(from: number, to: number): Promise { + const reply = await this.command(`XOVER ${from}-${to}`) + if (reply.startsWith('420') || reply.startsWith('423')) return [] + await this.expect(reply, ['224']) + const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody() + return parseOverviews(data) + } + + close() { + try { + this.sock.end('QUIT\r\n') + } catch { + // already closed + } + this.sock.destroy() + } + + private async command(line: string): Promise { + this.sock.write(`${line}\r\n`) + return this.line() + } + + private async expect(reply: string, codes: string[]) { + if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`) + } + + private buffer(): Buffer { + if (this.chunks.length > 1) this.chunks = [Buffer.concat(this.chunks, this.length)] + return this.chunks[0] ?? Buffer.alloc(0) + } + + private take(n: number): Buffer { + const all = this.buffer() + const out = all.subarray(0, n) + const rest = all.subarray(n) + this.chunks = rest.length > 0 ? [rest] : [] + this.length = rest.length + return out + } + + private async more() { + if (this.failure) throw this.failure + await new Promise((resolve) => (this.wake = resolve)) + this.wake = undefined + if (this.failure && this.length === 0) throw this.failure + } + + private async line(): Promise { + while (true) { + const at = this.buffer().indexOf('\r\n') + if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1') + await this.more() + } + } + + /** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */ + private async plainBody(): Promise { + let searched = 0 + while (true) { + const all = this.buffer() + // An empty body is just ".\r\n". + if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) { + this.take(3) + return '' + } + const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length)) + if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.') + searched = all.length + await this.more() + } + } + + private async compressedBody(): Promise { + return (await this.compressedRaw()).text + } + + /** A zlib/gzip stream followed by a "." line: complete once it ends in ".\r\n" and inflates. */ + private async compressedRaw(): Promise<{ raw: Buffer; text: string }> { + while (true) { + const all = this.buffer() + if (all.length > DOT_LINE.length && all.subarray(all.length - DOT_LINE.length).equals(DOT_LINE)) { + const raw = all.subarray(0, all.length - DOT_LINE.length) + try { + const text = unzipSync(raw).toString('utf8') + this.take(all.length) + return { raw: Buffer.from(raw), text } + } catch { + // ".\r\n" inside the compressed data: wait for more. + } + } + await this.more() + } + } +} + +/** Overview lines (tab-separated, after the article number) as nntp-js returns them. */ +export function parseOverviews(data: string): Overview[] { + const out: Overview[] = [] + for (const line of data.split('\r\n')) { + if (!line) continue + const [number, ...tokens] = line.split('\t') + const fields: Record = {} + for (let i = 0; i < FIELDS.length && i < tokens.length; i++) { + // Full fields after the defaults carry their name: "Xref: host group:number". + fields[FIELDS[i]] = i >= 7 ? tokens[i].replace(/^[^:\s]+:\s*/, '') : tokens[i] + } + out.push([Number.parseInt(number, 10), fields]) + } + return out +} diff --git a/lib/nntp/xref.ts b/lib/nntp/xref.ts new file mode 100644 index 0000000..2a95dd2 --- /dev/null +++ b/lib/nntp/xref.ts @@ -0,0 +1,13 @@ +/** + * Returns the group names from an Xref header, + * e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`. + */ +export function parseXrefGroups(xref: string | undefined): string[] { + if (!xref) return [] + return xref + .trim() + .split(/\s+/) + .slice(1) + .map((entry) => entry.slice(0, entry.lastIndexOf(':'))) + .filter(Boolean) +} diff --git a/lib/release/identity.ts b/lib/release/identity.ts new file mode 100644 index 0000000..f8b5778 --- /dev/null +++ b/lib/release/identity.ts @@ -0,0 +1,223 @@ +import { createHash } from 'node:crypto' +import { looksLikeFilename } from '../headers/file_key.js' + +export interface ReleaseIdentity { + key: string + /** Human-readable name: the subject text or the shared filename stem. */ + name: string + /** Number of files announced by the `[i/M]` counter, when present. */ + fileTotal?: number + /** Which rule recognised the release. */ + source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot' +} + +export interface ReleaseInput { + poster: string + /** Subject without segment counter (the binary's `subject`). */ + subject: string + /** Filename of the binary. */ + name: string + /** True when the identity came from the yEnc body (obfuscated post). */ + weak: boolean +} + +// Suffixes that differ between the files of one release. Stripped repeatedly, +// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x". +const SUFFIXES = [ + /\.vol\d+[+-]\d+\.par2$/i, + /\.par2$/i, + /\.part\d+\.rar$/i, + /\.r\d{2,3}$/i, + /\.rar$/i, + /\.7z(\.\d{3})?$/i, + /\.zip(\.\d{3})?$/i, + // Split volumes (x.mkv.001), not codec names (H.264, H.265). + /(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i, + /\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i, + /\.tar$/i, + /\.(zst|gz|xz|bz2)$/i, + /\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i, +] + +/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */ +export function fileStem(name: string): string { + let stem = name.trim() + let changed = true + while (changed) { + changed = false + for (const suffix of SUFFIXES) { + const next = stem.replace(suffix, '') + if (next !== stem && next.length > 0) { + stem = next + changed = true + } + } + } + return stem +} + +/** + * What remains of a subject once the filename, counters, "yEnc" and sizes are + * removed. Posters that name the release outside the quotes put it here. + */ +export function subjectRemainder(subject: string): string { + return subject + .replace(/"[^"]*"/g, ' ') + .replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ') + // File sizes follow "yEnc" at the end; other numbers (years, post ids) stay. + .replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ') + .replace(/\byEnc\b/gi, ' ') + .replace(/[\s\-–.:|]+/g, ' ') + .trim() +} + +/** + * Posting tools put random per-file tokens and poster IDs in subjects (e.g. + * `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a + * name once those are removed identifies a release. + */ +function isRandomToken(token: string): boolean { + const t = token.replace(/^\[|\]$/g, '') + if (t.length < 6) return false + if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true + return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t) +} + +/** + * A stable per-post token: one short run of letters/digits with at least one + * digit. Requiring a digit keeps constant labels like `[PRiVATE]` from + * grouping different posts of the same poster. + */ +export function isPostToken(remainder: string): boolean { + const compact = remainder.replace(/[\s\[\]()_-]+/g, '') + return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact) +} + +export function namesRelease(remainder: string): boolean { + const words = remainder + .split(/\s+/) + .filter((w) => w && !isRandomToken(w)) + .join(' ') + // At least two words of letters, joined the way release names are. + return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words) +} + +function hash(...parts: (string | number | undefined)[]) { + return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex') +} + +/** + * Decides which release (download) a file belongs to. Returns null when the + * file cannot be grouped from its name alone (random extensionless names in + * obfuscated posts; those need par2 matching). See CONCEPT.md (D16). + */ +export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { + const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject) + const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined + const stem = fileStem(input.name) + + if (input.weak) { + // Obfuscating posters randomise the poster per article; a real filename + // stem is specific enough on its own. + if (!looksLikeFilename(input.name)) return null + return { key: hash('weak', stem), name: stem, source: 'weak-stem' } + } + + // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". + const remainder = subjectRemainder(input.subject) + if (namesRelease(remainder)) { + return { + key: hash('subject', input.poster, remainder), + name: remainder, + fileTotal, + source: 'subject', + } + } + // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the + // [i/M] counter it groups the files of one post whose names are random. + if (fileTotal && isPostToken(remainder)) { + return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' } + } + // The [i/M] total is informational, not part of the key: posters often + // upload the par2 index without a counter, and it belongs to the same post. + return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' } +} + +/** Release of a par2 recovery set; every file listed in it belongs to it. */ +export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity { + return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' } +} + +const SEPARATOR = /[._ -]/ + +/** + * A release name from the files a par2 set lists. Files that differ only in + * the middle (episodes S03E01…S03E20, parts) give their shared name, with + * the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to + * SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato. + * Otherwise the longest stem, which tends to be the most descriptive (the + * video file). + */ +export function par2ReleaseName(fileNames: string[]): string | undefined { + const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean) + if (stems.length === 0) return undefined + const longest = [...stems].sort((a, b) => b.length - a.length)[0] + if (stems.length < 2) return longest + + let prefix = stems.reduce((p, s) => { + let i = 0 + while (i < p.length && i < s.length && p[i] === s[i]) i++ + return p.slice(0, i) + }) + let suffix = stems.reduce((p, s) => { + let i = 0 + while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++ + return p.slice(p.length - i) + }) + const shortest = Math.min(...stems.map((s) => s.length)) + if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest) + + // Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token. + const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix) + if (season) prefix = season[1] + else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) { + prefix = prefix.replace(/[^._ -]*$/, '') + } + suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '') + prefix = prefix.replace(/[._ -]+$/, '') + + const name = suffix ? prefix + suffix : prefix + // Only when the shared part carries most of the name (not just "The."). + return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest +} + +/** + * Release of a RAR volume set, from what its headers list: the shared + * top-level directory if there is one, otherwise the first entry. For + * volumes after the first, the first entry is the file that continues from + * the previous volume, so all volumes of a single-file archive agree. + */ +/** + * Folder and file names that say nothing about what a release is: disc + * structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers. + * Kept in step with SQL's is_readable_name(). + */ +export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i + +export function isGenericName(name: string): boolean { + return GENERIC_NAMES.test(name.trim()) +} + +export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null { + if (entries.length === 0) return null + const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0])) + const nested = entries.some((e) => /[\\/]/.test(e.name)) + // A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead. + if (nested && tops.size === 1 && !isGenericName([...tops][0])) { + const [top] = tops + return { key: hash('rar-dir', top), name: top, source: 'rar' } + } + const first = entries[0] + const base = first.name.split(/[\\/]/).pop() ?? first.name + return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' } +} diff --git a/lib/release/index.ts b/lib/release/index.ts new file mode 100644 index 0000000..55e8d36 --- /dev/null +++ b/lib/release/index.ts @@ -0,0 +1,5 @@ +/** + * Grouping of files into releases (downloads), independent of AdonisJS. + */ +export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js' +export { looksRandom } from './random.js' diff --git a/lib/release/random.ts b/lib/release/random.ts new file mode 100644 index 0000000..1f9318d --- /dev/null +++ b/lib/release/random.ts @@ -0,0 +1,27 @@ +import { isGenericName } from './identity.js' + +/** + * Obfuscated names: one long token of mixed letters and digits, no + * separators, and the other generated shapes seen on Usenet (LOG.md + * 2026-10-01 (3)). Kept in step with SQL's is_readable_name(). + */ +export function looksRandom(name: string): boolean { + // An unknown short extension (e.g. ".bin") says nothing about the name. + name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') + // VIDEO_TS, Sample, CD1: real words, but no name. + if (isGenericName(name)) return true + if (/^[0-9a-f]{16,}$/i.test(name)) return true + // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time) + if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true + // UUIDs + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true + // 1650777167 (yEncBin), "83111763 n" (rclone uploads) + if (/^\d{6,}( n)?$/.test(name)) return true + // prvjr7hapd0i1ny14pz4, and rclone's long base32 names + if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true + // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts + if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true + if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false + // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). + return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 +} diff --git a/lib/rules/index.ts b/lib/rules/index.ts new file mode 100644 index 0000000..112f217 --- /dev/null +++ b/lib/rules/index.ts @@ -0,0 +1,2 @@ +export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js' +export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js' diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts new file mode 100644 index 0000000..b3a3dce --- /dev/null +++ b/lib/rules/rule_set.ts @@ -0,0 +1,69 @@ +import { explored, signatureKey, uploadTool } from './upload_tools.js' +import { fileStem, looksRandom } from '../release/index.js' + +/** + * What a worker drops, and why (REBUILD.md, rules and recovery). Sent with + * every lease; `version` is recorded on the range, so a changed rule set + * shows which ranges to read again. + */ +export interface Rules { + version: number + /** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */ + priorities: Record + /** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */ + droppedSignatures: string[] + /** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */ + droppedTools: string[] + /** A random-named file of at most this many parts: a random subject per article, needs bodies. */ + perArticleMaxParts: number + /** Share of each rule's drops kept in full as a sample (by a hash of the release key). */ + samplePercent: number +} + +export const DEFAULT_RULES: Omit = { + version: 1, + droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'], + perArticleMaxParts: 3, + samplePercent: 1, +} + +export type Verdict = { rule: null } | { rule: 'garbage' | 'per-article'; key: string; sample: boolean } + +export function isRandomFile(name: string) { + return looksRandom(fileStem(name)) +} + +/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */ +export function ownerOf(groups: string[], priorities: Record): string | null { + let owner: string | null = null + for (const g of groups) { + const p = priorities[g] + if (p !== undefined && (owner === null || p < priorities[owner])) owner = g + } + return owner +} + +export class RuleSet { + private signatures: Set + private tools: Set + constructor(readonly rules: Rules) { + this.signatures = new Set(rules.droppedSignatures) + this.tools = new Set(rules.droppedTools) + } + + /** + * A release's verdict from its files: kept as a whole when any file has a + * readable name; else dropped as per-article random subjects when every + * file has few parts, as garbage when its signature or tool is dropped. + */ + judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict { + if (files.some((f) => !isRandomFile(f.name))) return { rule: null } + const first = files[0] + const tool = uploadTool(first.firstMessageId) + const sample = explored(releaseKey, this.rules.samplePercent) + if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample } + const group = first.groups[0] ?? '' + if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample } + return { rule: null } + } +} diff --git a/lib/rules/upload_tools.ts b/lib/rules/upload_tools.ts new file mode 100644 index 0000000..f99590f --- /dev/null +++ b/lib/rules/upload_tools.ts @@ -0,0 +1,52 @@ +import { createHash } from 'node:crypto' + +/** + * Upload signatures (CONCEPT.md D42): which tool posted an article, read from + * its Message-ID, and the group it was posted to first. Together they predict + * whether a random-named upload will ever get a name: ngPost uploads in + * a.b.ath almost never do, nyuu uploads in a.b.friends always do (LOG.md + * 2026-10-08 (5)). + */ +const TOOLS: [string, RegExp][] = [ + // FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu: 12 random capital+small letter pairs, a ms timestamp + ['nyuu', /^(?:[A-Z][a-z]){12}-\d{13}@nyuu$/], + ['nyuu-host', /^(?:[A-Z][a-z]){12}-\d{13}@.+$/], + // b94268bc702040f7b811d5318f727e4f@ngPost: a UUID's hex + ['ngPost', /^[0-9a-f]{32}@ngPost$/], + ['yuun', /^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$/], + ['hex32', /^[0-9a-f]{32}@[A-Za-z0-9.-]+$/], + ['JBinUp', /@JBin(?:Up|Down)\.local$/], + // part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local + ['partNofM', /^part\d+of\d+\./i], + // 1790923856877$1729905757859$1@localhost + ['ts-dollar', /^\d{13}\$\d+\$\d+@localhost$/], + // t3Y4mV92YCp0McxDc5@zLZlsw.RYqes: random local part and random host + ['random', /^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$/], +] + +/** The posting tool of a Message-ID (without angle brackets), or 'other'. */ +export function uploadTool(messageId: string | null | undefined): string { + if (!messageId) return 'other' + for (const [tool, pattern] of TOOLS) if (pattern.test(messageId)) return tool + return 'other' +} + +/** The group of a signature: the first the upload was posted to, as stored in `groups`. */ +export function signatureGroup(groups: unknown): string { + return Array.isArray(groups) && typeof groups[0] === 'string' ? groups[0] : '' +} + +/** The key of a signature in a Set. */ +export function signatureKey(tool: string, group: string) { + return `${tool}\t${group}` +} + +/** + * Whether an upload is in the share of dropped signatures that is still kept + * (and probed), so their rates keep being measured. Decided per release key, + * so a release is kept or dropped as a whole. + */ +export function explored(key: string, percent: number): boolean { + if (percent <= 0) return false + return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100 +} diff --git a/tests/unit/chunks/codec.spec.ts b/tests/unit/chunks/codec.spec.ts new file mode 100644 index 0000000..624bc03 --- /dev/null +++ b/tests/unit/chunks/codec.spec.ts @@ -0,0 +1,18 @@ +import { test } from '@japa/runner' +import { decodeChunk, encodeChunk, type ChunkFile } from '#lib/chunks/index' + +test.group('chunks / codec', () => { + test('round-trips files with their segments, part numbers when not 1..N', ({ assert }) => { + const files: ChunkFile[] = [ + { name: 'a.part1.rar', poster: 'p ', subject: 'A [1/2] - "a.part1.rar" yEnc', postedAt: 1_700_000_000_000, groups: ['alt.binaries.test'], segments: [[1, 'one@x', 700_000], [2, 'two@x', 1234]] }, + { name: 'a.par2', poster: 'p ', subject: 'A [2/2] - "a.par2" yEnc', postedAt: null, groups: ['alt.binaries.test', 'alt.binaries.x'], segments: [[2, 'gap@x', 10], [5, 'gap2@x', 11]] }, + ] + const data = encodeChunk(files) + assert.equal(data[0], 1) + assert.deepEqual(decodeChunk(data), files) + }) + + test('refuses an unknown version', ({ assert }) => { + assert.throws(() => decodeChunk(Buffer.from([9, 0])), /Unknown chunk version 9/) + }) +}) diff --git a/tests/unit/files/aggregator.spec.ts b/tests/unit/files/aggregator.spec.ts new file mode 100644 index 0000000..0125170 --- /dev/null +++ b/tests/unit/files/aggregator.spec.ts @@ -0,0 +1,39 @@ +import { test } from '@japa/runner' +import { FileAggregator, type AggregatableSegment } from '#lib/files/index' + +const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({ + fileKey, + number, + totalParts, + messageId, + bytes: 100, +}) + +test.group('aggregate / FileAggregator', () => { + test('completes a file once every part arrived, across batches and out of order', ({ assert }) => { + const aggregator = new FileAggregator() + assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b']) + assert.equal(aggregator.openFiles, 1) + const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)]) + assert.equal(done.fileKey, 'a') + assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3]) + assert.equal(aggregator.openFiles, 0) + }) + + test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => { + const aggregator = new FileAggregator() + const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)]) + assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test']) + assert.deepEqual(aggregator.evict().map((s) => s.number), [5]) + }) + + test('gives up idle files as fragments, all of them at the end', ({ assert }) => { + const aggregator = new FileAggregator() + aggregator.add([seg('old', 1, 3)]) + aggregator.add([seg('new', 1, 3)]) + aggregator.add([seg('new', 2, 3)]) + assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old']) + assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2]) + assert.equal(aggregator.openFiles, 0) + }) +}) diff --git a/tests/unit/headers/subject.spec.ts b/tests/unit/headers/subject.spec.ts new file mode 100644 index 0000000..35b9b45 --- /dev/null +++ b/tests/unit/headers/subject.spec.ts @@ -0,0 +1,65 @@ +import { test } from '@japa/runner' +import { fileKey, parseSubject } from '#lib/headers/index' +import { parseXrefGroups } from '#lib/nntp/index' + +test.group('subject / parseSubject', () => { + test('parses file counter, filename and segment counter', ({ assert }) => { + assert.deepEqual(parseSubject('. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc (56/274)'), { + name: 'obrMkrDeLmbhaouwYO34.part08.rar', + segment: 56, + totalSegments: 274, + fileIndex: 9, + fileTotal: 51, + base: '. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc', + }) + }) + + test('handles a file size after the segment counter', ({ assert }) => { + const subject = '[8/9] - "x.tar.zst.vol063+064.par2" yEnc (37/72) 51315352' + const parsed = parseSubject(subject)! + assert.equal(parsed.segment, 37) + assert.equal(parsed.totalSegments, 72) + assert.equal(parsed.base, '[8/9] - "x.tar.zst.vol063+064.par2" yEnc 51315352') + }) + + test('ignores counters inside the filename', ({ assert }) => { + const parsed = parseSubject('"Show (1/2) Special.mkv" yEnc (3/10)')! + assert.equal(parsed.name, 'Show (1/2) Special.mkv') + assert.equal(parsed.segment, 3) + assert.equal(parsed.totalSegments, 10) + }) + + test('the base is equal for all segments of one file', ({ assert }) => { + const a = parseSubject('[1/9] - "a.tar.zst" yEnc (253/1000) 716554110')! + const b = parseSubject('[1/9] - "a.tar.zst" yEnc (254/1000) 716554110')! + assert.equal(a.base, b.base) + }) + + test('returns null for obfuscated or invalid subjects', ({ assert }) => { + assert.isNull(parseSubject('GZFDJc0hB3Y0FwwAznkk')) + assert.isNull(parseSubject('"name.rar" yEnc')) + assert.isNull(parseSubject('"name.rar" yEnc (0/5)')) + assert.isNull(parseSubject('"name.rar" yEnc (6/5)')) + assert.isNull(parseSubject('"" yEnc (1/5)')) + }) +}) + +test.group('subject / fileKey', () => { + test('differs per poster and per segment count, is stable otherwise', ({ assert }) => { + const key = fileKey('poster@a', '"x.rar" yEnc', 10) + assert.lengthOf(key, 40) + assert.equal(key, fileKey('poster@a', '"x.rar" yEnc', 10)) + assert.notEqual(key, fileKey('poster@b', '"x.rar" yEnc', 10)) + assert.notEqual(key, fileKey('poster@a', '"x.rar" yEnc', 11)) + }) +}) + +test.group('nntp / parseXrefGroups', () => { + test('extracts crossposted groups', ({ assert }) => { + assert.deepEqual( + parseXrefGroups('news.eweka.nl alt.binaries.nl:35801424104 alt.binaries.test:27275430913'), + ['alt.binaries.nl', 'alt.binaries.test'] + ) + assert.deepEqual(parseXrefGroups(undefined), []) + }) +}) diff --git a/tests/unit/nntp/group.spec.ts b/tests/unit/nntp/group.spec.ts new file mode 100644 index 0000000..1e24503 --- /dev/null +++ b/tests/unit/nntp/group.spec.ts @@ -0,0 +1,18 @@ +import { test } from '@japa/runner' +import { parseGroupResponse } from '#lib/nntp/index' + +test.group('nntp / parseGroupResponse', () => { + test('parses the reply from news.eweka.nl', ({ assert }) => { + // Captured 2026-09-27. count == last - first + 1, so the field order is standard. + assert.deepEqual(parseGroupResponse('211 27275164317 266599 27275430915 alt.binaries.test'), { + count: 27275164317n, + first: 266599n, + last: 27275430915n, + name: 'alt.binaries.test', + }) + }) + + test('rejects non-211 replies', ({ assert }) => { + assert.throws(() => parseGroupResponse('411 No such group'), /Unexpected GROUP response/) + }) +}) diff --git a/tests/unit/nntp/xover_client.spec.ts b/tests/unit/nntp/xover_client.spec.ts new file mode 100644 index 0000000..0b770ba --- /dev/null +++ b/tests/unit/nntp/xover_client.spec.ts @@ -0,0 +1,106 @@ +import { test } from '@japa/runner' +import net from 'node:net' +import { deflateSync } from 'node:zlib' +import { XoverClient, parseOverviews } from '#lib/nntp/index' +import { unzipSync } from 'node:zlib' + +const LINES = [ + '101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter \tMon, 05 Oct 2026 10:00:00 +0000\t\t\t700000\t5000\tXref: news.example a.b.test:101', + '102\tShow.S01E01 - "a.mkv" yEnc (2/2)\tposter \tMon, 05 Oct 2026 10:00:01 +0000\t\t\t650000\t4600\tXref: news.example a.b.test:102 a.b.other:7', +] + +/** A tiny NNTP server: plain replies, gzip after XFEATURE, 420 for 1-1, replies split into small chunks. */ +function server(): Promise<{ port: number; close: () => void }> { + return new Promise((resolve) => { + const srv = net.createServer((sock) => { + let gzip = false + const send = (data: Buffer) => { + for (let i = 0; i < data.length; i += 7) sock.write(data.subarray(i, i + 7)) + } + sock.write('200 welcome\r\n') + let pending = '' + sock.on('data', (chunk) => { + pending += chunk.toString() + let at: number + while ((at = pending.indexOf('\r\n')) !== -1) { + const line = pending.slice(0, at) + pending = pending.slice(at + 2) + if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n') + else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n') + else if (line === 'XFEATURE COMPRESS GZIP') (gzip = true), sock.write('290 Feature Enabled\r\n') + else if (line.startsWith('GROUP')) sock.write('211 2 101 102 a.b.test\r\n') + else if (line === 'XOVER 1-1') sock.write('420 No Articles Selected\r\n') + else if (line.startsWith('XOVER')) { + const body = Buffer.from(LINES.join('\r\n') + '\r\n') + if (gzip) { + sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n') + send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')])) + } else { + sock.write('224 Overview Information Follows\r\n') + send(Buffer.concat([body, Buffer.from('.\r\n')])) + } + } else if (line === 'QUIT') sock.end('205 bye\r\n') + } + }) + }) + srv.listen(0, '127.0.0.1', () => resolve({ port: (srv.address() as net.AddressInfo).port, close: () => srv.close() })) + }) +} + +test.group('nntp / XoverClient', () => { + test('parses overview lines into the fields nntp-js gives', ({ assert }) => { + const [[number, fields]] = parseOverviews(LINES[0] + '\r\n') + assert.equal(number, 101) + assert.equal(fields['message-id'], '') + assert.equal(fields[':bytes'], '700000') + assert.equal(fields.xref, 'news.example a.b.test:101') + }) + + for (const compress of [false, true]) { + test(`fetches overviews ${compress ? 'gzip-compressed' : 'plain'}, and an empty range`, async ({ assert }) => { + const { port, close } = await server() + const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress }) + try { + assert.equal(client.compressed, compress) + assert.deepEqual(await client.group('a.b.test'), { count: 2, low: 101, high: 102 }) + const overviews = await client.xover(101, 102) + assert.deepEqual(overviews.map(([n]) => n), [101, 102]) + assert.equal(overviews[1][1].subject, 'Show.S01E01 - "a.mkv" yEnc (2/2)') + assert.deepEqual(await client.xover(1, 1), []) + assert.lengthOf(await client.xover(101, 102), 2) + } finally { + client.close() + close() + } + }) + } + + test('idle time between commands does not count against the reply timeout', async ({ assert }) => { + const { port, close } = await server() + const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: true, timeoutMs: 200 }) + try { + await client.group('a.b.test') + await new Promise((resolve) => setTimeout(resolve, 500)) + assert.lengthOf(await client.xover(101, 102), 2) + } finally { + client.close() + close() + } + }) + + for (const compress of [false, true]) { + test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({ assert }) => { + const { port, close } = await server() + const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress }) + try { + await client.group('a.b.test') + const raw = await client.xoverRaw(101, 102) + assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [101, 102]) + assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0) + } finally { + client.close() + close() + } + }) + } +}) diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts new file mode 100644 index 0000000..636fb06 --- /dev/null +++ b/tests/unit/release/identity.spec.ts @@ -0,0 +1,169 @@ +import { test } from '@japa/runner' +import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index' +import { looksLikeFilename, weakFileKey } from '#lib/headers/index' + +test.group('release / fileStem', () => { + test('reduces all files of a set to the same stem', ({ assert }) => { + const names = [ + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst', + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2', + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2', + ] + assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715'])) + }) + + test('handles rar, split and par2 volume naming', ({ assert }) => { + assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0') + assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265') + assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264') + assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024') + assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026') + assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b') + }) + + test('never reduces a name to nothing', ({ assert }) => { + assert.equal(fileStem('.par2'), '.par2') + }) +}) + +test.group('release / subjectRemainder', () => { + test('is empty when the subject only has counters, filename and size', ({ assert }) => { + assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '') + assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '') + }) + + test('keeps a release name written outside the quotes', ({ assert }) => { + assert.equal( + subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'), + 'Some Show S01E02 1080p' + ) + }) +}) + +test.group('release / releaseIdentity', () => { + test('groups a par2 set with its archive by stem, per poster', ({ assert }) => { + const base = { poster: 'p', weak: false } + const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })! + const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })! + assert.equal(a.key, b.key) + assert.equal(a.name, 'x') + assert.equal(a.fileTotal, 4) + const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })! + assert.notEqual(a.key, other.key) + }) + + test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => { + const base = { poster: 'nicovideo_archiver', weak: false } + const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })! + const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })! + assert.equal(index.key, archive.key) + assert.equal(archive.fileTotal, 6) + assert.isUndefined(index.fileTotal) + }) + + test('uses the subject text when it names the release', ({ assert }) => { + const base = { poster: 'p', weak: false } + const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })! + const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })! + assert.equal(a.key, b.key) + assert.equal(a.name, 'My Release') + }) + + test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => { + const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })! + const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })! + assert.equal(a.key, b.key) + }) + + test('random extensionless names cannot be grouped by name', ({ assert }) => { + const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231' + assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true })) + }) +}) + +test.group('subject / weakFileKey', () => { + test('ignores random per-article names, keeps real filenames', ({ assert }) => { + assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231')) + assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar')) + // Same file, different random names per article: same key. + assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7)) + // Equal-sized volumes with real names: different keys. + assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7)) + assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7)) + }) +}) + +test.group('release / rarReleaseIdentity', () => { + test('groups volumes by the shared top-level directory', async ({ assert }) => { + const { rarReleaseIdentity } = await import('#lib/release/index') + const first = rarReleaseIdentity([ + { name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 }, + { name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 }, + ])! + const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])! + assert.equal(first.key, later.key) + assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM') + assert.equal(first.source, 'rar') + }) + + test('without a directory, volumes of one file share its first entry', async ({ assert }) => { + const { rarReleaseIdentity } = await import('#lib/release/index') + const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])! + const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])! + assert.equal(a.key, b.key) + assert.equal(a.name, 'Some.Show.S01E01.1080p') + assert.isNull(rarReleaseIdentity([])) + }) +}) + +test.group('release / namesRelease', () => { + test('random per-file tags and poster ids are not a release name', async ({ assert }) => { + const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index') + const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc' + assert.isFalse(namesRelease(subjectRemainder(subject))) + const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })! + assert.equal(a.source, 'stem') + assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'))) + assert.isTrue(namesRelease('My Release')) + }) +}) + +test.group('release / post id', () => { + test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => { + const { releaseIdentity, isPostToken } = await import('#lib/release/index') + const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false } + const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })! + const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })! + assert.equal(a.key, b.key) + assert.equal(a.source, 'post') + assert.equal(a.fileTotal, 18) + const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })! + assert.notEqual(a.key, other.key) + + assert.isTrue(isPostToken('83110629 n')) + assert.isFalse(isPostToken('PRiVATE')) + assert.isFalse(isPostToken('')) + // Without a counter there is nothing to bound the post: fall back to the stem. + assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem') + }) +}) + +test.group('release / par2 names', () => { + test('names a season pack after what its episodes share', async ({ assert }) => { + const { par2ReleaseName } = await import('#lib/release/index') + const eps = Array.from({ length: 20 }, (_, i) => + `SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`) + assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato') + // Movie plus sample and nfo: the movie's (longest) name. + assert.equal( + par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']), + 'Some.Movie.2024.1080p.BluRay.x264-GRP' + ) + // Numbered parts: the shared name without the part number. + assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP') + assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024') + }) +}) diff --git a/tests/unit/rules/rule_set.spec.ts b/tests/unit/rules/rule_set.spec.ts new file mode 100644 index 0000000..66caf0b --- /dev/null +++ b/tests/unit/rules/rule_set.spec.ts @@ -0,0 +1,56 @@ +import { test } from '@japa/runner' +import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rules/index' +import { sevenZipVolume } from '#lib/files/index' + +const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] }) +const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath') => ({ name, totalParts, groups: [group], firstMessageId }) +const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost' +const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu' + +test.group('rules / owner', () => { + test('a crosspost belongs to the indexed group with the lowest priority, whatever the Xref order', ({ assert }) => { + const priorities = { 'alt.binaries.boneless': 1, 'alt.binaries.ath': 3, 'alt.binaries.added.later': 1001 } + assert.equal(ownerOf(['alt.binaries.ath', 'alt.binaries.boneless'], priorities), 'alt.binaries.boneless') + assert.equal(ownerOf(['alt.binaries.added.later', 'alt.binaries.ath'], priorities), 'alt.binaries.ath') + assert.equal(ownerOf(['alt.binaries.not.indexed'], priorities), null) + }) +}) + +test.group('rules / verdicts', () => { + test('a readable file keeps its whole release', ({ assert }) => { + assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null }) + }) + + test('random names: per-article with few parts, garbage by tool or signature, else kept', ({ assert }) => { + assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa', 2, NYUU)]), { rule: 'per-article', key: 'nyuu' }) + assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST)]), { rule: 'garbage', key: 'ngPost alt.binaries.ath' }) + assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU, 'alt.binaries.misc')]), { rule: 'garbage', key: 'nyuu alt.binaries.misc' }) + assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU)]), { rule: null }) + }) +}) + +test.group('rules / upload tools', () => { + test('posting tools are told apart by their Message-IDs', ({ assert }) => { + assert.equal(uploadTool(NYUU), 'nyuu') + assert.equal(uploadTool(NGPOST), 'ngPost') + assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM') + assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp') + assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random') + assert.equal(uploadTool(null), 'other') + }) + + test('the sampled share is stable per key and about the percentage', ({ assert }) => { + assert.equal(explored('some-release', 2), explored('some-release', 2)) + const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length + assert.isAbove(kept, 150) + assert.isBelow(kept, 250) + }) +}) + +test.group('files / volumes', () => { + test('7z volume numbers', ({ assert }) => { + assert.equal(sevenZipVolume('x.7z.001'), 1) + assert.equal(sevenZipVolume('x.7z'), 0) + assert.isNull(sevenZipVolume('x.rar')) + }) +})