Rebuild: phase 1 modules from the current app, ordered by pipeline stage
lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
96f0ade10e
commit
d773c91ad0
26 files changed
+1486
No files matched your search
@@ -0,0 +1,45 @@
|
||||
import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
|
||||
|
||||
/** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */
|
||||
export interface ChunkFile {
|
||||
name: string
|
||||
poster: string
|
||||
subject: string
|
||||
/** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */
|
||||
postedAt: number | null
|
||||
groups: string[]
|
||||
segments: [number, string, number][]
|
||||
}
|
||||
|
||||
const VERSION = 1
|
||||
|
||||
/**
|
||||
* A release chunk's payload (REBUILD.md, chunk store): a version byte, then
|
||||
* zstd of a JSON line with the files (name, poster, subject, date, groups,
|
||||
* sizes, and part numbers unless they are 1..N) followed by one message-id
|
||||
* per line, in file order. Message-ids are mostly random, so this is close
|
||||
* to what plain compression gets (LOG.md 2026-10-08 (3)).
|
||||
*/
|
||||
export function encodeChunk(files: ChunkFile[]): Buffer {
|
||||
const head = files.map((f) => {
|
||||
const sequential = f.segments.every(([n], i) => n === i + 1)
|
||||
return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) }
|
||||
})
|
||||
const ids = files.flatMap((f) => f.segments.map(([, id]) => id))
|
||||
const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8')
|
||||
return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })])
|
||||
}
|
||||
|
||||
export function decodeChunk(data: Buffer): ChunkFile[] {
|
||||
if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`)
|
||||
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
||||
let at = 0
|
||||
return (JSON.parse(head) as any[]).map((f) => ({
|
||||
name: f.n,
|
||||
poster: f.p,
|
||||
subject: f.j,
|
||||
postedAt: f.d,
|
||||
groups: f.g,
|
||||
segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]),
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { encodeChunk, decodeChunk, type ChunkFile } from './codec.js'
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Edge aggregation (CONCEPT.md D37): groups header segments into files in
|
||||
* memory, so a seeding node writes one row per file instead of one per
|
||||
* article. A file is done once every part number from 1 to its total has
|
||||
* an article; what never completes within the node's window (an upload cut
|
||||
* by the range edges, a random subject per article) leaves as fragments
|
||||
* for the regular ingest path, where it can meet its other parts.
|
||||
*/
|
||||
|
||||
/** What the aggregator needs of a segment (IngestSegment has all of it). */
|
||||
export interface AggregatableSegment {
|
||||
fileKey: string
|
||||
number: number
|
||||
totalParts: number
|
||||
messageId: string
|
||||
bytes: number
|
||||
}
|
||||
|
||||
export interface AggregatedFile<S extends AggregatableSegment> {
|
||||
fileKey: string
|
||||
/** One segment per part number, in part order; the first article seen wins (a later one is a repost). */
|
||||
segments: S[]
|
||||
}
|
||||
|
||||
interface OpenFile<S> {
|
||||
parts: Map<number, S>
|
||||
totalParts: number
|
||||
/** Batch in which a segment last arrived. */
|
||||
touched: number
|
||||
}
|
||||
|
||||
export class FileAggregator<S extends AggregatableSegment> {
|
||||
private open = new Map<string, OpenFile<S>>()
|
||||
private batch = 0
|
||||
/** Segments with a part number outside 1..total: never part of a complete file. */
|
||||
private stray: S[] = []
|
||||
|
||||
/** Adds one batch of segments; returns the files they completed. */
|
||||
add(segments: S[]): AggregatedFile<S>[] {
|
||||
this.batch++
|
||||
const touched = new Set<string>()
|
||||
for (const segment of segments) {
|
||||
if (segment.number < 1 || segment.number > segment.totalParts) {
|
||||
this.stray.push(segment)
|
||||
continue
|
||||
}
|
||||
let file = this.open.get(segment.fileKey)
|
||||
if (!file) {
|
||||
file = { parts: new Map(), totalParts: segment.totalParts, touched: this.batch }
|
||||
this.open.set(segment.fileKey, file)
|
||||
}
|
||||
if (!file.parts.has(segment.number)) file.parts.set(segment.number, segment)
|
||||
file.touched = this.batch
|
||||
touched.add(segment.fileKey)
|
||||
}
|
||||
|
||||
const complete: AggregatedFile<S>[] = []
|
||||
for (const key of touched) {
|
||||
const file = this.open.get(key)!
|
||||
if (file.parts.size < file.totalParts) continue
|
||||
this.open.delete(key)
|
||||
complete.push({ fileKey: key, segments: [...file.parts.values()].sort((a, b) => a.number - b.number) })
|
||||
}
|
||||
return complete
|
||||
}
|
||||
|
||||
/**
|
||||
* Gives up on files that got no segment in the last `idleBatches` batches
|
||||
* (all open files when 0) and returns their segments, plus stray ones.
|
||||
*/
|
||||
evict(idleBatches = 0): S[] {
|
||||
const out = this.stray
|
||||
this.stray = []
|
||||
for (const [key, file] of this.open) {
|
||||
if (idleBatches > 0 && this.batch - file.touched < idleBatches) continue
|
||||
this.open.delete(key)
|
||||
out.push(...file.parts.values())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Files still waiting for parts. */
|
||||
get openFiles(): number {
|
||||
return this.open.size
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { FileAggregator, type AggregatableSegment, type AggregatedFile } from './aggregator.js'
|
||||
export { isArchiveStart, isLaterVolume, rarVolume, sevenZipVolume } from './volumes.js'
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* A later RAR volume (.part02.rar, .r00) or a par2 recovery volume
|
||||
* (.vol03+04.par2): the volume number is readable even under a random stem.
|
||||
* The set is listed from its first volume and named from its index .par2,
|
||||
* so the first segment of these says nothing new (LOG.md 2026-10-04).
|
||||
*/
|
||||
export function isLaterVolume(name: string): boolean {
|
||||
const part = /\.part(\d+)\.rar$/i.exec(name)
|
||||
if (part) return Number(part[1]) > 1
|
||||
return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of
|
||||
* `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for
|
||||
* other names.
|
||||
*/
|
||||
export function rarVolume(name: string): { base: string; number: number } | null {
|
||||
const part = /^(.*)\.part(\d+)\.rar$/i.exec(name)
|
||||
if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) }
|
||||
const old = /^(.*)\.r(\d{2,3})$/i.exec(name)
|
||||
if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 }
|
||||
const first = /^(.*)\.rar$/i.exec(name)
|
||||
if (first) return { base: first[1].toLowerCase(), number: 1 }
|
||||
return null
|
||||
}
|
||||
|
||||
/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */
|
||||
export function sevenZipVolume(name: string): number | null {
|
||||
const split = /\.7z\.(\d{1,4})$/i.exec(name)
|
||||
if (split) return Number(split[1])
|
||||
return /\.7z$/i.test(name) ? 0 : null
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Identity of one uploaded file: every segment of it has the same poster, the
|
||||
* same subject apart from the segment counter, and the same segment count.
|
||||
* The group is deliberately left out so crossposts resolve to one file.
|
||||
* See CONCEPT.md (D12).
|
||||
*/
|
||||
export function fileKey(poster: string, base: string, totalSegments: number): string {
|
||||
return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex')
|
||||
}
|
||||
|
||||
/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */
|
||||
export function looksLikeFilename(name: string): boolean {
|
||||
return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file known only from its yEnc header (obfuscated posts).
|
||||
* The poster is left out: obfuscating posters randomise it per article.
|
||||
* - A real filename is stable across the articles of a file, and needed to
|
||||
* tell apart the equal-sized volumes of one archive.
|
||||
* - A random, extensionless name can change per article too; the exact
|
||||
* byte size and part count are what stays the same.
|
||||
* Collisions are caught by the conflict check in IngestService.
|
||||
*/
|
||||
export function weakFileKey(name: string, size: number, totalParts: number): string {
|
||||
const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
|
||||
return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file by its counters, for uploads that give every article
|
||||
* its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`,
|
||||
* part count and posting day stay the same across the articles of one file.
|
||||
* The day keeps different uploads with the same shape apart; collisions that
|
||||
* remain are caught by the conflict check (these files are weak identities).
|
||||
* Null when the subject has no file counter.
|
||||
*/
|
||||
export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
|
||||
if (!counter || segment.totalParts < 2) return null
|
||||
const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
|
||||
return createHash('sha1')
|
||||
.update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
|
||||
.digest('hex')
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export { parseSubject, type ParsedSubject } from './subject.js'
|
||||
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
||||
export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js'
|
||||
@@ -0,0 +1,72 @@
|
||||
import { parseXrefGroups } from '../nntp/xref.js'
|
||||
import { fileKey } from './file_key.js'
|
||||
import { parseSubject } from './subject.js'
|
||||
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
||||
|
||||
/** One segment, with the identity of the file it belongs to already resolved. */
|
||||
export interface IngestSegment {
|
||||
fileKey: string
|
||||
poster: string
|
||||
/** Subject without segment counter. */
|
||||
subject: string
|
||||
name: string
|
||||
totalParts: number
|
||||
number: number
|
||||
/** Without angle brackets. */
|
||||
messageId: string
|
||||
bytes: number
|
||||
articleNumber?: bigint | number
|
||||
postedAt?: Date
|
||||
groups: string[]
|
||||
/**
|
||||
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
||||
* can be shared by several equal-sized files of one post, so a second
|
||||
* article for the same part number marks the binary as conflicted.
|
||||
*/
|
||||
weakIdentity?: boolean
|
||||
/** Release (download) this file belongs to, when it can be determined. */
|
||||
release?: ReleaseIdentity | null
|
||||
/** Exact file size from =ybegin (body-derived segments only). */
|
||||
size?: number
|
||||
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
||||
pcrc32?: string
|
||||
partSize?: number
|
||||
fileCrc32?: string
|
||||
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
||||
md5_16k?: string
|
||||
}
|
||||
|
||||
/** An XOVER overview entry as returned by nntp-js. */
|
||||
export type OverviewHeader = Record<string, string>
|
||||
|
||||
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
||||
const xrefGroups = parseXrefGroups(header.xref)
|
||||
return {
|
||||
poster: header.from ?? '',
|
||||
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
||||
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
||||
articleNumber,
|
||||
postedAt: header.date ? new Date(header.date) : undefined,
|
||||
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||
export function segmentFromHeader(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||
if (!parsed || !header['message-id']) return null
|
||||
const base = common(header, groupName, articleNumber)
|
||||
return {
|
||||
...base,
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
subject: parsed.base,
|
||||
name: parsed.name,
|
||||
totalParts: parsed.totalSegments,
|
||||
number: parsed.segment,
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
export interface ParsedSubject {
|
||||
/** Filename from the quoted part of the subject. */
|
||||
name: string
|
||||
/** Segment number `k` from `(k/N)`. */
|
||||
segment: number
|
||||
/** Segment count `N` from `(k/N)`. */
|
||||
totalSegments: number
|
||||
/** File counter `[i/M]`, when present. */
|
||||
fileIndex?: number
|
||||
fileTotal?: number
|
||||
/**
|
||||
* The subject with the segment counter removed. Identical for every segment
|
||||
* of one uploaded file, so it is part of the file identity (see fileKey).
|
||||
*/
|
||||
base: string
|
||||
}
|
||||
|
||||
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
|
||||
|
||||
/**
|
||||
* Parses a binary post subject such as
|
||||
* `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`.
|
||||
*
|
||||
* Returns null for subjects without a quoted filename and segment counter
|
||||
* (obfuscated posts), which need their yEnc header read from the body instead.
|
||||
*/
|
||||
export function parseSubject(subject: string): ParsedSubject | null {
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name) return null
|
||||
|
||||
// The segment counter is the last "(k/N)" after the filename; a file size may follow it.
|
||||
let counter: RegExpExecArray | undefined
|
||||
for (const match of subject.matchAll(SEGMENT_COUNTER)) {
|
||||
if (match.index > nameEnd) counter = match
|
||||
}
|
||||
if (!counter) return null
|
||||
|
||||
const segment = Number.parseInt(counter[1], 10)
|
||||
const totalSegments = Number.parseInt(counter[2], 10)
|
||||
if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null
|
||||
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
|
||||
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
|
||||
return {
|
||||
name,
|
||||
segment,
|
||||
totalSegments,
|
||||
fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined,
|
||||
fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined,
|
||||
base,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
export interface GroupInfo {
|
||||
count: bigint
|
||||
first: bigint
|
||||
last: bigint
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1).
|
||||
*
|
||||
* nntp-js's own `group()` destructures this reply without skipping the status
|
||||
* code, so its `first`/`last` are off by one field. Pass it `result.resp`.
|
||||
*/
|
||||
export function parseGroupResponse(resp: string): GroupInfo {
|
||||
const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim())
|
||||
if (!match) throw new Error(`Unexpected GROUP response: ${resp}`)
|
||||
return {
|
||||
count: BigInt(match[1]),
|
||||
first: BigInt(match[2]),
|
||||
last: BigInt(match[3]),
|
||||
name: match[4],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
|
||||
export { parseXrefGroups } from './xref.js'
|
||||
export { parseGroupResponse, type GroupInfo } from './group.js'
|
||||
@@ -0,0 +1,238 @@
|
||||
import { promisify } from 'node:util'
|
||||
import net from 'node:net'
|
||||
import tls from 'node:tls'
|
||||
import { deflate, deflateSync, unzipSync } from 'node:zlib'
|
||||
|
||||
const deflateAsync = promisify(deflate)
|
||||
|
||||
export interface XoverClientOptions {
|
||||
host: string
|
||||
port: number
|
||||
user?: string
|
||||
password?: string
|
||||
secure?: boolean
|
||||
/** Hostname the TLS certificate must match, when connecting through a tunnel (default: host). */
|
||||
servername?: string
|
||||
/** A reply with no data for this long fails, and the connection with it (inactivity, not duration). */
|
||||
timeoutMs?: number
|
||||
/** Ask for gzip-compressed overview data (XFEATURE COMPRESS GZIP), when the server has it. */
|
||||
compress?: boolean
|
||||
}
|
||||
|
||||
export type Overview = [number, Record<string, string>]
|
||||
|
||||
/** The overview fields in RFC 3977 order, named as nntp-js names them (segmentFromHeader reads these). */
|
||||
const FIELDS = ['subject', 'from', 'date', 'message-id', 'references', ':bytes', ':lines', 'xref']
|
||||
const TERMINATOR = Buffer.from('\r\n.\r\n')
|
||||
const DOT_LINE = Buffer.from('.\r\n')
|
||||
|
||||
/**
|
||||
* A minimal NNTP client for seeding nodes (CONCEPT.md D37): one connection,
|
||||
* GROUP and XOVER only, with gzip-compressed overviews. nntp-js can't read
|
||||
* those: after "224 … [COMPRESS=GZIP]" the lines come as one zlib stream,
|
||||
* followed by a plain "." line; an empty range is a 420 without body.
|
||||
* About six times less traffic per header (LOG.md 2026-10-05).
|
||||
*/
|
||||
export class XoverClient {
|
||||
private chunks: Buffer[] = []
|
||||
private length = 0
|
||||
private wake?: () => void
|
||||
private failure?: Error
|
||||
compressed = false
|
||||
|
||||
private constructor(
|
||||
private sock: net.Socket,
|
||||
private timeoutMs: number
|
||||
) {
|
||||
sock.on('data', (chunk: Buffer) => {
|
||||
this.chunks.push(chunk)
|
||||
this.length += chunk.length
|
||||
this.wake?.()
|
||||
})
|
||||
const fail = (error: Error) => {
|
||||
this.failure ??= error
|
||||
this.wake?.()
|
||||
}
|
||||
sock.on('error', fail)
|
||||
sock.on('close', () => fail(new Error('NNTP connection closed')))
|
||||
sock.on('timeout', () => {
|
||||
fail(new Error('NNTP reply timed out'))
|
||||
sock.destroy()
|
||||
})
|
||||
}
|
||||
|
||||
static async connect(options: XoverClientOptions): Promise<XoverClient> {
|
||||
const sock = options.secure
|
||||
? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host })
|
||||
: net.connect({ host: options.host, port: options.port })
|
||||
const client = new XoverClient(sock, options.timeoutMs ?? 120_000)
|
||||
sock.setTimeout(client.timeoutMs)
|
||||
try {
|
||||
await client.expect(await client.line(), ['200', '201'])
|
||||
if (options.user) {
|
||||
const reply = await client.command(`AUTHINFO USER ${options.user}`)
|
||||
if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281'])
|
||||
else await client.expect(reply, ['281'])
|
||||
}
|
||||
if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
} catch (error) {
|
||||
client.close()
|
||||
throw error
|
||||
}
|
||||
// Idle between commands is fine (a node waiting for the seed API); only replies are timed.
|
||||
sock.setTimeout(0)
|
||||
return client
|
||||
}
|
||||
|
||||
async group(name: string): Promise<{ count: number; low: number; high: number }> {
|
||||
return this.timed(() => this.selectGroup(name))
|
||||
}
|
||||
|
||||
/** Overviews of articles `from`..`to` in the current group; an empty range gives []. */
|
||||
async xover(from: number, to: number): Promise<Overview[]> {
|
||||
return this.timed(() => this.overviews(from, to))
|
||||
}
|
||||
|
||||
/**
|
||||
* Overviews of `from`..`to` as one zlib/gzip stream, as the server sent it when it
|
||||
* compresses (plain replies are deflated here): the seed API passes these to nodes
|
||||
* without unpacking them (CONCEPT.md D38). `parseOverviews(unzipSync(raw).toString())`.
|
||||
*/
|
||||
async xoverRaw(from: number, to: number): Promise<Buffer> {
|
||||
return this.timed(async () => {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0))
|
||||
await this.expect(reply, ['224'])
|
||||
if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw
|
||||
// Plain replies (the relay's default): compressed off the event loop, at the fastest level.
|
||||
return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
|
||||
})
|
||||
}
|
||||
|
||||
/** A reply that stalls for timeoutMs fails the connection; idle time between commands doesn't count. */
|
||||
private async timed<T>(exchange: () => Promise<T>): Promise<T> {
|
||||
this.sock.setTimeout(this.timeoutMs)
|
||||
try {
|
||||
return await exchange()
|
||||
} finally {
|
||||
this.sock.setTimeout(0)
|
||||
}
|
||||
}
|
||||
|
||||
private async selectGroup(name: string): Promise<{ count: number; low: number; high: number }> {
|
||||
const reply = await this.command(`GROUP ${name}`)
|
||||
await this.expect(reply, ['211'])
|
||||
const [, count, low, high] = reply.split(' ')
|
||||
return { count: Number(count), low: Number(low), high: Number(high) }
|
||||
}
|
||||
|
||||
private async overviews(from: number, to: number): Promise<Overview[]> {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return []
|
||||
await this.expect(reply, ['224'])
|
||||
const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody()
|
||||
return parseOverviews(data)
|
||||
}
|
||||
|
||||
close() {
|
||||
try {
|
||||
this.sock.end('QUIT\r\n')
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
this.sock.destroy()
|
||||
}
|
||||
|
||||
private async command(line: string): Promise<string> {
|
||||
this.sock.write(`${line}\r\n`)
|
||||
return this.line()
|
||||
}
|
||||
|
||||
private async expect(reply: string, codes: string[]) {
|
||||
if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
}
|
||||
|
||||
private buffer(): Buffer {
|
||||
if (this.chunks.length > 1) this.chunks = [Buffer.concat(this.chunks, this.length)]
|
||||
return this.chunks[0] ?? Buffer.alloc(0)
|
||||
}
|
||||
|
||||
private take(n: number): Buffer {
|
||||
const all = this.buffer()
|
||||
const out = all.subarray(0, n)
|
||||
const rest = all.subarray(n)
|
||||
this.chunks = rest.length > 0 ? [rest] : []
|
||||
this.length = rest.length
|
||||
return out
|
||||
}
|
||||
|
||||
private async more() {
|
||||
if (this.failure) throw this.failure
|
||||
await new Promise<void>((resolve) => (this.wake = resolve))
|
||||
this.wake = undefined
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
}
|
||||
|
||||
private async line(): Promise<string> {
|
||||
while (true) {
|
||||
const at = this.buffer().indexOf('\r\n')
|
||||
if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1')
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
|
||||
/** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
|
||||
private async plainBody(): Promise<string> {
|
||||
let searched = 0
|
||||
while (true) {
|
||||
const all = this.buffer()
|
||||
// An empty body is just ".\r\n".
|
||||
if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
|
||||
this.take(3)
|
||||
return ''
|
||||
}
|
||||
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
|
||||
if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.')
|
||||
searched = all.length
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
|
||||
private async compressedBody(): Promise<string> {
|
||||
return (await this.compressedRaw()).text
|
||||
}
|
||||
|
||||
/** A zlib/gzip stream followed by a "." line: complete once it ends in ".\r\n" and inflates. */
|
||||
private async compressedRaw(): Promise<{ raw: Buffer; text: string }> {
|
||||
while (true) {
|
||||
const all = this.buffer()
|
||||
if (all.length > DOT_LINE.length && all.subarray(all.length - DOT_LINE.length).equals(DOT_LINE)) {
|
||||
const raw = all.subarray(0, all.length - DOT_LINE.length)
|
||||
try {
|
||||
const text = unzipSync(raw).toString('utf8')
|
||||
this.take(all.length)
|
||||
return { raw: Buffer.from(raw), text }
|
||||
} catch {
|
||||
// ".\r\n" inside the compressed data: wait for more.
|
||||
}
|
||||
}
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Overview lines (tab-separated, after the article number) as nntp-js returns them. */
|
||||
export function parseOverviews(data: string): Overview[] {
|
||||
const out: Overview[] = []
|
||||
for (const line of data.split('\r\n')) {
|
||||
if (!line) continue
|
||||
const [number, ...tokens] = line.split('\t')
|
||||
const fields: Record<string, string> = {}
|
||||
for (let i = 0; i < FIELDS.length && i < tokens.length; i++) {
|
||||
// Full fields after the defaults carry their name: "Xref: host group:number".
|
||||
fields[FIELDS[i]] = i >= 7 ? tokens[i].replace(/^[^:\s]+:\s*/, '') : tokens[i]
|
||||
}
|
||||
out.push([Number.parseInt(number, 10), fields])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Returns the group names from an Xref header,
|
||||
* e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`.
|
||||
*/
|
||||
export function parseXrefGroups(xref: string | undefined): string[] {
|
||||
if (!xref) return []
|
||||
return xref
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.slice(1)
|
||||
.map((entry) => entry.slice(0, entry.lastIndexOf(':')))
|
||||
.filter(Boolean)
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { looksLikeFilename } from '../headers/file_key.js'
|
||||
|
||||
export interface ReleaseIdentity {
|
||||
key: string
|
||||
/** Human-readable name: the subject text or the shared filename stem. */
|
||||
name: string
|
||||
/** Number of files announced by the `[i/M]` counter, when present. */
|
||||
fileTotal?: number
|
||||
/** Which rule recognised the release. */
|
||||
source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
|
||||
}
|
||||
|
||||
export interface ReleaseInput {
|
||||
poster: string
|
||||
/** Subject without segment counter (the binary's `subject`). */
|
||||
subject: string
|
||||
/** Filename of the binary. */
|
||||
name: string
|
||||
/** True when the identity came from the yEnc body (obfuscated post). */
|
||||
weak: boolean
|
||||
}
|
||||
|
||||
// Suffixes that differ between the files of one release. Stripped repeatedly,
|
||||
// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
|
||||
const SUFFIXES = [
|
||||
/\.vol\d+[+-]\d+\.par2$/i,
|
||||
/\.par2$/i,
|
||||
/\.part\d+\.rar$/i,
|
||||
/\.r\d{2,3}$/i,
|
||||
/\.rar$/i,
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
export function fileStem(name: string): string {
|
||||
let stem = name.trim()
|
||||
let changed = true
|
||||
while (changed) {
|
||||
changed = false
|
||||
for (const suffix of SUFFIXES) {
|
||||
const next = stem.replace(suffix, '')
|
||||
if (next !== stem && next.length > 0) {
|
||||
stem = next
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return stem
|
||||
}
|
||||
|
||||
/**
|
||||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||||
* removed. Posters that name the release outside the quotes put it here.
|
||||
*/
|
||||
export function subjectRemainder(subject: string): string {
|
||||
return subject
|
||||
.replace(/"[^"]*"/g, ' ')
|
||||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||||
.replace(/\byEnc\b/gi, ' ')
|
||||
.replace(/[\s\-–.:|]+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||||
* name once those are removed identifies a release.
|
||||
*/
|
||||
function isRandomToken(token: string): boolean {
|
||||
const t = token.replace(/^\[|\]$/g, '')
|
||||
if (t.length < 6) return false
|
||||
if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
|
||||
return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
|
||||
}
|
||||
|
||||
/**
|
||||
* A stable per-post token: one short run of letters/digits with at least one
|
||||
* digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
|
||||
* grouping different posts of the same poster.
|
||||
*/
|
||||
export function isPostToken(remainder: string): boolean {
|
||||
const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
|
||||
return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
|
||||
}
|
||||
|
||||
export function namesRelease(remainder: string): boolean {
|
||||
const words = remainder
|
||||
.split(/\s+/)
|
||||
.filter((w) => w && !isRandomToken(w))
|
||||
.join(' ')
|
||||
// At least two words of letters, joined the way release names are.
|
||||
return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
|
||||
}
|
||||
|
||||
function hash(...parts: (string | number | undefined)[]) {
|
||||
return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides which release (download) a file belongs to. Returns null when the
|
||||
* file cannot be grouped from its name alone (random extensionless names in
|
||||
* obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
|
||||
*/
|
||||
export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||||
const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
||||
const stem = fileStem(input.name)
|
||||
|
||||
if (input.weak) {
|
||||
// Obfuscating posters randomise the poster per article; a real filename
|
||||
// stem is specific enough on its own.
|
||||
if (!looksLikeFilename(input.name)) return null
|
||||
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
|
||||
}
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
if (namesRelease(remainder)) {
|
||||
return {
|
||||
key: hash('subject', input.poster, remainder),
|
||||
name: remainder,
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
}
|
||||
}
|
||||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||||
// [i/M] counter it groups the files of one post whose names are random.
|
||||
if (fileTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||||
}
|
||||
// The [i/M] total is informational, not part of the key: posters often
|
||||
// upload the par2 index without a counter, and it belongs to the same post.
|
||||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||||
}
|
||||
|
||||
/** Release of a par2 recovery set; every file listed in it belongs to it. */
|
||||
export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
|
||||
return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
|
||||
}
|
||||
|
||||
const SEPARATOR = /[._ -]/
|
||||
|
||||
/**
|
||||
* A release name from the files a par2 set lists. Files that differ only in
|
||||
* the middle (episodes S03E01…S03E20, parts) give their shared name, with
|
||||
* the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
|
||||
* SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
|
||||
* Otherwise the longest stem, which tends to be the most descriptive (the
|
||||
* video file).
|
||||
*/
|
||||
export function par2ReleaseName(fileNames: string[]): string | undefined {
|
||||
const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
|
||||
if (stems.length === 0) return undefined
|
||||
const longest = [...stems].sort((a, b) => b.length - a.length)[0]
|
||||
if (stems.length < 2) return longest
|
||||
|
||||
let prefix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[i] === s[i]) i++
|
||||
return p.slice(0, i)
|
||||
})
|
||||
let suffix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
|
||||
return p.slice(p.length - i)
|
||||
})
|
||||
const shortest = Math.min(...stems.map((s) => s.length))
|
||||
if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)
|
||||
|
||||
// Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
|
||||
const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
|
||||
if (season) prefix = season[1]
|
||||
else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
|
||||
prefix = prefix.replace(/[^._ -]*$/, '')
|
||||
}
|
||||
suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
|
||||
prefix = prefix.replace(/[._ -]+$/, '')
|
||||
|
||||
const name = suffix ? prefix + suffix : prefix
|
||||
// Only when the shared part carries most of the name (not just "The.").
|
||||
return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
|
||||
}
|
||||
|
||||
/**
|
||||
* Release of a RAR volume set, from what its headers list: the shared
|
||||
* top-level directory if there is one, otherwise the first entry. For
|
||||
* volumes after the first, the first entry is the file that continues from
|
||||
* the previous volume, so all volumes of a single-file archive agree.
|
||||
*/
|
||||
/**
|
||||
* Folder and file names that say nothing about what a release is: disc
|
||||
* structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers.
|
||||
* Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i
|
||||
|
||||
export function isGenericName(name: string): boolean {
|
||||
return GENERIC_NAMES.test(name.trim())
|
||||
}
|
||||
|
||||
export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
|
||||
if (entries.length === 0) return null
|
||||
const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
const nested = entries.some((e) => /[\\/]/.test(e.name))
|
||||
// A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead.
|
||||
if (nested && tops.size === 1 && !isGenericName([...tops][0])) {
|
||||
const [top] = tops
|
||||
return { key: hash('rar-dir', top), name: top, source: 'rar' }
|
||||
}
|
||||
const first = entries[0]
|
||||
const base = first.name.split(/[\\/]/).pop() ?? first.name
|
||||
return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Grouping of files into releases (downloads), independent of AdonisJS.
|
||||
*/
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
|
||||
export { looksRandom } from './random.js'
|
||||
@@ -0,0 +1,27 @@
|
||||
import { isGenericName } from './identity.js'
|
||||
|
||||
/**
|
||||
* Obfuscated names: one long token of mixed letters and digits, no
|
||||
* separators, and the other generated shapes seen on Usenet (LOG.md
|
||||
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
// VIDEO_TS, Sample, CD1: real words, but no name.
|
||||
if (isGenericName(name)) return true
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
|
||||
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
|
||||
// UUIDs
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
|
||||
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
|
||||
if (/^\d{6,}( n)?$/.test(name)) return true
|
||||
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
|
||||
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
|
||||
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
|
||||
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
||||
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js'
|
||||
@@ -0,0 +1,69 @@
|
||||
import { explored, signatureKey, uploadTool } from './upload_tools.js'
|
||||
import { fileStem, looksRandom } from '../release/index.js'
|
||||
|
||||
/**
|
||||
* What a worker drops, and why (REBUILD.md, rules and recovery). Sent with
|
||||
* every lease; `version` is recorded on the range, so a changed rule set
|
||||
* shows which ranges to read again.
|
||||
*/
|
||||
export interface Rules {
|
||||
version: number
|
||||
/** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */
|
||||
priorities: Record<string, number>
|
||||
/** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */
|
||||
droppedSignatures: string[]
|
||||
/** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */
|
||||
droppedTools: string[]
|
||||
/** A random-named file of at most this many parts: a random subject per article, needs bodies. */
|
||||
perArticleMaxParts: number
|
||||
/** Share of each rule's drops kept in full as a sample (by a hash of the release key). */
|
||||
samplePercent: number
|
||||
}
|
||||
|
||||
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||
version: 1,
|
||||
droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'],
|
||||
perArticleMaxParts: 3,
|
||||
samplePercent: 1,
|
||||
}
|
||||
|
||||
export type Verdict = { rule: null } | { rule: 'garbage' | 'per-article'; key: string; sample: boolean }
|
||||
|
||||
export function isRandomFile(name: string) {
|
||||
return looksRandom(fileStem(name))
|
||||
}
|
||||
|
||||
/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
|
||||
export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
|
||||
let owner: string | null = null
|
||||
for (const g of groups) {
|
||||
const p = priorities[g]
|
||||
if (p !== undefined && (owner === null || p < priorities[owner])) owner = g
|
||||
}
|
||||
return owner
|
||||
}
|
||||
|
||||
export class RuleSet {
|
||||
private signatures: Set<string>
|
||||
private tools: Set<string>
|
||||
constructor(readonly rules: Rules) {
|
||||
this.signatures = new Set(rules.droppedSignatures)
|
||||
this.tools = new Set(rules.droppedTools)
|
||||
}
|
||||
|
||||
/**
|
||||
* A release's verdict from its files: kept as a whole when any file has a
|
||||
* readable name; else dropped as per-article random subjects when every
|
||||
* file has few parts, as garbage when its signature or tool is dropped.
|
||||
*/
|
||||
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
|
||||
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
|
||||
const first = files[0]
|
||||
const tool = uploadTool(first.firstMessageId)
|
||||
const sample = explored(releaseKey, this.rules.samplePercent)
|
||||
if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
|
||||
const group = first.groups[0] ?? ''
|
||||
if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }
|
||||
return { rule: null }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Upload signatures (CONCEPT.md D42): which tool posted an article, read from
|
||||
* its Message-ID, and the group it was posted to first. Together they predict
|
||||
* whether a random-named upload will ever get a name: ngPost uploads in
|
||||
* a.b.ath almost never do, nyuu uploads in a.b.friends always do (LOG.md
|
||||
* 2026-10-08 (5)).
|
||||
*/
|
||||
const TOOLS: [string, RegExp][] = [
|
||||
// FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu: 12 random capital+small letter pairs, a ms timestamp
|
||||
['nyuu', /^(?:[A-Z][a-z]){12}-\d{13}@nyuu$/],
|
||||
['nyuu-host', /^(?:[A-Z][a-z]){12}-\d{13}@.+$/],
|
||||
// b94268bc702040f7b811d5318f727e4f@ngPost: a UUID's hex
|
||||
['ngPost', /^[0-9a-f]{32}@ngPost$/],
|
||||
['yuun', /^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$/],
|
||||
['hex32', /^[0-9a-f]{32}@[A-Za-z0-9.-]+$/],
|
||||
['JBinUp', /@JBin(?:Up|Down)\.local$/],
|
||||
// part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local
|
||||
['partNofM', /^part\d+of\d+\./i],
|
||||
// 1790923856877$1729905757859$1@localhost
|
||||
['ts-dollar', /^\d{13}\$\d+\$\d+@localhost$/],
|
||||
// t3Y4mV92YCp0McxDc5@zLZlsw.RYqes: random local part and random host
|
||||
['random', /^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$/],
|
||||
]
|
||||
|
||||
/** The posting tool of a Message-ID (without angle brackets), or 'other'. */
|
||||
export function uploadTool(messageId: string | null | undefined): string {
|
||||
if (!messageId) return 'other'
|
||||
for (const [tool, pattern] of TOOLS) if (pattern.test(messageId)) return tool
|
||||
return 'other'
|
||||
}
|
||||
|
||||
/** The group of a signature: the first the upload was posted to, as stored in `groups`. */
|
||||
export function signatureGroup(groups: unknown): string {
|
||||
return Array.isArray(groups) && typeof groups[0] === 'string' ? groups[0] : ''
|
||||
}
|
||||
|
||||
/** The key of a signature in a Set. */
|
||||
export function signatureKey(tool: string, group: string) {
|
||||
return `${tool}\t${group}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an upload is in the share of dropped signatures that is still kept
|
||||
* (and probed), so their rates keep being measured. Decided per release key,
|
||||
* so a release is kept or dropped as a whole.
|
||||
*/
|
||||
export function explored(key: string, percent: number): boolean {
|
||||
if (percent <= 0) return false
|
||||
return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { decodeChunk, encodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
|
||||
test.group('chunks / codec', () => {
|
||||
test('round-trips files with their segments, part numbers when not 1..N', ({ assert }) => {
|
||||
const files: ChunkFile[] = [
|
||||
{ name: 'a.part1.rar', poster: 'p <p@x>', subject: 'A [1/2] - "a.part1.rar" yEnc', postedAt: 1_700_000_000_000, groups: ['alt.binaries.test'], segments: [[1, 'one@x', 700_000], [2, 'two@x', 1234]] },
|
||||
{ name: 'a.par2', poster: 'p <p@x>', subject: 'A [2/2] - "a.par2" yEnc', postedAt: null, groups: ['alt.binaries.test', 'alt.binaries.x'], segments: [[2, 'gap@x', 10], [5, 'gap2@x', 11]] },
|
||||
]
|
||||
const data = encodeChunk(files)
|
||||
assert.equal(data[0], 1)
|
||||
assert.deepEqual(decodeChunk(data), files)
|
||||
})
|
||||
|
||||
test('refuses an unknown version', ({ assert }) => {
|
||||
assert.throws(() => decodeChunk(Buffer.from([9, 0])), /Unknown chunk version 9/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { FileAggregator, type AggregatableSegment } from '#lib/files/index'
|
||||
|
||||
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
|
||||
fileKey,
|
||||
number,
|
||||
totalParts,
|
||||
messageId,
|
||||
bytes: 100,
|
||||
})
|
||||
|
||||
test.group('aggregate / FileAggregator', () => {
|
||||
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
|
||||
assert.equal(aggregator.openFiles, 1)
|
||||
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
|
||||
assert.equal(done.fileKey, 'a')
|
||||
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
|
||||
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
|
||||
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
|
||||
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
|
||||
})
|
||||
|
||||
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
aggregator.add([seg('old', 1, 3)])
|
||||
aggregator.add([seg('new', 1, 3)])
|
||||
aggregator.add([seg('new', 2, 3)])
|
||||
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
|
||||
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fileKey, parseSubject } from '#lib/headers/index'
|
||||
import { parseXrefGroups } from '#lib/nntp/index'
|
||||
|
||||
test.group('subject / parseSubject', () => {
|
||||
test('parses file counter, filename and segment counter', ({ assert }) => {
|
||||
assert.deepEqual(parseSubject('. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc (56/274)'), {
|
||||
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
|
||||
segment: 56,
|
||||
totalSegments: 274,
|
||||
fileIndex: 9,
|
||||
fileTotal: 51,
|
||||
base: '. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc',
|
||||
})
|
||||
})
|
||||
|
||||
test('handles a file size after the segment counter', ({ assert }) => {
|
||||
const subject = '[8/9] - "x.tar.zst.vol063+064.par2" yEnc (37/72) 51315352'
|
||||
const parsed = parseSubject(subject)!
|
||||
assert.equal(parsed.segment, 37)
|
||||
assert.equal(parsed.totalSegments, 72)
|
||||
assert.equal(parsed.base, '[8/9] - "x.tar.zst.vol063+064.par2" yEnc 51315352')
|
||||
})
|
||||
|
||||
test('ignores counters inside the filename', ({ assert }) => {
|
||||
const parsed = parseSubject('"Show (1/2) Special.mkv" yEnc (3/10)')!
|
||||
assert.equal(parsed.name, 'Show (1/2) Special.mkv')
|
||||
assert.equal(parsed.segment, 3)
|
||||
assert.equal(parsed.totalSegments, 10)
|
||||
})
|
||||
|
||||
test('the base is equal for all segments of one file', ({ assert }) => {
|
||||
const a = parseSubject('[1/9] - "a.tar.zst" yEnc (253/1000) 716554110')!
|
||||
const b = parseSubject('[1/9] - "a.tar.zst" yEnc (254/1000) 716554110')!
|
||||
assert.equal(a.base, b.base)
|
||||
})
|
||||
|
||||
test('returns null for obfuscated or invalid subjects', ({ assert }) => {
|
||||
assert.isNull(parseSubject('GZFDJc0hB3Y0FwwAznkk'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (0/5)'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (6/5)'))
|
||||
assert.isNull(parseSubject('"" yEnc (1/5)'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('subject / fileKey', () => {
|
||||
test('differs per poster and per segment count, is stable otherwise', ({ assert }) => {
|
||||
const key = fileKey('poster@a', '"x.rar" yEnc', 10)
|
||||
assert.lengthOf(key, 40)
|
||||
assert.equal(key, fileKey('poster@a', '"x.rar" yEnc', 10))
|
||||
assert.notEqual(key, fileKey('poster@b', '"x.rar" yEnc', 10))
|
||||
assert.notEqual(key, fileKey('poster@a', '"x.rar" yEnc', 11))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('nntp / parseXrefGroups', () => {
|
||||
test('extracts crossposted groups', ({ assert }) => {
|
||||
assert.deepEqual(
|
||||
parseXrefGroups('news.eweka.nl alt.binaries.nl:35801424104 alt.binaries.test:27275430913'),
|
||||
['alt.binaries.nl', 'alt.binaries.test']
|
||||
)
|
||||
assert.deepEqual(parseXrefGroups(undefined), [])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,18 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { parseGroupResponse } from '#lib/nntp/index'
|
||||
|
||||
test.group('nntp / parseGroupResponse', () => {
|
||||
test('parses the reply from news.eweka.nl', ({ assert }) => {
|
||||
// Captured 2026-09-27. count == last - first + 1, so the field order is standard.
|
||||
assert.deepEqual(parseGroupResponse('211 27275164317 266599 27275430915 alt.binaries.test'), {
|
||||
count: 27275164317n,
|
||||
first: 266599n,
|
||||
last: 27275430915n,
|
||||
name: 'alt.binaries.test',
|
||||
})
|
||||
})
|
||||
|
||||
test('rejects non-211 replies', ({ assert }) => {
|
||||
assert.throws(() => parseGroupResponse('411 No such group'), /Unexpected GROUP response/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
import { test } from '@japa/runner'
|
||||
import net from 'node:net'
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { XoverClient, parseOverviews } from '#lib/nntp/index'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
|
||||
const LINES = [
|
||||
'101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<one@test>\t\t700000\t5000\tXref: news.example a.b.test:101',
|
||||
'102\tShow.S01E01 - "a.mkv" yEnc (2/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:01 +0000\t<two@test>\t\t650000\t4600\tXref: news.example a.b.test:102 a.b.other:7',
|
||||
]
|
||||
|
||||
/** A tiny NNTP server: plain replies, gzip after XFEATURE, 420 for 1-1, replies split into small chunks. */
|
||||
function server(): Promise<{ port: number; close: () => void }> {
|
||||
return new Promise((resolve) => {
|
||||
const srv = net.createServer((sock) => {
|
||||
let gzip = false
|
||||
const send = (data: Buffer) => {
|
||||
for (let i = 0; i < data.length; i += 7) sock.write(data.subarray(i, i + 7))
|
||||
}
|
||||
sock.write('200 welcome\r\n')
|
||||
let pending = ''
|
||||
sock.on('data', (chunk) => {
|
||||
pending += chunk.toString()
|
||||
let at: number
|
||||
while ((at = pending.indexOf('\r\n')) !== -1) {
|
||||
const line = pending.slice(0, at)
|
||||
pending = pending.slice(at + 2)
|
||||
if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
|
||||
else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
|
||||
else if (line === 'XFEATURE COMPRESS GZIP') (gzip = true), sock.write('290 Feature Enabled\r\n')
|
||||
else if (line.startsWith('GROUP')) sock.write('211 2 101 102 a.b.test\r\n')
|
||||
else if (line === 'XOVER 1-1') sock.write('420 No Articles Selected\r\n')
|
||||
else if (line.startsWith('XOVER')) {
|
||||
const body = Buffer.from(LINES.join('\r\n') + '\r\n')
|
||||
if (gzip) {
|
||||
sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n')
|
||||
send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')]))
|
||||
} else {
|
||||
sock.write('224 Overview Information Follows\r\n')
|
||||
send(Buffer.concat([body, Buffer.from('.\r\n')]))
|
||||
}
|
||||
} else if (line === 'QUIT') sock.end('205 bye\r\n')
|
||||
}
|
||||
})
|
||||
})
|
||||
srv.listen(0, '127.0.0.1', () => resolve({ port: (srv.address() as net.AddressInfo).port, close: () => srv.close() }))
|
||||
})
|
||||
}
|
||||
|
||||
test.group('nntp / XoverClient', () => {
|
||||
test('parses overview lines into the fields nntp-js gives', ({ assert }) => {
|
||||
const [[number, fields]] = parseOverviews(LINES[0] + '\r\n')
|
||||
assert.equal(number, 101)
|
||||
assert.equal(fields['message-id'], '<one@test>')
|
||||
assert.equal(fields[':bytes'], '700000')
|
||||
assert.equal(fields.xref, 'news.example a.b.test:101')
|
||||
})
|
||||
|
||||
for (const compress of [false, true]) {
|
||||
test(`fetches overviews ${compress ? 'gzip-compressed' : 'plain'}, and an empty range`, async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
|
||||
try {
|
||||
assert.equal(client.compressed, compress)
|
||||
assert.deepEqual(await client.group('a.b.test'), { count: 2, low: 101, high: 102 })
|
||||
const overviews = await client.xover(101, 102)
|
||||
assert.deepEqual(overviews.map(([n]) => n), [101, 102])
|
||||
assert.equal(overviews[1][1].subject, 'Show.S01E01 - "a.mkv" yEnc (2/2)')
|
||||
assert.deepEqual(await client.xover(1, 1), [])
|
||||
assert.lengthOf(await client.xover(101, 102), 2)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('idle time between commands does not count against the reply timeout', async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: true, timeoutMs: 200 })
|
||||
try {
|
||||
await client.group('a.b.test')
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
assert.lengthOf(await client.xover(101, 102), 2)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
|
||||
for (const compress of [false, true]) {
|
||||
test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
|
||||
try {
|
||||
await client.group('a.b.test')
|
||||
const raw = await client.xoverRaw(101, 102)
|
||||
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [101, 102])
|
||||
assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,169 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index'
|
||||
import { looksLikeFilename, weakFileKey } from '#lib/headers/index'
|
||||
|
||||
test.group('release / fileStem', () => {
|
||||
test('reduces all files of a set to the same stem', ({ assert }) => {
|
||||
const names = [
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst',
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2',
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2',
|
||||
]
|
||||
assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715']))
|
||||
})
|
||||
|
||||
test('handles rar, split and par2 volume naming', ({ assert }) => {
|
||||
assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0')
|
||||
assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265')
|
||||
assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264')
|
||||
assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026')
|
||||
assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b')
|
||||
})
|
||||
|
||||
test('never reduces a name to nothing', ({ assert }) => {
|
||||
assert.equal(fileStem('.par2'), '.par2')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / subjectRemainder', () => {
|
||||
test('is empty when the subject only has counters, filename and size', ({ assert }) => {
|
||||
assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '')
|
||||
assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '')
|
||||
})
|
||||
|
||||
test('keeps a release name written outside the quotes', ({ assert }) => {
|
||||
assert.equal(
|
||||
subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'),
|
||||
'Some Show S01E02 1080p'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / releaseIdentity', () => {
|
||||
test('groups a par2 set with its archive by stem, per poster', ({ assert }) => {
|
||||
const base = { poster: 'p', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
||||
const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'x')
|
||||
assert.equal(a.fileTotal, 4)
|
||||
const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
||||
assert.notEqual(a.key, other.key)
|
||||
})
|
||||
|
||||
test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => {
|
||||
const base = { poster: 'nicovideo_archiver', weak: false }
|
||||
const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })!
|
||||
const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })!
|
||||
assert.equal(index.key, archive.key)
|
||||
assert.equal(archive.fileTotal, 6)
|
||||
assert.isUndefined(index.fileTotal)
|
||||
})
|
||||
|
||||
test('uses the subject text when it names the release', ({ assert }) => {
|
||||
const base = { poster: 'p', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })!
|
||||
const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'My Release')
|
||||
})
|
||||
|
||||
test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => {
|
||||
const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })!
|
||||
const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })!
|
||||
assert.equal(a.key, b.key)
|
||||
})
|
||||
|
||||
test('random extensionless names cannot be grouped by name', ({ assert }) => {
|
||||
const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'
|
||||
assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true }))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('subject / weakFileKey', () => {
|
||||
test('ignores random per-article names, keeps real filenames', ({ assert }) => {
|
||||
assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'))
|
||||
assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar'))
|
||||
// Same file, different random names per article: same key.
|
||||
assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7))
|
||||
// Equal-sized volumes with real names: different keys.
|
||||
assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7))
|
||||
assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / rarReleaseIdentity', () => {
|
||||
test('groups volumes by the shared top-level directory', async ({ assert }) => {
|
||||
const { rarReleaseIdentity } = await import('#lib/release/index')
|
||||
const first = rarReleaseIdentity([
|
||||
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 },
|
||||
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 },
|
||||
])!
|
||||
const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])!
|
||||
assert.equal(first.key, later.key)
|
||||
assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM')
|
||||
assert.equal(first.source, 'rar')
|
||||
})
|
||||
|
||||
test('without a directory, volumes of one file share its first entry', async ({ assert }) => {
|
||||
const { rarReleaseIdentity } = await import('#lib/release/index')
|
||||
const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
||||
const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'Some.Show.S01E01.1080p')
|
||||
assert.isNull(rarReleaseIdentity([]))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / namesRelease', () => {
|
||||
test('random per-file tags and poster ids are not a release name', async ({ assert }) => {
|
||||
const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index')
|
||||
const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc'
|
||||
assert.isFalse(namesRelease(subjectRemainder(subject)))
|
||||
const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })!
|
||||
assert.equal(a.source, 'stem')
|
||||
assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc')))
|
||||
assert.isTrue(namesRelease('My Release'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / post id', () => {
|
||||
test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => {
|
||||
const { releaseIdentity, isPostToken } = await import('#lib/release/index')
|
||||
const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })!
|
||||
const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.source, 'post')
|
||||
assert.equal(a.fileTotal, 18)
|
||||
const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })!
|
||||
assert.notEqual(a.key, other.key)
|
||||
|
||||
assert.isTrue(isPostToken('83110629 n'))
|
||||
assert.isFalse(isPostToken('PRiVATE'))
|
||||
assert.isFalse(isPostToken(''))
|
||||
// Without a counter there is nothing to bound the post: fall back to the stem.
|
||||
assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / par2 names', () => {
|
||||
test('names a season pack after what its episodes share', async ({ assert }) => {
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const eps = Array.from({ length: 20 }, (_, i) =>
|
||||
`SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`)
|
||||
assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato')
|
||||
// Movie plus sample and nfo: the movie's (longest) name.
|
||||
assert.equal(
|
||||
par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']),
|
||||
'Some.Movie.2024.1080p.BluRay.x264-GRP'
|
||||
)
|
||||
// Numbered parts: the shared name without the part number.
|
||||
assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP')
|
||||
assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rules/index'
|
||||
import { sevenZipVolume } from '#lib/files/index'
|
||||
|
||||
const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
|
||||
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath') => ({ name, totalParts, groups: [group], firstMessageId })
|
||||
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
||||
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
||||
|
||||
test.group('rules / owner', () => {
|
||||
test('a crosspost belongs to the indexed group with the lowest priority, whatever the Xref order', ({ assert }) => {
|
||||
const priorities = { 'alt.binaries.boneless': 1, 'alt.binaries.ath': 3, 'alt.binaries.added.later': 1001 }
|
||||
assert.equal(ownerOf(['alt.binaries.ath', 'alt.binaries.boneless'], priorities), 'alt.binaries.boneless')
|
||||
assert.equal(ownerOf(['alt.binaries.added.later', 'alt.binaries.ath'], priorities), 'alt.binaries.ath')
|
||||
assert.equal(ownerOf(['alt.binaries.not.indexed'], priorities), null)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rules / verdicts', () => {
|
||||
test('a readable file keeps its whole release', ({ assert }) => {
|
||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
||||
})
|
||||
|
||||
test('random names: per-article with few parts, garbage by tool or signature, else kept', ({ assert }) => {
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa', 2, NYUU)]), { rule: 'per-article', key: 'nyuu' })
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST)]), { rule: 'garbage', key: 'ngPost alt.binaries.ath' })
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU, 'alt.binaries.misc')]), { rule: 'garbage', key: 'nyuu alt.binaries.misc' })
|
||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU)]), { rule: null })
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rules / upload tools', () => {
|
||||
test('posting tools are told apart by their Message-IDs', ({ assert }) => {
|
||||
assert.equal(uploadTool(NYUU), 'nyuu')
|
||||
assert.equal(uploadTool(NGPOST), 'ngPost')
|
||||
assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
|
||||
assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
|
||||
assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
|
||||
assert.equal(uploadTool(null), 'other')
|
||||
})
|
||||
|
||||
test('the sampled share is stable per key and about the percentage', ({ assert }) => {
|
||||
assert.equal(explored('some-release', 2), explored('some-release', 2))
|
||||
const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
|
||||
assert.isAbove(kept, 150)
|
||||
assert.isBelow(kept, 250)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('files / volumes', () => {
|
||||
test('7z volume numbers', ({ assert }) => {
|
||||
assert.equal(sevenZipVolume('x.7z.001'), 1)
|
||||
assert.equal(sevenZipVolume('x.7z'), 0)
|
||||
assert.isNull(sevenZipVolume('x.rar'))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user