Daan MeijerandClaude Opus 5.5
7dc316c987
Overview chunks in, per batch out: releases as encoded chunks, probe candidates, counts per rule (crossposts per owner group, obfuscated, garbage, per-article, incomplete), a sample of what was dropped, and fragments near the range's edges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
68 lines
4.1 KiB
TypeScript
68 lines
4.1 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { RangeIndexer } from '#lib/edge/index'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
|
import type { Overview } from '#lib/nntp/index'
|
|
|
|
const GROUP = 'alt.binaries.test'
|
|
const rules = { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { 'alt.binaries.boneless': 1, [GROUP]: 2, 'alt.binaries.x': 3 } }
|
|
let id = 0
|
|
|
|
/** Overview lines of one posted file: all its parts, from article number `at` on. */
|
|
function post(at: number, name: string, parts: number, opts: { counter?: string; groups?: string[]; tool?: 'ngPost' | 'nyuu'; poster?: string } = {}): Overview[] {
|
|
return Array.from({ length: parts }, (_, i) => {
|
|
id++
|
|
const mid = opts.tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
|
|
const groups = opts.groups ?? [GROUP]
|
|
return [at + i, {
|
|
'subject': `${opts.counter ? `${opts.counter} - ` : ''}"${name}" yEnc (${i + 1}/${parts})`,
|
|
'from': opts.poster ?? 'poster <p@x>',
|
|
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
|
'message-id': `<${mid}>`,
|
|
':bytes': '700000',
|
|
'xref': `news.example ${groups.map((g, k) => `${g}:${at + i + k}`).join(' ')}`,
|
|
}] as Overview
|
|
})
|
|
}
|
|
|
|
test.group('edge / RangeIndexer', () => {
|
|
test('groups files into releases with probe candidates, and counts what the rules dropped', ({ assert }) => {
|
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
|
|
indexer.add([
|
|
...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 3, { counter: 'Some.Show.S01E01 [1/3]' }),
|
|
...post(100_010, 'Some.Show.S01E01.1080p.WEB.part2.rar', 3, { counter: 'Some.Show.S01E01 [2/3]' }),
|
|
...post(100_020, 'Some.Show.S01E01.1080p.WEB.par2', 1, { counter: 'Some.Show.S01E01 [3/3]' }),
|
|
// A crosspost whose owner is boneless (priority 1): skipped here.
|
|
...post(100_030, 'Other.Release.2020.mkv', 2, { groups: [GROUP, 'alt.binaries.boneless'] }),
|
|
// Random-named ngPost upload: garbage everywhere.
|
|
...post(100_040, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 4, { tool: 'ngPost' }),
|
|
// A random subject per article: per-article.
|
|
...post(100_050, 'Pq4ZmXc8VbN2LkJh6GfDs3', 1),
|
|
], 200_000)
|
|
indexer.add([[100_060, { 'subject': 'obfuscated', 'from': 'x', 'message-id': '<obf@test>', 'date': '', ':bytes': '1', 'xref': `news.example ${GROUP}:100060` }]], 300_000)
|
|
indexer.finish()
|
|
const batch = indexer.take()
|
|
|
|
assert.equal(batch.headers, 3 + 3 + 1 + 2 + 4 + 1 + 1)
|
|
assert.lengthOf(batch.releases, 1)
|
|
const [release] = batch.releases
|
|
assert.include(release, { files: 3, segments: 7, firstArticle: 100_000, lastArticle: 100_020 })
|
|
assert.deepEqual(decodeChunk(release.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar', 'Some.Show.S01E01.1080p.WEB.part2.rar'])
|
|
assert.deepEqual(batch.candidates.map((c) => [c.kind, c.file]).sort(), [['archive', 'Some.Show.S01E01.1080p.WEB.part1.rar'], ['par2', 'Some.Show.S01E01.1080p.WEB.par2']])
|
|
|
|
const counts = Object.fromEntries(batch.counts.map((c) => [`${c.rule} ${c.key}`.trim(), [c.articles, c.files]]))
|
|
assert.deepEqual(counts, { 'crosspost alt.binaries.boneless': [2, 0], 'garbage ngPost alt.binaries.test': [4, 1], 'per-article other': [1, 1], 'obfuscated': [1, 0] })
|
|
assert.equal(batch.next, 1_000_001)
|
|
})
|
|
|
|
test('a file cut off near an edge leaves as fragments, one in the middle is only counted', ({ assert }) => {
|
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
|
|
indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000)
|
|
indexer.finish()
|
|
const batch = indexer.take()
|
|
assert.deepEqual(batch.fragments.map((f) => [f.name, f.number]), [['Edge.File.2020.part1.rar', 1], ['Edge.File.2020.part1.rar', 2]])
|
|
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]])
|
|
assert.lengthOf(batch.releases, 0)
|
|
})
|
|
})
|