Daan MeijerandClaude Opus 5.5
fa7d4f3394
End to end (workstation, recent a.b.boneless ranges from the spool): a 2M range in 10–12 s, 166k–202k headers/s per worker process (edge:bench on one core: 215k). - Prefetcher: spooled_at was set against a total counted from the range's resume point, which shrinks as the worker advances: the last chunks were never fetched and the worker waited. Now counted against the whole range, only new chunks counted, and leased ranges are always checked against the files. Planned ranges ahead are bounded (it had spooled ever more); the spool size is measured every 30 s (a stat per file). - The stream sends an empty frame every 20 s while waiting for a chunk (Node's fetch gave up on a quiet body after 5 minutes). - A range taken over midway: the first batch starts at the resume point (it started at the range's start and was refused). - edge:node inflates the next chunk on the thread pool while indexing the current one, and doesn't wait for a post before indexing on (posts stay in order; a failed post stops the range). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
76 lines
4.5 KiB
TypeScript
76 lines
4.5 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { RangeIndexer } from '#lib/edge/index'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
|
import type { Overview } from '#lib/nntp/index'
|
|
|
|
const GROUP = 'alt.binaries.test'
|
|
const rules = { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { 'alt.binaries.boneless': 1, [GROUP]: 2, 'alt.binaries.x': 3 } }
|
|
let id = 0
|
|
|
|
/** Overview lines of one posted file: all its parts, from article number `at` on. */
|
|
function post(at: number, name: string, parts: number, opts: { counter?: string; groups?: string[]; tool?: 'ngPost' | 'nyuu'; poster?: string } = {}): Overview[] {
|
|
return Array.from({ length: parts }, (_, i) => {
|
|
id++
|
|
const mid = opts.tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
|
|
const groups = opts.groups ?? [GROUP]
|
|
return [at + i, {
|
|
'subject': `${opts.counter ? `${opts.counter} - ` : ''}"${name}" yEnc (${i + 1}/${parts})`,
|
|
'from': opts.poster ?? 'poster <p@x>',
|
|
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
|
'message-id': `<${mid}>`,
|
|
':bytes': '700000',
|
|
'xref': `news.example ${groups.map((g, k) => `${g}:${at + i + k}`).join(' ')}`,
|
|
}] as Overview
|
|
})
|
|
}
|
|
|
|
test.group('edge / RangeIndexer', () => {
|
|
test('groups files into releases with probe candidates, and counts what the rules dropped', ({ assert }) => {
|
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
|
|
indexer.add([
|
|
...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 3, { counter: 'Some.Show.S01E01 [1/3]' }),
|
|
...post(100_010, 'Some.Show.S01E01.1080p.WEB.part2.rar', 3, { counter: 'Some.Show.S01E01 [2/3]' }),
|
|
...post(100_020, 'Some.Show.S01E01.1080p.WEB.par2', 1, { counter: 'Some.Show.S01E01 [3/3]' }),
|
|
// A crosspost whose owner is boneless (priority 1): skipped here.
|
|
...post(100_030, 'Other.Release.2020.mkv', 2, { groups: [GROUP, 'alt.binaries.boneless'] }),
|
|
// Random-named ngPost upload: garbage everywhere.
|
|
...post(100_040, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 4, { tool: 'ngPost' }),
|
|
// A random subject per article: per-article.
|
|
...post(100_050, 'Pq4ZmXc8VbN2LkJh6GfDs3', 1),
|
|
], 200_000)
|
|
indexer.add([[100_060, { 'subject': 'obfuscated', 'from': 'x', 'message-id': '<obf@test>', 'date': '', ':bytes': '1', 'xref': `news.example ${GROUP}:100060` }]], 300_000)
|
|
indexer.finish()
|
|
const batch = indexer.take()
|
|
|
|
assert.equal(batch.headers, 3 + 3 + 1 + 2 + 4 + 1 + 1)
|
|
assert.lengthOf(batch.releases, 1)
|
|
const [release] = batch.releases
|
|
assert.include(release, { files: 3, segments: 7, firstArticle: 100_000, lastArticle: 100_020 })
|
|
assert.deepEqual(decodeChunk(release.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar', 'Some.Show.S01E01.1080p.WEB.part2.rar'])
|
|
assert.deepEqual(batch.candidates.map((c) => [c.kind, c.file]).sort(), [['archive', 'Some.Show.S01E01.1080p.WEB.part1.rar'], ['par2', 'Some.Show.S01E01.1080p.WEB.par2']])
|
|
|
|
const counts = Object.fromEntries(batch.counts.map((c) => [`${c.rule} ${c.key}`.trim(), [c.articles, c.files]]))
|
|
assert.deepEqual(counts, { 'crosspost alt.binaries.boneless': [2, 0], 'garbage ngPost alt.binaries.test': [4, 1], 'per-article other': [1, 1], 'obfuscated': [1, 0] })
|
|
assert.equal(batch.next, 1_000_001)
|
|
})
|
|
|
|
test('a file cut off near an edge leaves as fragments, one in the middle is only counted', ({ assert }) => {
|
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
|
|
indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000)
|
|
indexer.finish()
|
|
const batch = indexer.take()
|
|
assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => [s[0], s[3]])]), [['Edge.File.2020.part1.rar', [[1, 5], [2, 6]]]])
|
|
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]])
|
|
assert.lengthOf(batch.releases, 0)
|
|
})
|
|
})
|
|
|
|
test.group('edge / taking over a range', () => {
|
|
test("a worker that resumes a range midway starts its first batch at the range's resume point", ({ assert }) => {
|
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, resumeAt: 500_001, rules })
|
|
indexer.add(post(500_001, 'A.Release.2020.mkv', 1), 600_000)
|
|
assert.deepEqual([indexer.take().from, indexer.take().from], [500_001, 600_000])
|
|
})
|
|
})
|