Rebuild: first end-to-end runs; prefetch per chunk, executable rule, wire fixes
End to end on the workstation (coordinator:run, the API, edge:node, real NNTP): a.b.e-book.french and a.b.cd.image.playstation2.dvdiso ranges. - Prefetcher: every connection takes the next missing chunk (leased ranges first, in order); one connection per range (~26k lines/s) kept the worker waiting - executable rule: releases of bare executables (malware floods named after films and software) are dropped; they were 92% of the catalog rows of the test ranges, 21% of the segments - fragments per file (details once) and gzipped batch bodies; the JSON limit (1 MB by default) raised to 256 MB, empty strings kept (a rule count's key is '') - release_chunks.batch_from is a bigint (boneless article numbers) - tests get their own database; edge:bench --store times BatchStore: 1.3 s for a boneless range (1.24 ms per release chunk) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d5f168d8d2
commit
cd06f96f85
15 files changed
+201
-99
No files matched your search
@@ -19,6 +19,8 @@ function chunk(at: number): Buffer {
|
||||
const lines = [1, 2].map(
|
||||
(n) => `${at + n - 1}\tSome.Show.S01E0${at % 7} [1/1] - "Some.Show.S01E0${at % 7}.1080p.mkv" yEnc (${n}/2)\tp <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<a${at}-${n}@test>\t\t700000\t5000\tXref: news.example ${GROUP}:${at + n - 1}`
|
||||
)
|
||||
// An obfuscated subject: counted under the key '' (which must survive the body parser).
|
||||
lines.push(`${at + 2}\tobfuscated\tp <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<o${at}@test>\t\t700000\t5000\tXref: news.example ${GROUP}:${at + 2}`)
|
||||
return deflateSync(Buffer.from(lines.join('\r\n') + '\r\n'))
|
||||
}
|
||||
|
||||
@@ -75,6 +77,8 @@ test.group('Node API', (group) => {
|
||||
// The same batch again: refused.
|
||||
;(await client.post(`/api/v1/node/ranges/${rangeId}/batches`).bearerToken(token).json(batch)).assertStatus(409)
|
||||
|
||||
assert.deepEqual((await db.from('rule_counts').where('range_id', rangeId)).map((c) => [c.rule, c.key, Number(c.articles)]), [['obfuscated', '', 3]])
|
||||
|
||||
;(await client.post(`/api/v1/node/ranges/${rangeId}/finish`).bearerToken(token)).assertStatus(200)
|
||||
const range = await db.from('ranges').where('id', rangeId).first()
|
||||
assert.equal(range.status, 'done')
|
||||
|
||||
@@ -60,7 +60,7 @@ test.group('edge / RangeIndexer', () => {
|
||||
indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000)
|
||||
indexer.finish()
|
||||
const batch = indexer.take()
|
||||
assert.deepEqual(batch.fragments.map((f) => [f.name, f.number]), [['Edge.File.2020.part1.rar', 1], ['Edge.File.2020.part1.rar', 2]])
|
||||
assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => [s[0], s[3]])]), [['Edge.File.2020.part1.rar', [[1, 5], [2, 6]]]])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]])
|
||||
assert.lengthOf(batch.releases, 0)
|
||||
})
|
||||
|
||||
@@ -21,6 +21,12 @@ test.group('rules / verdicts', () => {
|
||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
||||
})
|
||||
|
||||
test('bare executables are dropped, sidecars aside; an executable among other content is kept', ({ assert }) => {
|
||||
assert.include(rules.judge('k', [file('How to Make Love to a Woman 2010.scr', 10, NYUU), file('How to Make Love to a Woman 2010.par2', 1, NYUU)]), { rule: 'executable', key: 'scr' })
|
||||
assert.include(rules.judge('k', [file('Adobe Illustrator CS2 + ssg keygen.zip .exe', 3, NYUU)]), { rule: 'executable', key: 'exe' })
|
||||
assert.deepEqual(rules.judge('k', [file('Some.App.v1.0.part1.rar', 50, NYUU), file('setup.exe', 2, NYUU)]), { rule: null })
|
||||
})
|
||||
|
||||
test('random names: per-article with few parts, garbage by tool or signature, else kept', ({ assert }) => {
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa', 2, NYUU)]), { rule: 'per-article', key: 'nyuu' })
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST)]), { rule: 'garbage', key: 'ngPost alt.binaries.ath' })
|
||||
|
||||
Reference in new issue
Block a user