From 8cf965af80781f4f5ddbccdf07514184609eb40a Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Thu, 29 Oct 2026 10:41:50 +0100 Subject: [PATCH] The namer re-reads partial releases as the Rust window writes them The pending namer's TypeScript re-index gave a file short of parts on the server to the range path's fragment rule, so a partial release the edge wrote (Bob's hex-named uploads in chello, ratcave, font, storage) never came out again: 10 of 148k named releases moved. RangeIndexer's partialFiles keeps such a file in its release, marked partial; tools/namer_check.ts replays sample rows from the corpus (10 of 10 found with it, 1 without). Co-Authored-By: Claude Opus 5.5 --- REBUILD.md | 5 ++- app/services/pending_namer.ts | 2 + lib/edge/range_indexer.ts | 18 ++++++++ tests/unit/edge/range_indexer.spec.ts | 18 ++++++++ tools/namer_check.ts | 60 +++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tools/namer_check.ts diff --git a/REBUILD.md b/REBUILD.md index 8603279..972a6ca 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -880,7 +880,10 @@ f. **Nothing enters the catalog without a usable name** (amended (`app/services/pending_namer.ts`, in the prober's process): a named release's article span read again from the corpus (the stored range's chunks) or by XOVER on a connection of its own, through the TypeScript - indexer, written as a release under the pending key with `source` + indexer with `partialFiles` (a file given up short of parts stays a + partial file of its release, as the Rust window writes it; without it + the range path's fragment rule left nothing to find, and 10 of 148k named + releases were moved by 2026-10-29), written as a release under the pending key with `source` weak-stem, the name as `inner_name`, its chunk and its candidates (the prober answers those from `probe_cache`: inspections, par2 sets, hashes, the password flag). Not built: the spot/NZB match on pending keys (step diff --git a/app/services/pending_namer.ts b/app/services/pending_namer.ts index 7f8ea68..6c8c126 100644 --- a/app/services/pending_namer.ts +++ b/app/services/pending_namer.ts @@ -112,6 +112,8 @@ export class PendingNamer { rules: { ...rules, samplePercent: 0 }, idleChunks: 0, edgeArticles: 0, + // The edge wrote it partial when its file is short of parts on the server: so must the re-read. + partialFiles: true, }) ix.addText(text, row.last_article + 1) ix.finish() diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index c8ee31c..6dca98b 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -123,6 +123,11 @@ export interface RangeIndexerOptions { * chunks seen so far), as uploads in busy groups spread over minutes. */ edgeArticles?: number + /** + * A file given up short of parts is a file of its release all the same (partial), as the Rust edge's window writes + * it, not a fragment: the pending namer reads such a release again (LOG.md 2026-10-29 (121)). + */ + partialFiles?: boolean } const MAX_CANDIDATES = 4 @@ -274,6 +279,18 @@ export class RangeIndexer { else byFile.set(s.fileKey, [s]) } for (const list of byFile.values()) { + if (this.options.partialFiles) { + // As the Rust window's files(): one article of a multi-part random file is no release. + if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) { + this.count('per-article', uploadTool(list[0].messageId), 1, 0) + continue + } + // One segment per part (two copies of a file share its key here), the first article seen. + const parts = new Map() + for (const s of list.sort((a, b) => a.articleNumber - b.articleNumber)) if (!parts.has(s.number)) parts.set(s.number, s) + this.files([{ fileKey: list[0].fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }]) + continue + } // One article of a "file" of many parts under a random name: a random subject per article, which // never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09). if (list.length === 1 && isRandomFile(list[0].file.name)) { @@ -430,6 +447,7 @@ export function chunkOf(key: string, files: AggregatedFile[], uploa lastArticle, readable: chunkFiles.some((f) => !isRandomFile(f.name)), hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)), + partial: files.some((f) => f.segments.length < f.segments[0].totalParts), payload: encodeChunk(chunkFiles), } } diff --git a/tests/unit/edge/range_indexer.spec.ts b/tests/unit/edge/range_indexer.spec.ts index 655779e..0127bb0 100644 --- a/tests/unit/edge/range_indexer.spec.ts +++ b/tests/unit/edge/range_indexer.spec.ts @@ -87,6 +87,24 @@ test.group('edge / RangeIndexer', () => { assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]]) assert.lengthOf(batch.releases, 0) }) + + test("partialFiles: a file short of parts is a partial release, as the Rust window writes it; one article of a random multi-part file isn't", ({ assert }) => { + const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 200, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true }) + const name = '8c21f5c36450f9e2ccc6ff387d159d4d' + indexer.add([ + // Parts 1, 3 and 5 of 6 (the others gone from the server), from a re-send's copy of part 3 too. + ...post(100, name, 6, { poster: 'Bob ' }).filter((_, i) => i % 2 === 0), + ...post(150, name, 6, { poster: 'Bob ' }).slice(2, 3), + ...post(160, 'Pq4ZmXc8VbN2LkJh6GfDs3', 4).slice(1, 2), + ], 201) + indexer.finish() + const batch = indexer.take() + assert.lengthOf(batch.releases, 1) + assert.include(batch.releases[0], { name, files: 1, segments: 3, partial: true, firstArticle: 100, lastArticle: 104 }) + assert.deepEqual(decodeChunk(batch.releases[0].payload)[0].segments.map((s) => s[0]), [1, 3, 5]) + assert.deepEqual(batch.fragments, []) + assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]]) + }) }) test.group('edge / taking over a range', () => { diff --git a/tools/namer_check.ts b/tools/namer_check.ts new file mode 100644 index 0000000..b8bf134 --- /dev/null +++ b/tools/namer_check.ts @@ -0,0 +1,60 @@ +/** + * Why the pending namer's move fails: for sample pending releases (a JSON dump of rows, the rule set and the + * priorities), the stored corpus chunks of their span through the TypeScript indexer, once with the rule set as + * stored and once without its drop rules; prints whether the pending identity comes out and what does. + * + * node --import=@poppinss/ts-exec tools/namer_check.ts sample.json /corpus + */ +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { unzipSync } from 'node:zlib' +import { RangeIndexer } from '#lib/edge/index' +import { DEFAULT_RULES, type Rules } from '#lib/rules/index' +import { identityKey } from '#lib/release/keys' + +const [file, corpus] = process.argv.slice(2) +const dump = JSON.parse(readFileSync(file, 'utf8')) +const { version: _v, ...defaults } = DEFAULT_RULES +const stored: Rules = { ...defaults, ...dump.rules, version: dump.version, priorities: dump.priorities, samplePercent: 0 } +const lenient: Rules = { ...stored, droppedTools: [], droppedSignatures: [], droppedPosters: [], dropExecutables: false } + +function text(row: any): string | null { + const parts: string[] = [] + for (const r of row.ranges) { + const dir = join(corpus, row.grp, `${r.start}-${r.end}`) + let names: string[] + try { + names = readdirSync(dir).filter((n) => n.endsWith('.gz')).sort() + } catch { + return null + } + for (const n of names) { + const [, from, to] = n.slice(0, -3).split('-').map(Number) + if (to >= row.first_article && from <= row.last_article) parts.push(unzipSync(readFileSync(join(dir, n))).toString('latin1')) + } + } + return parts.length ? parts.join('') : null +} + +function run(row: any, overview: string, rules: Rules) { + const ix = new RangeIndexer({ group: row.grp, start: row.first_article, end: row.last_article, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true }) + ix.addText(overview, row.last_article + 1) + ix.finish() + const batch = ix.take() + const found = batch.releases.some((r) => identityKey(r.key).toString('hex') === row.identity) + const near = batch.releases.filter((r) => r.firstArticle !== undefined && Math.abs(Number(r.firstArticle) - row.first_article) < 50) + return { found, releases: batch.releases.length, near: near.map((r) => `${identityKey(r.key).toString('hex').slice(0, 12)} ${r.name} @${r.firstArticle}`), counts: batch.counts } +} + +for (const row of dump.rows) { + const overview = text(row) + console.log(`\n${row.grp} tries ${row.tries} ${row.first_article}–${row.last_article} "${row.name}" by ${row.poster}, ${row.segments} segments; identity ${row.identity.slice(0, 12)}`) + if (!overview) { + console.log(' no corpus text') + continue + } + for (const [label, rules] of [['stored rules', stored], ['no drop rules', lenient]] as const) { + const r = run(row, overview, rules) + console.log(` ${label}: found ${r.found}; ${r.releases} releases; near: ${r.near.join(' | ') || '-'}; counts ${JSON.stringify(r.counts)}`) + } +}