The namer re-reads partial releases as the Rust window writes them
The pending namer's TypeScript re-index gave a file short of parts on the server to the range path's fragment rule, so a partial release the edge wrote (Bob's hex-named uploads in chello, ratcave, font, storage) never came out again: 10 of 148k named releases moved. RangeIndexer's partialFiles keeps such a file in its release, marked partial; tools/namer_check.ts replays sample rows from the corpus (10 of 10 found with it, 1 without). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0ccb850ea4
commit
8cf965af80
5 files changed
+102
-1
No files matched your search
+4
-1
@@ -880,7 +880,10 @@ f. **Nothing enters the catalog without a usable name** (amended
|
||||
(`app/services/pending_namer.ts`, in the prober's process): a named
|
||||
release's article span read again from the corpus (the stored range's
|
||||
chunks) or by XOVER on a connection of its own, through the TypeScript
|
||||
indexer, written as a release under the pending key with `source`
|
||||
indexer with `partialFiles` (a file given up short of parts stays a
|
||||
partial file of its release, as the Rust window writes it; without it
|
||||
the range path's fragment rule left nothing to find, and 10 of 148k named
|
||||
releases were moved by 2026-10-29), written as a release under the pending key with `source`
|
||||
weak-stem, the name as `inner_name`, its chunk and its candidates (the
|
||||
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
|
||||
the password flag). Not built: the spot/NZB match on pending keys (step
|
||||
|
||||
@@ -112,6 +112,8 @@ export class PendingNamer {
|
||||
rules: { ...rules, samplePercent: 0 },
|
||||
idleChunks: 0,
|
||||
edgeArticles: 0,
|
||||
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
|
||||
partialFiles: true,
|
||||
})
|
||||
ix.addText(text, row.last_article + 1)
|
||||
ix.finish()
|
||||
|
||||
@@ -123,6 +123,11 @@ export interface RangeIndexerOptions {
|
||||
* chunks seen so far), as uploads in busy groups spread over minutes.
|
||||
*/
|
||||
edgeArticles?: number
|
||||
/**
|
||||
* A file given up short of parts is a file of its release all the same (partial), as the Rust edge's window writes
|
||||
* it, not a fragment: the pending namer reads such a release again (LOG.md 2026-10-29 (121)).
|
||||
*/
|
||||
partialFiles?: boolean
|
||||
}
|
||||
|
||||
const MAX_CANDIDATES = 4
|
||||
@@ -274,6 +279,18 @@ export class RangeIndexer {
|
||||
else byFile.set(s.fileKey, [s])
|
||||
}
|
||||
for (const list of byFile.values()) {
|
||||
if (this.options.partialFiles) {
|
||||
// As the Rust window's files(): one article of a multi-part random file is no release.
|
||||
if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) {
|
||||
this.count('per-article', uploadTool(list[0].messageId), 1, 0)
|
||||
continue
|
||||
}
|
||||
// One segment per part (two copies of a file share its key here), the first article seen.
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const s of list.sort((a, b) => a.articleNumber - b.articleNumber)) if (!parts.has(s.number)) parts.set(s.number, s)
|
||||
this.files([{ fileKey: list[0].fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }])
|
||||
continue
|
||||
}
|
||||
// One article of a "file" of many parts under a random name: a random subject per article, which
|
||||
// never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09).
|
||||
if (list.length === 1 && isRandomFile(list[0].file.name)) {
|
||||
@@ -430,6 +447,7 @@ export function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[], uploa
|
||||
lastArticle,
|
||||
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
|
||||
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
||||
partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
|
||||
payload: encodeChunk(chunkFiles),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,6 +87,24 @@ test.group('edge / RangeIndexer', () => {
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
|
||||
assert.lengthOf(batch.releases, 0)
|
||||
})
|
||||
|
||||
test("partialFiles: a file short of parts is a partial release, as the Rust window writes it; one article of a random multi-part file isn't", ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 200, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
|
||||
const name = '8c21f5c36450f9e2ccc6ff387d159d4d'
|
||||
indexer.add([
|
||||
// Parts 1, 3 and 5 of 6 (the others gone from the server), from a re-send's copy of part 3 too.
|
||||
...post(100, name, 6, { poster: 'Bob <bob@home.me>' }).filter((_, i) => i % 2 === 0),
|
||||
...post(150, name, 6, { poster: 'Bob <bob@home.me>' }).slice(2, 3),
|
||||
...post(160, 'Pq4ZmXc8VbN2LkJh6GfDs3', 4).slice(1, 2),
|
||||
], 201)
|
||||
indexer.finish()
|
||||
const batch = indexer.take()
|
||||
assert.lengthOf(batch.releases, 1)
|
||||
assert.include(batch.releases[0], { name, files: 1, segments: 3, partial: true, firstArticle: 100, lastArticle: 104 })
|
||||
assert.deepEqual(decodeChunk(batch.releases[0].payload)[0].segments.map((s) => s[0]), [1, 3, 5])
|
||||
assert.deepEqual(batch.fragments, [])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
|
||||
})
|
||||
})
|
||||
|
||||
test.group('edge / taking over a range', () => {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Why the pending namer's move fails: for sample pending releases (a JSON dump of rows, the rule set and the
|
||||
* priorities), the stored corpus chunks of their span through the TypeScript indexer, once with the rule set as
|
||||
* stored and once without its drop rules; prints whether the pending identity comes out and what does.
|
||||
*
|
||||
* node --import=@poppinss/ts-exec tools/namer_check.ts sample.json /corpus
|
||||
*/
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { identityKey } from '#lib/release/keys'
|
||||
|
||||
const [file, corpus] = process.argv.slice(2)
|
||||
const dump = JSON.parse(readFileSync(file, 'utf8'))
|
||||
const { version: _v, ...defaults } = DEFAULT_RULES
|
||||
const stored: Rules = { ...defaults, ...dump.rules, version: dump.version, priorities: dump.priorities, samplePercent: 0 }
|
||||
const lenient: Rules = { ...stored, droppedTools: [], droppedSignatures: [], droppedPosters: [], dropExecutables: false }
|
||||
|
||||
function text(row: any): string | null {
|
||||
const parts: string[] = []
|
||||
for (const r of row.ranges) {
|
||||
const dir = join(corpus, row.grp, `${r.start}-${r.end}`)
|
||||
let names: string[]
|
||||
try {
|
||||
names = readdirSync(dir).filter((n) => n.endsWith('.gz')).sort()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
for (const n of names) {
|
||||
const [, from, to] = n.slice(0, -3).split('-').map(Number)
|
||||
if (to >= row.first_article && from <= row.last_article) parts.push(unzipSync(readFileSync(join(dir, n))).toString('latin1'))
|
||||
}
|
||||
}
|
||||
return parts.length ? parts.join('') : null
|
||||
}
|
||||
|
||||
function run(row: any, overview: string, rules: Rules) {
|
||||
const ix = new RangeIndexer({ group: row.grp, start: row.first_article, end: row.last_article, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
|
||||
ix.addText(overview, row.last_article + 1)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.some((r) => identityKey(r.key).toString('hex') === row.identity)
|
||||
const near = batch.releases.filter((r) => r.firstArticle !== undefined && Math.abs(Number(r.firstArticle) - row.first_article) < 50)
|
||||
return { found, releases: batch.releases.length, near: near.map((r) => `${identityKey(r.key).toString('hex').slice(0, 12)} ${r.name} @${r.firstArticle}`), counts: batch.counts }
|
||||
}
|
||||
|
||||
for (const row of dump.rows) {
|
||||
const overview = text(row)
|
||||
console.log(`\n${row.grp} tries ${row.tries} ${row.first_article}–${row.last_article} "${row.name}" by ${row.poster}, ${row.segments} segments; identity ${row.identity.slice(0, 12)}`)
|
||||
if (!overview) {
|
||||
console.log(' no corpus text')
|
||||
continue
|
||||
}
|
||||
for (const [label, rules] of [['stored rules', stored], ['no drop rules', lenient]] as const) {
|
||||
const r = run(row, overview, rules)
|
||||
console.log(` ${label}: found ${r.found}; ${r.releases} releases; near: ${r.near.join(' | ') || '-'}; counts ${JSON.stringify(r.counts)}`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user