The namer re-reads partial releases as the Rust window writes them
Test and deploy / test (push) Successful in 3m10s
Test and deploy / deploy (push) Successful in 5m38s

The pending namer's TypeScript re-index gave a file short of parts on the
server to the range path's fragment rule, so a partial release the edge
wrote (Bob's hex-named uploads in chello, ratcave, font, storage) never came
out again: 10 of 148k named releases moved. RangeIndexer's partialFiles
keeps such a file in its release, marked partial; tools/namer_check.ts
replays sample rows from the corpus (10 of 10 found with it, 1 without).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-29 10:41:50 +01:00
1 parent 0ccb850ea4
commit 8cf965af80
5 files changed
+102 -1

No files matched your search

+60
View File
@@ -0,0 +1,60 @@
/**
* Why the pending namer's move fails: for sample pending releases (a JSON dump of rows, the rule set and the
* priorities), the stored corpus chunks of their span through the TypeScript indexer, once with the rule set as
* stored and once without its drop rules; prints whether the pending identity comes out and what does.
*
* node --import=@poppinss/ts-exec tools/namer_check.ts sample.json /corpus
*/
import { readdirSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import { RangeIndexer } from '#lib/edge/index'
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
import { identityKey } from '#lib/release/keys'
const [file, corpus] = process.argv.slice(2)
const dump = JSON.parse(readFileSync(file, 'utf8'))
const { version: _v, ...defaults } = DEFAULT_RULES
const stored: Rules = { ...defaults, ...dump.rules, version: dump.version, priorities: dump.priorities, samplePercent: 0 }
const lenient: Rules = { ...stored, droppedTools: [], droppedSignatures: [], droppedPosters: [], dropExecutables: false }
function text(row: any): string | null {
const parts: string[] = []
for (const r of row.ranges) {
const dir = join(corpus, row.grp, `${r.start}-${r.end}`)
let names: string[]
try {
names = readdirSync(dir).filter((n) => n.endsWith('.gz')).sort()
} catch {
return null
}
for (const n of names) {
const [, from, to] = n.slice(0, -3).split('-').map(Number)
if (to >= row.first_article && from <= row.last_article) parts.push(unzipSync(readFileSync(join(dir, n))).toString('latin1'))
}
}
return parts.length ? parts.join('') : null
}
function run(row: any, overview: string, rules: Rules) {
const ix = new RangeIndexer({ group: row.grp, start: row.first_article, end: row.last_article, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
ix.addText(overview, row.last_article + 1)
ix.finish()
const batch = ix.take()
const found = batch.releases.some((r) => identityKey(r.key).toString('hex') === row.identity)
const near = batch.releases.filter((r) => r.firstArticle !== undefined && Math.abs(Number(r.firstArticle) - row.first_article) < 50)
return { found, releases: batch.releases.length, near: near.map((r) => `${identityKey(r.key).toString('hex').slice(0, 12)} ${r.name} @${r.firstArticle}`), counts: batch.counts }
}
for (const row of dump.rows) {
const overview = text(row)
console.log(`\n${row.grp} tries ${row.tries} ${row.first_article}–${row.last_article} "${row.name}" by ${row.poster}, ${row.segments} segments; identity ${row.identity.slice(0, 12)}`)
if (!overview) {
console.log(' no corpus text')
continue
}
for (const [label, rules] of [['stored rules', stored], ['no drop rules', lenient]] as const) {
const r = run(row, overview, rules)
console.log(` ${label}: found ${r.found}; ${r.releases} releases; near: ${r.near.join(' | ') || '-'}; counts ${JSON.stringify(r.counts)}`)
}
}