Daan MeijerandClaude Opus 5.5
8cf965af80
The pending namer's TypeScript re-index gave a file short of parts on the server to the range path's fragment rule, so a partial release the edge wrote (Bob's hex-named uploads in chello, ratcave, font, storage) never came out again: 10 of 148k named releases moved. RangeIndexer's partialFiles keeps such a file in its release, marked partial; tools/namer_check.ts replays sample rows from the corpus (10 of 10 found with it, 1 without). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.9 KiB
TypeScript
61 lines
2.9 KiB
TypeScript
/**
|
||
* Why the pending namer's move fails: for sample pending releases (a JSON dump of rows, the rule set and the
|
||
* priorities), the stored corpus chunks of their span through the TypeScript indexer, once with the rule set as
|
||
* stored and once without its drop rules; prints whether the pending identity comes out and what does.
|
||
*
|
||
* node --import=@poppinss/ts-exec tools/namer_check.ts sample.json /corpus
|
||
*/
|
||
import { readdirSync, readFileSync } from 'node:fs'
|
||
import { join } from 'node:path'
|
||
import { unzipSync } from 'node:zlib'
|
||
import { RangeIndexer } from '#lib/edge/index'
|
||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||
import { identityKey } from '#lib/release/keys'
|
||
|
||
const [file, corpus] = process.argv.slice(2)
|
||
const dump = JSON.parse(readFileSync(file, 'utf8'))
|
||
const { version: _v, ...defaults } = DEFAULT_RULES
|
||
const stored: Rules = { ...defaults, ...dump.rules, version: dump.version, priorities: dump.priorities, samplePercent: 0 }
|
||
const lenient: Rules = { ...stored, droppedTools: [], droppedSignatures: [], droppedPosters: [], dropExecutables: false }
|
||
|
||
function text(row: any): string | null {
|
||
const parts: string[] = []
|
||
for (const r of row.ranges) {
|
||
const dir = join(corpus, row.grp, `${r.start}-${r.end}`)
|
||
let names: string[]
|
||
try {
|
||
names = readdirSync(dir).filter((n) => n.endsWith('.gz')).sort()
|
||
} catch {
|
||
return null
|
||
}
|
||
for (const n of names) {
|
||
const [, from, to] = n.slice(0, -3).split('-').map(Number)
|
||
if (to >= row.first_article && from <= row.last_article) parts.push(unzipSync(readFileSync(join(dir, n))).toString('latin1'))
|
||
}
|
||
}
|
||
return parts.length ? parts.join('') : null
|
||
}
|
||
|
||
function run(row: any, overview: string, rules: Rules) {
|
||
const ix = new RangeIndexer({ group: row.grp, start: row.first_article, end: row.last_article, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
|
||
ix.addText(overview, row.last_article + 1)
|
||
ix.finish()
|
||
const batch = ix.take()
|
||
const found = batch.releases.some((r) => identityKey(r.key).toString('hex') === row.identity)
|
||
const near = batch.releases.filter((r) => r.firstArticle !== undefined && Math.abs(Number(r.firstArticle) - row.first_article) < 50)
|
||
return { found, releases: batch.releases.length, near: near.map((r) => `${identityKey(r.key).toString('hex').slice(0, 12)} ${r.name} @${r.firstArticle}`), counts: batch.counts }
|
||
}
|
||
|
||
for (const row of dump.rows) {
|
||
const overview = text(row)
|
||
console.log(`\n${row.grp} tries ${row.tries} ${row.first_article}–${row.last_article} "${row.name}" by ${row.poster}, ${row.segments} segments; identity ${row.identity.slice(0, 12)}`)
|
||
if (!overview) {
|
||
console.log(' no corpus text')
|
||
continue
|
||
}
|
||
for (const [label, rules] of [['stored rules', stored], ['no drop rules', lenient]] as const) {
|
||
const r = run(row, overview, rules)
|
||
console.log(` ${label}: found ${r.found}; ${r.releases} releases; near: ${r.near.join(' | ') || '-'}; counts ${JSON.stringify(r.counts)}`)
|
||
}
|
||
}
|