Rebuild: the batch store (catalog upserts, write-once chunks, candidates, rule counts, fragments, resume point)
A batch is stored in one transaction and only at its range's resume point, so a retried batch can't count twice. Garbage is judged before per-article: a dropped tool isn't worth reading bodies for either. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0714b2c81a
commit
20910647fa
6 files changed
+242
-7
No files matched your search
@@ -53,17 +53,18 @@ export class RuleSet {
|
||||
|
||||
/**
|
||||
* A release's verdict from its files: kept as a whole when any file has a
|
||||
* readable name; else dropped as per-article random subjects when every
|
||||
* file has few parts, as garbage when its signature or tool is dropped.
|
||||
* readable name; else dropped as garbage when its signature or tool is
|
||||
* dropped, as per-article random subjects when every file has few parts.
|
||||
*/
|
||||
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
|
||||
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
|
||||
const first = files[0]
|
||||
const tool = uploadTool(first.firstMessageId)
|
||||
const sample = explored(releaseKey, this.rules.samplePercent)
|
||||
if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
|
||||
// Garbage first: a dropped tool or signature isn't worth reading bodies for either.
|
||||
const group = first.groups[0] ?? ''
|
||||
if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }
|
||||
if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
|
||||
return { rule: null }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user