Single-archive rules: stub archives and nonsense pieces dropped at the edge
The judge gets each file's articles seen and bytes, and whether the window closed on the release. A settled release of one unsplit archive posted to 2+ groups is dropped as a stub archive when its articles are <= 256 KiB with the release >= 1 MB, or it is in 4+ groups, or by a webmail poster (those two under 100 MB); releases touching music, e-book, comics, tabs or fonts groups are exempt. A one-article archive of 100 kB-1 MB under a release-like, unscrambled name without spaces or an e-book format is dropped as a nonsense piece. Both switched by the rule set (dropStubArchives, dropNonsensePieces), mirrored in lib/rules, sampled like every drop (LOG.md (123)-(127)). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
2f1bbf2538
commit
e470e231d0
8 files changed
+397
-15
No files matched your search
@@ -62,7 +62,7 @@ export interface ProbeCandidate {
|
||||
|
||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||
export interface RuleCount {
|
||||
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
||||
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'incomplete'
|
||||
key: string
|
||||
articles: number
|
||||
files: number
|
||||
@@ -246,9 +246,20 @@ export class RangeIndexer {
|
||||
this.completed = new Map()
|
||||
for (const [key, all] of byRelease) {
|
||||
uploadsOf(all).forEach((list, upload) => {
|
||||
// Settled as far as this indexer knows: it judges a release's files as each batch completes them (the Rust
|
||||
// window passes whether the window closed on the release and no block edge cut it).
|
||||
const verdict = this.rules.judge(
|
||||
key,
|
||||
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster }))
|
||||
list.map((f) => ({
|
||||
name: f.segments[0].file.name,
|
||||
totalParts: f.segments[0].totalParts,
|
||||
groups: f.segments[0].file.groups,
|
||||
firstMessageId: f.segments[0].messageId,
|
||||
poster: f.segments[0].file.poster,
|
||||
segments: f.segments.length,
|
||||
bytes: f.segments.reduce((n, s) => n + s.bytes, 0),
|
||||
})),
|
||||
true
|
||||
)
|
||||
if (verdict.rule === null) {
|
||||
this.batch.releases.push(chunkOf(key, list, upload))
|
||||
|
||||
Reference in new issue
Block a user