Pending ladder by class, twins by content, dropped classes at the edge
The prober names 6% of the unnamed releases it finishes and the pending queue (44M) grows eleven times faster than it probes. A sample of 44,530 finished pending rows shows the headers predict a name: size bucket, name shape, completeness, tool and group separate 0% classes from 95% ones, while the learner keyed on tool and group alone (LOG.md (130)). - lib/release/classes.ts, names::pending_class, SQL pending_class(): a release's class (size × shape × partial, 1..70), the three in step. - Migration 6800: pending_releases.klass and target_bytes (the first target's article; candidates carry bytes now), pending_classes with each class's climbs, names, waiting and verdict, the claim's index. - The ladder claims the classes with the best record first (a prior for the unexplored, at most a quarter of a claim from one), the cheapest target first within one, then the oldest of the rest; records classes like signatures and drops them at the same thresholds; discards their waiting releases. Claimed rows sorted in code: an UPDATE returns none. - The rule set's droppedClasses: the edge keeps unnamed releases of a dropped class out of the pending store (counted as garbage), the explored sample excepted. - hash_twins.ts: a probed file whose first 16 KiB and size match a named catalog release's names a pending release the probe didn't, and sets the inner name of a catalog release without one. - pending:classes classes the rows from before and seeds the record from the finished ones; rules:classes publishes the dropped classes; releases:readable also marks the rows' candidates unreadable. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
81a5dce45a
commit
44c0c73b05
20 files changed
+927
-55
No files matched your search
@@ -2,7 +2,8 @@ import { hash } from 'node:crypto'
|
||||
import type { Overview } from '../nntp/index.js'
|
||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
|
||||
import { isUnreadableName } from '../release/index.js'
|
||||
import { classKey, isUnreadableName, pendingClass } from '../release/index.js'
|
||||
import { explored } from '../rules/upload_tools.js'
|
||||
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
|
||||
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
||||
|
||||
@@ -59,6 +60,8 @@ export interface ProbeCandidate {
|
||||
kind: CandidateKind
|
||||
messageId: string
|
||||
file: string
|
||||
/** The target article's bytes: the cheapest probe first within a class (LOG.md (130)). */
|
||||
bytes?: number
|
||||
}
|
||||
|
||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||
@@ -263,7 +266,18 @@ export class RangeIndexer {
|
||||
true
|
||||
)
|
||||
if (verdict.rule === null) {
|
||||
this.batch.releases.push(chunkOf(key, list, upload))
|
||||
const chunk = chunkOf(key, list, upload)
|
||||
if (!chunk.readable) {
|
||||
// Unnamed: a class the ladder found never names anything stays out of the pending store, its explored
|
||||
// sample excepted (so the rate keeps being measured); counted as garbage (LOG.md (130)).
|
||||
const tool = uploadTool(list[0].segments[0].messageId)
|
||||
const klass = pendingClass(chunk.name, chunk.bytes, chunk.partial === true)
|
||||
if (this.rules.classes.has(classKey(tool, this.options.group, klass)) && !explored(key, this.options.rules.samplePercent)) {
|
||||
this.count('garbage', `${tool} ${this.options.group} #${klass}`, list.reduce((n, f) => n + f.segments.length, 0), list.length)
|
||||
return
|
||||
}
|
||||
}
|
||||
this.batch.releases.push(chunk)
|
||||
this.batch.candidates.push(...candidatesOf(key, list, upload))
|
||||
return
|
||||
}
|
||||
@@ -483,12 +497,12 @@ export function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[],
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if (volume === 0 && f.segments.length > 1 && !tail) tail = { volume, file: f }
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name })
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name, bytes: f.segments[0].bytes })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name, bytes: f.segments[0].bytes })
|
||||
}
|
||||
if (tail) out.push({ releaseKey: key, upload, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name })
|
||||
if (tail) out.push({ releaseKey: key, upload, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name, bytes: tail.file.segments.at(-1)!.bytes })
|
||||
if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].file.name)) {
|
||||
out.push({ releaseKey: key, upload, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name })
|
||||
out.push({ releaseKey: key, upload, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name, bytes: files[0].segments[0].bytes })
|
||||
}
|
||||
return out.slice(0, MAX_CANDIDATES)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user