Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -51,7 +51,7 @@ export default class SeedRange extends BaseCommand {
|
||||
const first = Number(this.start)
|
||||
if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
|
||||
const last = first + this.count - 1
|
||||
const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, fragments: 0, inspections: 0 }
|
||||
const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, dropped: 0, fragments: 0, inspections: 0 }
|
||||
let storeMs = 0
|
||||
const cpuStart = process.cpuUsage()
|
||||
const started = Date.now()
|
||||
@@ -90,6 +90,7 @@ export default class SeedRange extends BaseCommand {
|
||||
storeMs += Date.now() - t
|
||||
stats.files += stored.files
|
||||
stats.existing += stored.existing
|
||||
stats.dropped += stored.dropped
|
||||
stats.inspections += stored.inspections
|
||||
}
|
||||
this.progress(stats, started)
|
||||
@@ -106,7 +107,7 @@ export default class SeedRange extends BaseCommand {
|
||||
this.logger.info(
|
||||
`${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
|
||||
`${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
|
||||
`${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known), ` +
|
||||
`${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known, ${n(stats.dropped)} garbage), ` +
|
||||
`${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
|
||||
)
|
||||
this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)
|
||||
|
||||
Reference in new issue
Block a user