Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -11,6 +11,8 @@ import StagingService from '#services/StagingService'
|
||||
import FilePartitions from '#services/FilePartitions'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
import FileKeys from '#services/FileKeys'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
|
||||
const CLEAN_BATCH = 1000
|
||||
@@ -34,6 +36,8 @@ export default class GcScheduler extends BaseCommand {
|
||||
|
||||
/** When release completeness was last recomputed (it runs less often than the sweep). */
|
||||
private completenessAt = 0
|
||||
/** When the garbage signatures were last learned (D42). */
|
||||
private learnedAt = 0
|
||||
/** When split uploads were last merged. */
|
||||
private mergedAt = 0
|
||||
|
||||
@@ -59,6 +63,11 @@ export default class GcScheduler extends BaseCommand {
|
||||
} catch (err: any) {
|
||||
this.logger.error(`file_keys fill failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.learnGarbage()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Garbage signatures failed: ${err.message}`)
|
||||
}
|
||||
if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
|
||||
} while (!this.once && !Shutdown.requested)
|
||||
} finally {
|
||||
@@ -155,6 +164,15 @@ export default class GcScheduler extends BaseCommand {
|
||||
if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
|
||||
/** Learns which upload signatures are garbage from the probed releases (CONCEPT.md D42). */
|
||||
private async learnGarbage() {
|
||||
if (this.dryRun || (!this.once && Date.now() - this.learnedAt < indexerConfig.garbage.learnIntervalMs)) return
|
||||
this.learnedAt = Date.now()
|
||||
const { signatures, dropped, changed } = await GarbageFilter.learn()
|
||||
this.logger.info(`Garbage signatures: ${dropped} of ${signatures} dropped.`)
|
||||
for (const line of changed) this.logger.info(`Garbage signature ${line}`)
|
||||
}
|
||||
|
||||
private async rebuildIndexes() {
|
||||
const key = 'gc:reindex:last'
|
||||
const last = Number((await RedisService.client.get(key)) ?? 0)
|
||||
|
||||
Reference in new issue
Block a user