Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -0,0 +1,38 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/**
|
||||
* Learns the garbage upload signatures now (gc:scheduler does it every
|
||||
* GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
|
||||
* fills signature_probes from a sample of the files probed before it
|
||||
* existed, so there is something to learn from right away.
|
||||
*/
|
||||
export default class GarbageLearn extends BaseCommand {
|
||||
public static commandName = 'garbage:learn'
|
||||
public static description = 'Learns which upload signatures are garbage and lists them.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
|
||||
declare seed?: number
|
||||
|
||||
public async run() {
|
||||
if (this.seed) {
|
||||
const started = Date.now()
|
||||
const added = await GarbageFilter.seed(this.seed)
|
||||
this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
const { signatures, dropped, changed } = await GarbageFilter.learn()
|
||||
for (const line of changed) this.logger.info(`Changed: ${line}`)
|
||||
const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
|
||||
for (const row of rows) {
|
||||
const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
|
||||
this.logger.info(`${row.dropped ? 'DROP' : 'keep'} ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
|
||||
}
|
||||
this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user