Daan MeijerandClaude Opus 5.5
211820067a
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
39 lines
1.8 KiB
TypeScript
39 lines
1.8 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import GarbageFilter from '#services/GarbageFilter'
|
|
|
|
/**
|
|
* Learns the garbage upload signatures now (gc:scheduler does it every
|
|
* GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
|
|
* fills signature_probes from a sample of the files probed before it
|
|
* existed, so there is something to learn from right away.
|
|
*/
|
|
export default class GarbageLearn extends BaseCommand {
|
|
public static commandName = 'garbage:learn'
|
|
public static description = 'Learns which upload signatures are garbage and lists them.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
@flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
|
|
declare seed?: number
|
|
|
|
public async run() {
|
|
if (this.seed) {
|
|
const started = Date.now()
|
|
const added = await GarbageFilter.seed(this.seed)
|
|
this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
|
|
}
|
|
const { signatures, dropped, changed } = await GarbageFilter.learn()
|
|
for (const line of changed) this.logger.info(`Changed: ${line}`)
|
|
const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
|
|
for (const row of rows) {
|
|
const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
|
|
this.logger.info(`${row.dropped ? 'DROP' : 'keep'} ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
|
|
}
|
|
this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
|
|
}
|
|
}
|