Drop garbage uploads early (D42)

Random-named uploads that never get a real name or password were ~90% of
what seeding stores. Two filters:

- before storing (seeding, ingest): random-named uploads of a dropped
  signature (posting tool from the Message-ID, first group) or of a
  release dropped after its probe aren't stored; 2% of a dropped
  signature's releases are still kept and probed. Stored files of dropped
  signatures aren't probed.
- after a probe: a random-named release that is still unnamed, without
  password, names from files or spot, with all its probes done, and
  encrypted, a container or missing its first article, is deleted and
  tombstoned in dropped_releases.

Signatures are learned by gc:scheduler from a sample of probed releases
(signature_probes); garbage:learn --seed fills it from existing files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/GarbageLearn.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import GarbageFilter from '#services/GarbageFilter'
+
+/**
+ * Learns the garbage upload signatures now (gc:scheduler does it every
+ * GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
+ * fills signature_probes from a sample of the files probed before it
+ * existed, so there is something to learn from right away.
+ */
+export default class GarbageLearn extends BaseCommand {
+  public static commandName = 'garbage:learn'
+  public static description = 'Learns which upload signatures are garbage and lists them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
+  declare seed?: number
+
+  public async run() {
+    if (this.seed) {
+      const started = Date.now()
+      const added = await GarbageFilter.seed(this.seed)
+      this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
+    }
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    for (const line of changed) this.logger.info(`Changed: ${line}`)
+    const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
+    for (const row of rows) {
+      const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
+      this.logger.info(`${row.dropped ? 'DROP' : 'keep'}  ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
+    }
+    this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
+  }
+}

commands/GcScheduler.ts

@@ -11,6 +11,8 @@ import StagingService from '#services/StagingService'
 import FilePartitions from '#services/FilePartitions'
 import ReleaseCompleteness from '#services/ReleaseCompleteness'
 import FileKeys from '#services/FileKeys'
+import GarbageFilter from '#services/GarbageFilter'
+import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
@@ -34,6 +36,8 @@ export default class GcScheduler extends BaseCommand {
 
   /** When release completeness was last recomputed (it runs less often than the sweep). */
   private completenessAt = 0
+  /** When the garbage signatures were last learned (D42). */
+  private learnedAt = 0
   /** When split uploads were last merged. */
   private mergedAt = 0
 
@@ -59,6 +63,11 @@ export default class GcScheduler extends BaseCommand {
         } catch (err: any) {
           this.logger.error(`file_keys fill failed: ${err.message}`)
         }
+        try {
+          await this.learnGarbage()
+        } catch (err: any) {
+          this.logger.error(`Garbage signatures failed: ${err.message}`)
+        }
         if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
       } while (!this.once && !Shutdown.requested)
     } finally {
@@ -155,6 +164,15 @@ export default class GcScheduler extends BaseCommand {
     if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
   }
 
+  /** Learns which upload signatures are garbage from the probed releases (CONCEPT.md D42). */
+  private async learnGarbage() {
+    if (this.dryRun || (!this.once && Date.now() - this.learnedAt < indexerConfig.garbage.learnIntervalMs)) return
+    this.learnedAt = Date.now()
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    this.logger.info(`Garbage signatures: ${dropped} of ${signatures} dropped.`)
+    for (const line of changed) this.logger.info(`Garbage signature ${line}`)
+  }
+
   private async rebuildIndexes() {
     const key = 'gc:reindex:last'
     const last = Number((await RedisService.client.get(key)) ?? 0)

commands/SeedRange.ts

@@ -51,7 +51,7 @@ export default class SeedRange extends BaseCommand {
     const first = Number(this.start)
     if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
     const last = first + this.count - 1
-    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, fragments: 0, inspections: 0 }
+    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, dropped: 0, fragments: 0, inspections: 0 }
     let storeMs = 0
     const cpuStart = process.cpuUsage()
     const started = Date.now()
@@ -90,6 +90,7 @@ export default class SeedRange extends BaseCommand {
             storeMs += Date.now() - t
             stats.files += stored.files
             stats.existing += stored.existing
+            stats.dropped += stored.dropped
             stats.inspections += stored.inspections
           }
           this.progress(stats, started)
@@ -106,7 +107,7 @@ export default class SeedRange extends BaseCommand {
     this.logger.info(
       `${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
         `${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
-        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known), ` +
+        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known, ${n(stats.dropped)} garbage), ` +
         `${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
     )
     this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)