Drop garbage uploads early (D42)

Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 11:21:01 +02:00
1 parent 35d0b2da65
commit 211820067a
@@ -0,0 +1,190 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import indexerConfig from '#config/indexer'
+import FileStore from '#services/FileStore'
+import SeedService from '#services/SeedService'
+import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import ContentService from '#services/ContentService'
+import GarbageFilter from '#services/GarbageFilter'
+import QueueService from '#services/QueueService'
+import { FileAggregator } from '#lib/aggregate/index'
+import { toSeedFile } from '#lib/seed/wire'
+import { explored, uploadTool } from '#lib/garbage/index'
+
+const GROUP = 'alt.binaries.test'
+let n = 0
+
+/** Headers of a file posted by ngPost (or nyuu), all its parts. */
+function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') {
+  const id = () => {
+    n++
+    return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu`
+  }
+  return Array.from({ length: parts }, (_, i) =>
+    segmentFromHeader(
+      {
+        'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`,
+        'from': 'poster <poster@example.invalid>',
+        'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
+        'message-id': `<${id()}>`,
+        ':bytes': '700000',
+      },
+      GROUP,
+      n
+    )!
+  )
+}
+
+async function dropSignature(tool: string, group = GROUP) {
+  await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true })
+  GarbageFilter.reset()
+}
+
+/** A random-named release with one stored file, as seeding leaves it. */
+async function storedRelease(release: Record<string, unknown> = {}, file: Record<string, unknown> = {}) {
+  const now = new Date()
+  n++
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release })
+    .returning(['id', 'release_key'])
+  const [{ id }] = await FileStore.insert([
+    { filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]),
+      segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file },
+  ])
+  return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id }
+}
+
+test.group('Garbage filter (D42)', (group) => {
+  const explore = indexerConfig.garbage.explorePercent
+  group.each.setup(() => {
+    // Nothing explored, so a test's release is dropped whatever its key hashes to.
+    indexerConfig.garbage.explorePercent = 0
+    GarbageFilter.reset()
+    return testUtils.db().withGlobalTransaction()
+  })
+  group.each.teardown(() => {
+    indexerConfig.garbage.explorePercent = explore
+    GarbageFilter.reset()
+  })
+  group.teardown(() => QueueService.closeAll())
+
+  test('posting tools are told apart by their Message-IDs', ({ assert }) => {
+    assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu')
+    assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost')
+    assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
+    assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
+    assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
+    assert.equal(uploadTool('seed-1-12@test'), 'other')
+    assert.equal(uploadTool(null), 'other')
+  })
+
+  test('the explored share is stable per key and about the percentage', ({ assert }) => {
+    assert.equal(explored('some-release', 2), explored('some-release', 2))
+    assert.isFalse(explored('some-release', 0))
+    const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
+    assert.isAbove(kept, 150)
+    assert.isBelow(kept, 250)
+  })
+
+  test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => {
+    await dropSignature('ngPost')
+    const aggregator = new FileAggregator<IngestSegment>()
+    const files = aggregator
+      .add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')])
+      .map(toSeedFile)
+    assert.lengthOf(files, 3)
+    const result = await SeedService.store(files)
+    assert.include(result, { files: 2, dropped: 1 })
+    assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
+    assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first())
+    assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first())
+    assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first())
+  })
+
+  test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => {
+    await dropSignature('ngPost')
+    await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2))
+    assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
+
+    // Off: everything is stored.
+    indexerConfig.garbage.enabled = false
+    try {
+      await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2))
+    } finally {
+      indexerConfig.garbage.enabled = true
+    }
+    assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first())
+  })
+
+  test('a release dropped after its probe is not stored again', async ({ assert }) => {
+    const aggregator = new FileAggregator<IngestSegment>()
+    const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile)
+    await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' })
+    assert.include(await SeedService.store(files), { files: 0, dropped: 1 })
+  })
+
+  test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => {
+    const garbage = await storedRelease({ encrypted: true })
+    assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted')
+    assert.isNull(await db.from('files').where('id', garbage.fileId).first())
+    assert.isNull(await db.from('releases').where('id', garbage.releaseId).first())
+    assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted')
+
+    const container = await storedRelease({ container: 'rclone', encrypted: true })
+    assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone')
+    const missing = await storedRelease()
+    assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing')
+  })
+
+  test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => {
+    const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' })
+    assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5'))
+    const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
+    assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5'))
+
+    const waiting = await storedRelease({ encrypted: true })
+    const now = new Date()
+    await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]',
+      first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true })
+    assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5'))
+
+    const readable = await storedRelease()
+    assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5'))
+    assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first())
+  })
+
+  test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => {
+    const { minProbes } = indexerConfig.garbage
+    const now = new Date()
+    const releases = await db
+      .table('releases')
+      .insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now })))
+      .returning('id')
+    await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP })))
+    // Five probes of another signature: too few to decide.
+    await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP })))
+
+    const learned = await GarbageFilter.learn()
+    assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`)
+    const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool')
+    assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]])
+
+    // One real name in a hundred is above the 0.5% limit: kept again.
+    await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
+    await GarbageFilter.learn()
+    assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped)
+  })
+
+  test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => {
+    const stored = await storedRelease()
+    const now = new Date()
+    await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]',
+      first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true })
+    assert.isNotNull(await ContentService.prepare(stored.fileId))
+    await dropSignature('ngPost')
+    assert.isNull(await ContentService.prepare(stored.fileId))
+    assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection)
+  })
+})