usenet-indexer/tests/functional/garbage.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import indexerConfig from '#config/indexer'
import FileStore from '#services/FileStore'
import SeedService from '#services/SeedService'
import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import ContentService from '#services/ContentService'
import GarbageFilter from '#services/GarbageFilter'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toSeedFile } from '#lib/seed/wire'
import { explored, uploadTool } from '#lib/garbage/index'

const GROUP = 'alt.binaries.test'
let n = 0

/** Headers of a file posted by ngPost (or nyuu), all its parts. */
function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') {
  const id = () => {
    n++
    return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu`
  }
  return Array.from({ length: parts }, (_, i) =>
    segmentFromHeader(
      {
        'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`,
        'from': 'poster <poster@example.invalid>',
        'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
        'message-id': `<${id()}>`,
        ':bytes': '700000',
      },
      GROUP,
      n
    )!
  )
}

async function dropSignature(tool: string, group = GROUP) {
  await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true })
  GarbageFilter.reset()
}

/** A random-named release with one stored file, as seeding leaves it. */
async function storedRelease(release: Record<string, unknown> = {}, file: Record<string, unknown> = {}) {
  const now = new Date()
  n++
  const [r] = await db
    .table('releases')
    .insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release })
    .returning(['id', 'release_key'])
  const [{ id }] = await FileStore.insert([
    { filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]),
      segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file },
  ])
  return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id }
}

test.group('Garbage filter (D42)', (group) => {
  const explore = indexerConfig.garbage.explorePercent
  group.each.setup(() => {
    // Nothing explored, so a test's release is dropped whatever its key hashes to.
    indexerConfig.garbage.explorePercent = 0
    GarbageFilter.reset()
    return testUtils.db().withGlobalTransaction()
  })
  group.each.teardown(() => {
    indexerConfig.garbage.explorePercent = explore
    GarbageFilter.reset()
  })
  group.teardown(() => QueueService.closeAll())

  test('posting tools are told apart by their Message-IDs', ({ assert }) => {
    assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu')
    assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost')
    assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
    assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
    assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
    assert.equal(uploadTool('seed-1-12@test'), 'other')
    assert.equal(uploadTool(null), 'other')
  })

  test('the explored share is stable per key and about the percentage', ({ assert }) => {
    assert.equal(explored('some-release', 2), explored('some-release', 2))
    assert.isFalse(explored('some-release', 0))
    const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
    assert.isAbove(kept, 150)
    assert.isBelow(kept, 250)
  })

  test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => {
    await dropSignature('ngPost')
    const aggregator = new FileAggregator<IngestSegment>()
    const files = aggregator
      .add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')])
      .map(toSeedFile)
    assert.lengthOf(files, 3)
    const result = await SeedService.store(files)
    assert.include(result, { files: 2, dropped: 1 })
    assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
    assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first())
    assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first())
    assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first())
  })

  test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => {
    await dropSignature('ngPost')
    await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2))
    assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())

    // Off: everything is stored.
    indexerConfig.garbage.enabled = false
    try {
      await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2))
    } finally {
      indexerConfig.garbage.enabled = true
    }
    assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first())
  })

  test('a release dropped after its probe is not stored again', async ({ assert }) => {
    const aggregator = new FileAggregator<IngestSegment>()
    const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile)
    await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' })
    assert.include(await SeedService.store(files), { files: 0, dropped: 1 })
  })

  test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => {
    const garbage = await storedRelease({ encrypted: true })
    assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted')
    assert.isNull(await db.from('files').where('id', garbage.fileId).first())
    assert.isNull(await db.from('releases').where('id', garbage.releaseId).first())
    assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted')

    const container = await storedRelease({ container: 'rclone', encrypted: true })
    assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone')
    const missing = await storedRelease()
    assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing')
  })

  test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => {
    const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' })
    assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5'))
    const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
    assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5'))

    const waiting = await storedRelease({ encrypted: true })
    const now = new Date()
    await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]',
      first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true })
    assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5'))

    const readable = await storedRelease()
    assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5'))
    assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first())
  })

  test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => {
    const { minProbes } = indexerConfig.garbage
    const now = new Date()
    const releases = await db
      .table('releases')
      .insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now })))
      .returning('id')
    await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP })))
    // Five probes of another signature: too few to decide.
    await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP })))

    const learned = await GarbageFilter.learn()
    assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`)
    const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool')
    assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]])

    // One real name in a hundred is above the 0.5% limit: kept again.
    await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
    await GarbageFilter.learn()
    assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped)
  })

  test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => {
    const stored = await storedRelease()
    const now = new Date()
    await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]',
      first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true })
    assert.isNotNull(await ContentService.prepare(stored.fileId))
    await dropSignature('ngPost')
    assert.isNull(await ContentService.prepare(stored.fileId))
    assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection)
  })
})