import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import indexerConfig from '#config/indexer' import FileStore from '#services/FileStore' import SeedService from '#services/SeedService' import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService' import ContentService from '#services/ContentService' import GarbageFilter from '#services/GarbageFilter' import QueueService from '#services/QueueService' import { FileAggregator } from '#lib/aggregate/index' import { toSeedFile } from '#lib/seed/wire' import { explored, uploadTool } from '#lib/garbage/index' const GROUP = 'alt.binaries.test' let n = 0 /** Headers of a file posted by ngPost (or nyuu), all its parts. */ function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') { const id = () => { n++ return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu` } return Array.from({ length: parts }, (_, i) => segmentFromHeader( { 'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'poster ', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<${id()}>`, ':bytes': '700000', }, GROUP, n )! ) } async function dropSignature(tool: string, group = GROUP) { await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true }) GarbageFilter.reset() } /** A random-named release with one stored file, as seeding leaves it. */ async function storedRelease(release: Record = {}, file: Record = {}) { const now = new Date() n++ const [r] = await db .table('releases') .insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release }) .returning(['id', 'release_key']) const [{ id }] = await FileStore.insert([ { filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]), segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file }, ]) return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id } } test.group('Garbage filter (D42)', (group) => { const explore = indexerConfig.garbage.explorePercent group.each.setup(() => { // Nothing explored, so a test's release is dropped whatever its key hashes to. indexerConfig.garbage.explorePercent = 0 GarbageFilter.reset() return testUtils.db().withGlobalTransaction() }) group.each.teardown(() => { indexerConfig.garbage.explorePercent = explore GarbageFilter.reset() }) group.teardown(() => QueueService.closeAll()) test('posting tools are told apart by their Message-IDs', ({ assert }) => { assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu') assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost') assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM') assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp') assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random') assert.equal(uploadTool('seed-1-12@test'), 'other') assert.equal(uploadTool(null), 'other') }) test('the explored share is stable per key and about the percentage', ({ assert }) => { assert.equal(explored('some-release', 2), explored('some-release', 2)) assert.isFalse(explored('some-release', 0)) const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length assert.isAbove(kept, 150) assert.isBelow(kept, 250) }) test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => { await dropSignature('ngPost') const aggregator = new FileAggregator() const files = aggregator .add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')]) .map(toSeedFile) assert.lengthOf(files, 3) const result = await SeedService.store(files) assert.include(result, { files: 2, dropped: 1 }) assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first()) assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first()) assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first()) assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first()) }) test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => { await dropSignature('ngPost') await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)) assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first()) // Off: everything is stored. indexerConfig.garbage.enabled = false try { await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2)) } finally { indexerConfig.garbage.enabled = true } assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first()) }) test('a release dropped after its probe is not stored again', async ({ assert }) => { const aggregator = new FileAggregator() const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile) await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' }) assert.include(await SeedService.store(files), { files: 0, dropped: 1 }) }) test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => { const garbage = await storedRelease({ encrypted: true }) assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted') assert.isNull(await db.from('files').where('id', garbage.fileId).first()) assert.isNull(await db.from('releases').where('id', garbage.releaseId).first()) assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted') const container = await storedRelease({ container: 'rclone', encrypted: true }) assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone') const missing = await storedRelease() assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing') }) test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => { const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' }) assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5')) const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' }) assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5')) const waiting = await storedRelease({ encrypted: true }) const now = new Date() await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true }) assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5')) const readable = await storedRelease() assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5')) assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first()) }) test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => { const { minProbes } = indexerConfig.garbage const now = new Date() const releases = await db .table('releases') .insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now }))) .returning('id') await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP }))) // Five probes of another signature: too few to decide. await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP }))) const learned = await GarbageFilter.learn() assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`) const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool') assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]]) // One real name in a hundred is above the 0.5% limit: kept again. await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' }) await GarbageFilter.learn() assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped) }) test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => { const stored = await storedRelease() const now = new Date() await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true }) assert.isNotNull(await ContentService.prepare(stored.fileId)) await dropSignature('ngPost') assert.isNull(await ContentService.prepare(stored.fileId)) assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection) }) })