usenet-indexer/tests/unit/rules/rule_set.spec.ts

import { test } from '@japa/runner'
import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rules/index'
import { sevenZipVolume } from '#lib/files/index'

const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath', poster = 'someone@example.com (Someone)') => ({
  name,
  totalParts,
  groups: [group],
  firstMessageId,
  poster,
  segments: totalParts,
  bytes: totalParts * 700_000,
})
/** A single archive of `segments` articles and `bytes` in all, posted to `groups` groups. */
const archive = (name: string, poster: string, groups: number, segments: number, bytes: number, firstMessageId = NYUU) => ({
  name,
  totalParts: segments,
  groups: Array.from({ length: groups }, (_, i) => `alt.binaries.g${i}`),
  firstMessageId,
  poster,
  segments,
  bytes,
})
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'

test.group('rules / owner', () => {
  test('a crosspost belongs to the indexed group with the lowest priority, whatever the Xref order', ({ assert }) => {
    const priorities = { 'alt.binaries.boneless': 1, 'alt.binaries.ath': 3, 'alt.binaries.added.later': 1001 }
    assert.equal(ownerOf(['alt.binaries.ath', 'alt.binaries.boneless'], priorities), 'alt.binaries.boneless')
    assert.equal(ownerOf(['alt.binaries.added.later', 'alt.binaries.ath'], priorities), 'alt.binaries.ath')
    assert.equal(ownerOf(['alt.binaries.not.indexed'], priorities), null)
  })
})

test.group('rules / verdicts', () => {
  test("a dropped poster's uploads are dropped whatever they are; others with the word in their name are not", ({ assert }) => {
    const pics = (poster: string) => rules.judge('k', [file('!index0001.jpg', 1, NYUU, 'alt.binaries.erotica', poster)])
    assert.include(pics('@_CelebMovie_@'), { rule: 'poster', key: 'celebmovie' })
    assert.include(pics('@__Celeb_Movie__@'), { rule: 'poster', key: 'celebmovie' })
    assert.include(pics('NudeCelebs@CelebWow.com'), { rule: 'poster', key: 'celebwow' })
    assert.include(pics('@nude-celebs'), { rule: 'poster', key: 'nudecelebs' })
    assert.deepEqual(pics('celeborn@not.here (Celeborn)'), { rule: null })
    assert.deepEqual(pics('"CelebTA.com" <from@celebta.com>'), { rule: null })
    assert.deepEqual(new RuleSet({ ...DEFAULT_RULES, droppedPosters: [], samplePercent: 0, priorities: {}, droppedSignatures: [] }).judge('k', [file('a.jpg', 1, NYUU, 'g', '@_CelebMovie_@')]), { rule: null })
  })

  test('a readable file keeps its whole release', ({ assert }) => {
    assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
  })

  test('bare executables are dropped, sidecars aside; an executable among other content is kept', ({ assert }) => {
    assert.include(rules.judge('k', [file('How to Make Love to a Woman 2010.scr', 10, NYUU), file('How to Make Love to a Woman 2010.par2', 1, NYUU)]), { rule: 'executable', key: 'scr' })
    assert.include(rules.judge('k', [file('Adobe Illustrator CS2 + ssg keygen.zip  .exe', 3, NYUU)]), { rule: 'executable', key: 'exe' })
    assert.deepEqual(rules.judge('k', [file('Some.App.v1.0.part1.rar', 50, NYUU), file('setup.exe', 2, NYUU)]), { rule: null })
  })

  test('random names: per-article with few parts, garbage by tool or signature, else kept', ({ assert }) => {
    assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa', 2, NYUU)]), { rule: 'per-article', key: 'nyuu' })
    assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST)]), { rule: 'garbage', key: 'ngPost alt.binaries.ath' })
    assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU, 'alt.binaries.misc')]), { rule: 'garbage', key: 'nyuu alt.binaries.misc' })
    assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU)]), { rule: null })
  })

  test('stub archives: a settled single archive in 2+ groups with 250 kB articles, or 4+ groups, or a webmail poster', ({ assert }) => {
    // Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave; the article-size leg alone without webmail.
    assert.include(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com (Jahnunsinos007)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'AW' })
    assert.include(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'Yenc@power-post.org (Yenc-PP-A&A)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'A' })
    // Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone).
    assert.include(rules.judge('k', [archive('Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar', 'Yenc@power-post.org', 5, 2, 319_000)]), { rule: 'stub-archive', key: 'X' })
    // A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave, but only under 100 MB without the article-size leg.
    assert.include(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 100, 65_000_000)]), { rule: 'stub-archive', key: 'W' })
    assert.deepEqual(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 400, 260_000_000)]), { rule: null })
    // Not settled: a par2 can still join.
    assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)], false), { rule: null })
  })

  test('stub archives: small, single-group, split and large hash-named archives are kept', ({ assert }) => {
    // An e-book: a one- or two-article release always has small articles (LOG.md (126)).
    assert.deepEqual(rules.judge('k', [archive('Maeve Binchy - Tara Road (epub).rar', 'kie@home.now (KiE)', 2, 2, 521_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 1, 48, 12_000_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'poster@example.org', 2, 100, 70_000_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.part01.rar', 'a@gmail.com', 2, 48, 12_000_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('cbcd32359460df1fa9b7d01d8720.rar', 'b648de0d1c0f7a9a77be273d@gmail.com', 2, 3163, 2_340_000_000)]), { rule: null })
    // ROT13-named volumes of passworded videos in a.b.erotica, 258 kB articles, two groups, 1-270 MB; and the 100 MB cap on every leg (LOG.md (128)).
    assert.deepEqual(rules.judge('k', [archive('Ghful.17.12.07.Qnvvfl.Fgbar.cneg3.rar', 'nospam@nospam.nl (EvH)', 2, 5, 1_147_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Ghful.16.07.05.Wbwb.Xvff.cneg1.rar', 'nospam@nospam.nl (EvH)', 2, 1051, 271_000_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Tinasoft EasyCafe 2.2.14 by OmerSalaj.rar', 'Seven7i@mail.com ({Seven7i})', 4, 463, 117_000_000)]), { rule: null })
    assert.include(rules.judge('k', [archive('Tipard DVD Ripper - 10 0 12 x86 x64 incl Patch.rar', 'Seven7i@mail.com ({Seven7i})', 4, 244, 62_000_000)]), { rule: 'stub-archive', key: 'AXW' })
    // A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)).
    assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl.zip', 'home@home.com (TexasDj)', 2, 13, 3_300_000), groups: ['alt.binaries.sounds.karaoke', 'alt.binaries.test'] }]), { rule: null })
    // A rar with its par2 is no single archive.
    assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'a@gmail.com', 2, 48, 12_000_000), file('Chat Watch 1.2.1.par2', 1, NYUU)]), { rule: null })
    const off = new RuleSet({ ...DEFAULT_RULES, dropStubArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
    assert.deepEqual(off.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)]), { rule: null })
  })

  test('nonsense pieces: one article of 100 kB to 1 MB under a release-like name', ({ assert }) => {
    assert.include(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: 'nonsense-piece', key: 'nyuu' })
    // A 3 kB cue/log sidecar rar with a year in its name: under the floor (the tiny-archive rule's outside the music groups). A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule.
    assert.deepEqual(rules.judge('k', [{ ...archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000), groups: ['alt.binaries.sounds.lossless'] }]), { rule: null })
    assert.include(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: 'tiny-archive' })
    assert.deepEqual(rules.judge('k', [archive('Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar', 'fake@address.com (Psychopath)', 2, 1, 303_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 382_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar', 'saggir@domain.com', 1, 1, 113_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 2, 790_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
    const off = new RuleSet({ ...DEFAULT_RULES, dropNonsensePieces: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
    assert.deepEqual(off.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
  })

  test('tiny archives: one article of 2-100 kB under a release name, spaces allowed', ({ assert }) => {
    // A RAR holding "<title>.exe": the 2015-17 flood in miniature (LOG.md (131)).
    assert.include(rules.judge('k', [archive('Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar', 'Yanc@power-post.org <Yanc@power-post.org>', 1, 1, 15_084)]), { rule: 'tiny-archive', key: 'nyuu' })
    assert.include(rules.judge('k', [archive('Cicero Magazin fuer politische Kultur Mai 05 2016.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 17_478)]), { rule: 'tiny-archive', key: 'nyuu' })
    // A scene name without a year, a tag or a group suffix: not release-like, so kept.
    assert.deepEqual(rules.judge('k', [archive('[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar', 'Yanc@power-post.org', 2, 1, 23_449)]), { rule: null })
    assert.include(rules.judge('k', [archive('Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 7_869)]), { rule: 'tiny-archive', key: 'nyuu' })
    // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
    assert.deepEqual(rules.judge('k', [archive('The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar', 'henk@example.org', 1, 1, 38_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar', 'poster@example.org', 1, 1, 60_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Some Band - Album (2011) covers.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 38_000)]), { rule: null })
    // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's; not settled.
    assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl (2017).zip', 'home@home.com', 1, 1, 60_000), groups: ['alt.binaries.sounds.karaoke'] }]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 2, 20_000)]), { rule: null })
    assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 1_500)]), { rule: null })
    assert.include(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 150_000)]), { rule: 'nonsense-piece' })
    assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)], false), { rule: null })
    const off = new RuleSet({ ...DEFAULT_RULES, dropTinyArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
    assert.deepEqual(off.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
  })
})

test.group('rules / upload tools', () => {
  test('posting tools are told apart by their Message-IDs', ({ assert }) => {
    assert.equal(uploadTool(NYUU), 'nyuu')
    assert.equal(uploadTool(NGPOST), 'ngPost')
    assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
    assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
    assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
    assert.equal(uploadTool(null), 'other')
  })

  test('the sampled share is stable per key and about the percentage', ({ assert }) => {
    assert.equal(explored('some-release', 2), explored('some-release', 2))
    const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
    assert.isAbove(kept, 150)
    assert.isBelow(kept, 250)
  })
})

test.group('files / volumes', () => {
  test('7z volume numbers', ({ assert }) => {
    assert.equal(sevenZipVolume('x.7z.001'), 1)
    assert.equal(sevenZipVolume('x.7z'), 0)
    assert.isNull(sevenZipVolume('x.rar'))
  })
})