Spots: the spotter and the spot inspector, passwords kept apart

Two processes, one NNTP connection each (the coordinator 40 -> 38): - spots:read (service spotter): free.pt's overviews after the last article seen, each spot's full header verified (lib/spotnet from voyager 1, as Spotweb), its description, website and NZB articles; the password its title or text names; delete/dispose messages applied. - spots:inspect (service spot-inspector): each spot's compressed NZB read, its files listed (spot_files), each matched to the header pass's release by the same release identity in one of its groups within 48 h (release_spots); unmatched spots matched again hourly, after 24 h a release of their own (source spot) with the spot's NZB. - release_passwords: per release and password, its source (nzb, spot-title, spot-text) and spot; the catalog keeps only passworded. XoverClient.head(); PROBE_CONNECTIONS default 10. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 12:10:10 +02:00
1 parent 3fd08b598a
commit 7df5a638ca
@@ -0,0 +1,109 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { Spotter } from '#services/spotter'
+import { SpotInspector } from '#services/spot_inspector'
+import { releaseIdentity } from '#lib/release/index'
+import { parseSubject } from '#lib/headers/index'
+
+// Message ids whose sha1('<id>') starts with 0000: self-signed spots (key 7) pass on that hashcash alone.
+const SPOT = 'spottest731@spot.net'
+const DISPOSE = 'spottest168100@spot.net'
+const OTHER = 'spottest173225@spot.net'
+// A poster's key longer than 50 characters: its spotter id makes a personal dispose count.
+const PUBKEY = 'A'.repeat(64)
+const STAMP = 1_760_000_000
+
+/** A spot's raw HEAD: From with the spot header (category 1, key 7), the X-XML with title, description and NZB. */
+function rawSpot(messageId: string, title: string, description: string, key = 7, stamp = STAMP) {
+  return [
+    `From: Spotter <${PUBKEY}@1${key}a09z00.734003200.0.${stamp}.0.SIG>`,
+    `Subject: ${title} | Tag`,
+    `Message-ID: <${messageId}>`,
+    `X-XML: <Spotnet><Posting><Key>${key}</Key><Title>${title}</Title><Description>${description}</Description>`,
+    `X-XML: <Category>01<Sub>01a09</Sub></Category><NZB><Segment>nzb1@spot.net</Segment></NZB><Size>734003200</Size></Posting></Spotnet>`,
+    '',
+  ].join('\r\n')
+}
+
+/** An NZB as spotters post them: the files' original subjects (buildNzb writes its own). */
+function nzbXml(subjects: string[], poster: string, date: number, password?: string) {
+  const esc = (t: string) => t.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+  const head = password ? `<head><meta type="password">${esc(password)}</meta></head>` : ''
+  const files = subjects
+    .map((s, i) => `<file poster="${esc(poster)}" date="${date}" subject="${esc(s)}"><groups><group>alt.binaries.test.spots</group></groups><segments><segment bytes="400000000" number="1">seg${i}@x</segment></segments></file>`)
+    .join('')
+  return `<?xml version="1.0" encoding="UTF-8"?><nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${head}${files}</nzb>`
+}
+
+test.group('Spots', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('the spotter lists a verified spot with the password its text names, and a dispose removes it', async ({ assert }) => {
+    const spotter = new Spotter()
+    assert.equal(await spotter.store(rawSpot(SPOT, 'Some.Movie.2024.1080p', 'Mooie film[br]Wachtwoord: geheim123'), SPOT, 10), 'listed')
+    const spot = await db.from('spots').where('message_id', SPOT).firstOrFail()
+    assert.equal(spot.title, 'Some.Movie.2024.1080p')
+    assert.equal(spot.password, 'geheim123')
+    assert.equal(spot.password_source, 'spot-text')
+    assert.deepEqual(spot.nzb_segments, ['nzb1@spot.net'])
+    // A {{pw}} in the title wins, and is said to come from the title.
+    assert.equal(await spotter.store(rawSpot(OTHER, 'Other.Show{{t1tl3pw}}', 'Geen ww'), OTHER, 11), 'listed')
+    assert.equal((await db.from('spots').where('message_id', OTHER).firstOrFail()).password_source, 'spot-title')
+    // A message id without the hashcash doesn't verify.
+    assert.equal(await spotter.store(rawSpot('nohash@spot.net', 'X', ''), 'nohash@spot.net', 12), 'unverified')
+    // The poster's own dispose within 5 days.
+    await db.table('release_spots').insert({ release_id: 1, spot_id: SPOT, files: 3 })
+    assert.equal(await spotter.store(rawSpot(DISPOSE, `dispose <${SPOT}>`, '', 2, STAMP + 3600), DISPOSE, 13), 'moderation')
+    assert.equal((await db.from('spots').where('message_id', SPOT).firstOrFail()).status, 'removed')
+    assert.equal((await db.from('release_spots').where('spot_id', SPOT).select('*')).length, 0)
+  })
+
+  test("the inspector lists an NZB's files and links the releases they match, with the password apart", async ({ assert }) => {
+    const [g] = await db.table('groups').insert({ name: 'alt.binaries.test.spots', priority: 1 }).returning('id')
+    const groupId = Number(g.id)
+    const poster = 'up <up@load.er>'
+    const posted = new Date('2025-10-09T10:00:00Z')
+    const subjects = [1, 2, 3].map((i) => `Some.Movie.2024.1080p [${i}/3] - "Some.Movie.2024.1080p.part0${i}.rar" yEnc (1/50)`)
+    // The release the header pass built for these files: the same identity, in the group.
+    const parsed = parseSubject(subjects[0])!
+    const identity = releaseIdentity({ poster, subject: parsed.base, name: parsed.name, weak: false })!
+    const [r] = await db
+      .table('releases')
+      .insert({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster, groups: '[]', group_id: groupId, files: 3, posted_at: posted, readable: true })
+      .returning('id')
+    const releaseId = Number(r.id)
+    const nzb = nzbXml(subjects, poster, posted.getTime() / 1000, 'nzbpw')
+    await new Spotter().store(rawSpot(SPOT, 'Some.Movie.2024.1080p', 'Wachtwoord: geheim123'), SPOT, 10)
+    const spot = await db.from('spots').where('message_id', SPOT).firstOrFail()
+
+    const outcome = await new SpotInspector().inspectNzb(spot, nzb)
+    assert.deepEqual(outcome, { status: 'inspected', matched: 1 })
+    const files = await db.from('spot_files').where('spot_id', SPOT).orderBy('idx')
+    assert.deepEqual(files.map((f) => f.name), ['Some.Movie.2024.1080p.part01.rar', 'Some.Movie.2024.1080p.part02.rar', 'Some.Movie.2024.1080p.part03.rar'])
+    assert.deepEqual([...new Set(files.map((f) => Number(f.release_id)))], [releaseId])
+    assert.equal((await db.from('release_spots').where({ release_id: releaseId, spot_id: SPOT }).firstOrFail()).files, 3)
+    // The NZB's own password over the one in the spot's text, kept apart from the release.
+    const passwords = await db.from('release_passwords').where('release_id', releaseId).select('password', 'source')
+    assert.deepEqual(passwords, [{ password: 'nzbpw', source: 'nzb' }])
+    assert.isTrue((await db.from('releases').where('id', releaseId).firstOrFail()).passworded)
+    const after = await db.from('spots').where('message_id', SPOT).firstOrFail()
+    assert.equal(after.status, 'inspected')
+    assert.equal(Number(after.release_id), releaseId)
+    assert.equal(after.matched, 3)
+  })
+
+  test('a spot whose files match no release becomes one of its own, with its NZB', async ({ assert }) => {
+    await db.table('groups').insert({ name: 'alt.binaries.test.spots', priority: 1 })
+    const nzb = nzbXml(['[1/1] - "lonely.file.mkv" yEnc (1/9)'], 'p', STAMP)
+    await new Spotter().store(rawSpot(SPOT, 'Lonely File', ''), SPOT, 10)
+    const inspector = new SpotInspector()
+    assert.deepEqual(await inspector.inspectNzb(await db.from('spots').where('message_id', SPOT).firstOrFail(), nzb), { status: 'inspected', matched: 0 })
+    assert.isTrue(await inspector.standalone(SPOT))
+    const release = await db.from('releases').where('source', 'spot').firstOrFail()
+    assert.equal(release.name, 'Lonely File')
+    assert.equal(release.files, 1)
+    const { default: NzbService } = await import('#services/nzb_service')
+    assert.include((await NzbService.build(Number(release.id)))!.xml, 'lonely.file.mkv')
+  })
+})
@@ -0,0 +1,186 @@
+import { test } from '@japa/runner'
+import { deflateRawSync } from 'node:zlib'
+import {
+  decodeSpotBinary,
+  decodeSpotText,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  spotCategoryLabels,
+  verifySpotHeader,
+} from '#lib/spotnet/index'
+import { ubbToText } from '#lib/newznab/xml'
+
+// A real spot from free.pt (self-signed, key 7), 2026-09-28.
+const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
+
+test.group('spotnet / headers', () => {
+  test('parses a spot header', ({ assert }) => {
+    const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
+    assert.include(spot, {
+      poster: 'Bart',
+      title: 'The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi',
+      tag: 'Bart',
+      category: 0,
+      keyId: 7,
+      size: 21651233204,
+      messageId: 'jWCbaAa4f1A7LC6agmOEx@spot.net',
+    })
+    assert.deepEqual(spot.subcats, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] })
+  })
+
+  test('verifies its signatures, and rejects another message id', ({ assert }) => {
+    const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
+    assert.isTrue(verifySpotHeader(spot))
+    assert.isFalse(verifySpotHeader({ ...spot, messageId: 'other@spot.net' }))
+    assert.isNull(parseSpotHeader('someone <x@y>', 'subject', '<a@b>'))
+  })
+})
+
+test.group('spotnet / xml and nzb', () => {
+  test('reads the NZB segments and category from the X-XML lines', ({ assert }) => {
+    const raw = [
+      'From: Bart <x@y>',
+      'X-XML: <Spotnet><Posting><Key>7</Key><Title>A &amp; B</Title><Category>01<Sub>01a09</Sub>',
+      'X-XML: <Sub>01z00</Sub></Category><NZB><Segment>abc@spot.net</Segment><Segment>def@spot.net</Segment></NZB>',
+      'X-XML: <Size>123</Size></Posting></Spotnet>',
+    ].join('\r\n')
+    const { headers, xml } = parseSpotArticleHeaders(raw)
+    assert.equal(headers.from, 'Bart <x@y>')
+    const spot = parseSpotXml(xml)!
+    assert.deepEqual(spot.nzbSegments, ['abc@spot.net', 'def@spot.net'])
+    assert.include(spot, { title: 'A & B', category: 0, size: 123 })
+    assert.deepEqual(spot.subcats, ['01a09', '01z00'])
+  })
+
+  test('reads the description, tag and image', ({ assert }) => {
+    const spot = parseSpotXml(
+      `<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
+        `<Description>Lily Tomlin[br][br]If you&#39;re &#8216;in&#8217; [url=https://example.com/a]here[/url]</Description>` +
+        `<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
+        `<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
+    )!
+    assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
+    assert.equal(spot.tag, 'Bart')
+    assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
+    // Older spots link an image on the web.
+    assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
+    assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
+  })
+
+  test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
+    const raw = [
+      'From: vanker',
+      ' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
+      'Subject: The postman | The postman |',
+      'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van &quot;The',
+      'X-XML:  postman&quot;]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
+      'Date: 01 Feb 2010 02:43:16 GMT',
+    ].join('\r\n')
+    const { headers, xml } = parseSpotArticleHeaders(raw)
+    assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
+    assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
+    const spot = parseSpotXml(xml)!
+    assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
+    assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
+    assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
+  })
+
+  test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
+    assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
+    assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
+  })
+
+  test('names categories as Spotweb does', ({ assert }) => {
+    assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
+      head: 'Image',
+      subcats: [
+        { type: 'Type', values: ['Movie'] },
+        { type: 'Format', values: ['x264'] },
+        { type: 'Source', values: ['Retail'] },
+        { type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
+        { type: 'Genre', values: ['Science Fiction'] },
+      ],
+    })
+    assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
+  })
+
+  test('turns UBB markup into plain text', ({ assert }) => {
+    assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
+  })
+
+  test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
+    const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
+    const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
+      .toString('latin1')
+      .replace(/[=\n\r\0]/g, (c) => ({ '=': '=D', '\n': '=C', '\r': '=B', '\0': '=A' })[c]!)
+    // Posted as lines, dot-stuffed, over two articles.
+    const lines = escaped.match(/.{1,40}/gs)!.map((l) => (l.startsWith('.') ? '.' + l : l))
+    const half = Math.ceil(lines.length / 2)
+    const bodies = [lines.slice(0, half), lines.slice(half)].map((ls) => Buffer.from(ls.join('\r\n'), 'latin1'))
+    assert.equal(decodeSpotBinary(bodies).toString('latin1'), nzb)
+  })
+})
+
+test.group('spotnet / moderation', () => {
+  test('recognises delete and dispose commands from key 2 only', async ({ assert }) => {
+    const { moderationTarget, spotterId } = await import('#lib/spotnet/index')
+    assert.equal(moderationTarget({ keyId: 2, title: 'DISPOSE 7IzfA0aCFTkOne6agvY06@spot.net - The Voice DE S15E07 (1080p)' }), '7IzfA0aCFTkOne6agvY06@spot.net')
+    assert.equal(moderationTarget({ keyId: 2, title: 'delete <abc@spot.net>' }), 'abc@spot.net')
+    assert.isNull(moderationTarget({ keyId: 7, title: 'DISPOSE abc@spot.net' }))
+    assert.isNull(moderationTarget({ keyId: 2, title: 'The Disposal 2024' }))
+    // Spotweb: base64 of the little-endian CRC32 of the key, without / + =.
+    assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
+  })
+})
+
+test.group('spotnet / passwords in descriptions', () => {
+  test('finds passwords written in the description or title', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
+    assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
+    assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
+    assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
+    assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
+    assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
+    assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
+  })
+
+  test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    // LOG.md 2026-10-08 (10)
+    assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg')
+    assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren')
+    assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD')
+    assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022')
+    assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg')
+    assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2')
+    assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos')
+    assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie')
+  })
+
+  test('ignores sentences about passwords', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    for (const description of [
+      'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
+      '[b]--> Geen wachtwoord benodigd [/b]',
+      'zitten 2 bestanden in en is met een password beschermd.',
+      'It also removes the need for manual password entry and improves the reliability',
+      'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
+      '29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
+      'Wachtwoord: geen',
+      'WW: zie nzb',
+      'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
+      'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen',
+      'Bringing History to Life - Greatest Battles of WW II, Sept 2026',
+      'WW II',
+      'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF',
+      'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.',
+      'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk',
+      'Het ww staat op het plaatje, type het EXACT zo in.',
+      'Wachtwoord:[br]Deze download heeft er geen',
+    ]) {
+      assert.isNull(passwordFromSpot('T', description), description)
+    }
+  })
+})