Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-03 19:59:23 +02:00
1 parent bcb9be5da9
commit be8a3d4580
97 files changed
+4454 -268

No files matched your search

+9
View File
@@ -30,6 +30,15 @@ test.group('content / sniffContentType', () => {
})
test.group('yenc / lenientSize', () => {
test("recognises yEncBin's encrypted .bin container by its header", ({ assert }) => {
// The first 48 bytes of 2141406641.bin (yEncBin@Poster.com).
const data = Buffer.from('54f399d117c492696e4dc4ef5b60c11d7f83e92dfdf23450e79370b048123da81e4c8013091b682fb22478daed2ae981', 'hex')
assert.equal(sniffContentType(data), 'yencbin')
// The second variant (1060238685.bin).
assert.equal(sniffContentType(Buffer.from('54f399e917c492696a4e9fb05b60c31d7f83e92df9f16f5086e3c3d9480d66ef1e48d915', 'hex')), 'yencbin')
assert.isNull(sniffContentType(Buffer.from('54f399d117c492696e4dc4ef5b60c11e7f83e92d', 'hex')))
})
test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => {
const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')])
const declared = data.length - 20
+14
View File
@@ -0,0 +1,14 @@
import { test } from '@japa/runner'
import { articleAtDate } from '#lib/nntp/date_search'
test.group('nntp / articleAtDate', () => {
test('finds the article posted at a time, skipping gaps', async ({ assert }) => {
// One article a second from t0, articles 1,000,000–1,100,000; 1,040,000–1,040,999 expired.
const t0 = Date.UTC(2026, 8, 1)
const dateOf = async (n: bigint) => (n >= 1_040_000n && n < 1_041_000n ? null : t0 + Number(n - 1_000_000n) * 1000)
const at = await articleAtDate(1_000_000n, 1_100_000n, t0 + 60_000 * 1000, dateOf, 100n)
assert.isAtLeast(Number(at), 1_059_900)
assert.isAtMost(Number(at), 1_060_000)
assert.equal(await articleAtDate(1_000_000n, 1_100_000n, t0 - 1, dateOf, 100n), 1_000_000n)
})
})
+52
View File
@@ -0,0 +1,52 @@
import { test } from '@japa/runner'
import { delimitedReader } from '#lib/nntp/delimited_reader'
const CRLF = Buffer.from('\r\n')
const END = Buffer.from('\r\n.\r\n')
async function* chunks(...parts: string[]) {
for (const part of parts) yield Buffer.from(part, 'latin1')
}
test.group('nntp / delimitedReader', () => {
test('reads an empty multi-line reply (XOVER of a range without articles)', async ({ assert }) => {
const reader = delimitedReader(chunks('224 Overview follows\r\n.\r\n', '211 1 2 3 g\r\n'), CRLF)
assert.equal((await reader.next()).value!.toString(), '224 Overview follows')
assert.equal((await reader.next(END)).value!.toString(), '')
assert.equal((await reader.next(CRLF)).value!.toString(), '211 1 2 3 g')
})
test('reads multi-line data, also with the terminator split over chunks', async ({ assert }) => {
for (const split of [1, 2, 3, 4, 5, 6, 7]) {
const reply = '224 ok\r\nline 1\r\n..dotted\r\nline 3\r\n.\r\n200 next\r\n'
const at = reply.indexOf('.\r\n200') - 2 + split - 1
const reader = delimitedReader(chunks(reply.slice(0, at), reply.slice(at)), CRLF)
assert.equal((await reader.next()).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), 'line 1\r\n..dotted\r\nline 3', `split ${split}`)
assert.equal((await reader.next(CRLF)).value!.toString(), '200 next')
}
})
test('reads replies byte by byte', async ({ assert }) => {
const text = '200 welcome\r\n224 ok\r\na\r\n.\r\n224 ok\r\n.\r\n211 x\r\n'
const reader = delimitedReader(chunks(...text.split('')), CRLF)
assert.equal((await reader.next()).value!.toString(), '200 welcome')
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), 'a')
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), '')
assert.equal((await reader.next(CRLF)).value!.toString(), '211 x')
})
test('keeps binary data intact', async ({ assert }) => {
const body = Buffer.from([0, 13, 255, 10, 46, 13, 46, 10, 13])
const source = (async function* () {
yield Buffer.from('222 body\r\n')
yield body
yield Buffer.from('\r\n.\r\n')
})()
const reader = delimitedReader(source, CRLF)
await reader.next()
assert.deepEqual((await reader.next(END)).value, body)
})
})
+46
View File
@@ -0,0 +1,46 @@
import { test } from '@japa/runner'
import { randomBytes } from 'node:crypto'
import { zstdCompressSync, constants } from 'node:zlib'
import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
function tarEntry(name: string, data: Buffer): Buffer {
const header = Buffer.alloc(512)
header.write(name, 0, 'utf8')
header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
header.write('ustar\0', 257, 'latin1')
return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
}
test.group('nicovideo captures', () => {
test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
const dir = '1790720489_nicovideo_jp_watch_sm27965722'
const response = {
data: {
metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
response: {
video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ/After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
owner: { nickname: 'まふまふ' },
tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
},
},
}
const tar = Buffer.concat([
tarEntry(`${dir}/`, Buffer.alloc(0)),
tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
])
const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
// Only the first part of the stream, as in the file's first article.
const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
assert.deepEqual(info, {
id: 'sm27965722',
title: '桜花ニ月夜ト袖シグレ/After the Rain【そらる×まふまふ】',
owner: 'まふまふ',
registeredAt: '2016-01-08T20:00:00+09:00',
tags: ['音楽', 'After_the_Rain'],
})
assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ/After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
})
})
+13 -1
View File
@@ -10,7 +10,7 @@ test.group('nzb / buildNzb', () => {
poster: 'poster@example.invalid',
date: 1_790_000_000_000,
groups: ['alt.binaries.test'],
segments: { '2': { id: 'b@x', size: 700 }, '1': { id: 'a@x', size: 800 } },
segments: [{ number: 2, id: 'b@x', size: 700 }, { number: 1, id: 'a@x', size: 800 }],
},
],
{ title: 'Some.Show.S01E01' }
@@ -53,3 +53,15 @@ test.group('nzb / parseNzb', () => {
assert.isNull(parseNzb('<html>not an nzb</html>'))
})
})
test.group('nzb / stored segment lists', () => {
test('pack and unpack round-trip, in order, also with gaps in the numbering', async ({ assert }) => {
const { packSegments, unpackSegments } = await import('#lib/nzb/segments')
const sequential = Array.from({ length: 300 }, (_, i) => ({ number: i + 1, id: `Ab${i}Cd-1790600369${i}@nyuu`, size: i === 299 ? 1234 : 740000 }))
assert.deepEqual(unpackSegments(packSegments([...sequential].reverse())), sequential)
const gaps = [{ number: 3, id: 'c@x', size: 3 }, { number: 1, id: 'a@x', size: 1 }, { number: 7, id: 'g@x', size: 7 }]
assert.deepEqual(unpackSegments(packSegments(gaps)), [gaps[1], gaps[0], gaps[2]])
assert.deepEqual(unpackSegments(packSegments([])), [])
assert.isBelow(packSegments(sequential).length / sequential.length, 30)
})
})
+71
View File
@@ -0,0 +1,71 @@
import { test } from '@japa/runner'
import {
checkHeader,
listingStart,
parseBsdtarListing,
parseSevenZipStart,
readNumber,
sevenZipVolume,
} from '#lib/sevenzip/index'
// `7z a tiny.7z Tiny.Release` (Tiny.Release/a.txt): an LZMA-encoded header.
const ENCODED = Buffer.from(
'N3q8ryccAATeIVLCfQAAAAAAAAAhAAAAAAAAAGkLW/MBAAtoZWxsbyB3b3JsZAoAAACBMweuD87ysgwHyEN/QbH6/duIttdfpOXRhZbi6t7tycGOUDyW1l2aTE7xmuZ/NAXKgg599VxKYAF1XImsfT9LzHQKaEH6PEZambLnwJM62mXXEJIpaYvhCrHFeQv+jLnYjgJMfwZEcAAAABcGEAEJbQAHCwEAASMDAQEFXQAQAAAMgJYKASTou9wAAA==',
'base64'
)
// The same with -mhc=off: a plain header.
const PLAIN = Buffer.from(
'N3q8ryccAASpRwRsEAAAAAAAAACWAAAAAAAAACTou9wBAAtoZWxsbyB3b3JsZAoAAQQGAAEJEAAHCwEAASEhAQAMDAAICgEtOwivAAAFAg4BgBkJAAAAAAAAAAAAEUEAVABpAG4AeQAuAFIAZQBsAGUAYQBzAGUAAABUAGkAbgB5AC4AUgBlAGwAZQBhAHMAZQAvAGEALgB0AHgAdAAAABkCAAAUEgEA/QYYLBBR3QH9BhgsEFHdARUKAQAQgP1BIIC0gQAA',
'base64'
)
test.group('sevenzip', () => {
test('finds the header at the end from the signature header', ({ assert }) => {
const start = parseSevenZipStart(ENCODED)!
assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })
const header = ENCODED.subarray(start.headerStart, start.archiveSize)
assert.isTrue(checkHeader(start, header))
assert.isFalse(checkHeader(start, Buffer.alloc(33)))
// Encoded: the packed header right after the file data (packPos 16) is needed too.
assert.equal(listingStart(start, header), 48)
const plain = parseSevenZipStart(PLAIN)!
assert.equal(plain.archiveSize, PLAIN.length)
assert.equal(listingStart(plain, PLAIN.subarray(plain.headerStart)), plain.headerStart)
})
test('rejects other data and a damaged signature header', ({ assert }) => {
assert.isNull(parseSevenZipStart(Buffer.from('Rar!\x1a\x07\x01\x00'.padEnd(40, '\0'), 'latin1')))
const damaged = Buffer.from(ENCODED)
damaged[14] ^= 1
assert.isNull(parseSevenZipStart(damaged))
})
test('reads 7z numbers', ({ assert }) => {
assert.deepEqual(readNumber(Buffer.from([0x10]), 0), [16, 1])
assert.deepEqual(readNumber(Buffer.from([0x81, 0x02]), 0), [0x102, 2])
assert.deepEqual(readNumber(Buffer.from([0xc0, 0x34, 0x12]), 0), [0x1234, 3])
assert.deepEqual(readNumber(Buffer.from([0xff, 1, 0, 0, 0, 0, 0, 0, 0]), 0), [1, 9])
})
test('parses bsdtar listings', ({ assert }) => {
const out = [
'drwxrwxr-x 0 0 0 0 Sep 30 19:13 Some.Release.2026/',
'-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release.2026/movie.mkv',
'-rw-rw-r-- 0 0 0 3 Jan 1 2019 Some.Release.2026/some info.nfo',
'',
].join('\n')
assert.deepEqual(parseBsdtarListing(out), [
{ name: 'Some.Release.2026', size: 0, directory: true },
{ name: 'Some.Release.2026/movie.mkv', size: 3000000, directory: false },
{ name: 'Some.Release.2026/some info.nfo', size: 3, directory: false },
])
})
test('numbers volumes', ({ assert }) => {
assert.equal(sevenZipVolume('x.7z.001'), 1)
assert.equal(sevenZipVolume('X.7Z.012'), 12)
assert.equal(sevenZipVolume('x.7z'), 0)
assert.isNull(sevenZipVolume('x.part01.rar'))
})
})
+96 -1
View File
@@ -1,6 +1,15 @@
import { test } from '@japa/runner'
import { deflateRawSync } from 'node:zlib'
import { decodeSpotBinary, parseSpotArticleHeaders, parseSpotHeader, parseSpotXml, verifySpotHeader } from '#lib/spotnet/index'
import {
decodeSpotBinary,
decodeSpotText,
parseSpotArticleHeaders,
parseSpotHeader,
parseSpotXml,
spotCategoryLabels,
verifySpotHeader,
} from '#lib/spotnet/index'
import { ubbToText } from '#lib/newznab/xml'
// A real spot from free.pt (self-signed, key 7), 2026-09-28.
const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
@@ -44,6 +53,62 @@ test.group('spotnet / xml and nzb', () => {
assert.deepEqual(spot.subcats, ['01a09', '01z00'])
})
test('reads the description, tag and image', ({ assert }) => {
const spot = parseSpotXml(
`<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
`<Description>Lily Tomlin[br][br]If you&#39;re &#8216;in&#8217; [url=https://example.com/a]here[/url]</Description>` +
`<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
`<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
)!
assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
assert.equal(spot.tag, 'Bart')
assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
// Older spots link an image on the web.
assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
})
test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
const raw = [
'From: vanker',
' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
'Subject: The postman | The postman |',
'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van &quot;The',
'X-XML: postman&quot;]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
'Date: 01 Feb 2010 02:43:16 GMT',
].join('\r\n')
const { headers, xml } = parseSpotArticleHeaders(raw)
assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
const spot = parseSpotXml(xml)!
assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
})
test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
})
test('names categories as Spotweb does', ({ assert }) => {
assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
head: 'Image',
subcats: [
{ type: 'Type', values: ['Movie'] },
{ type: 'Format', values: ['x264'] },
{ type: 'Source', values: ['Retail'] },
{ type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
{ type: 'Genre', values: ['Science Fiction'] },
],
})
assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
})
test('turns UBB markup into plain text', ({ assert }) => {
assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
})
test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
@@ -68,3 +133,33 @@ test.group('spotnet / moderation', () => {
assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
})
})
test.group('spotnet / passwords in descriptions', () => {
test('finds passwords written in the description or title', async ({ assert }) => {
const { passwordFromSpot } = await import('#lib/spotnet/index')
assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
})
test('ignores sentences about passwords', async ({ assert }) => {
const { passwordFromSpot } = await import('#lib/spotnet/index')
for (const description of [
'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
'[b]--> Geen wachtwoord benodigd [/b]',
'zitten 2 bestanden in en is met een password beschermd.',
'It also removes the need for manual password entry and improves the reliability',
'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
'29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
'Wachtwoord: geen',
'WW: zie nzb',
'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
]) {
assert.isNull(passwordFromSpot('T', description), description)
}
})
})