Spots: the spotter and the spot inspector, passwords kept apart
Two processes, one NNTP connection each (the coordinator 40 -> 38): - spots:read (service spotter): free.pt's overviews after the last article seen, each spot's full header verified (lib/spotnet from voyager 1, as Spotweb), its description, website and NZB articles; the password its title or text names; delete/dispose messages applied. - spots:inspect (service spot-inspector): each spot's compressed NZB read, its files listed (spot_files), each matched to the header pass's release by the same release identity in one of its groups within 48 h (release_spots); unmatched spots matched again hourly, after 24 h a release of their own (source spot) with the spot's NZB. - release_passwords: per release and password, its source (nzb, spot-title, spot-text) and spot; the catalog keeps only passworded. XoverClient.head(); PROBE_CONNECTIONS default 10. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
3fd08b598a
commit
7df5a638ca
20 files changed
+1782
-8
No files matched your search
@@ -246,6 +246,20 @@ export class XoverClient {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* An article's header by message-id, raw (as body(): "\r\n.\r\n" cut off). Spot headers carry the spot's XML
|
||||
* (X-XML), signed (lib/spotnet). 430 (no such article) throws MissingArticle.
|
||||
*/
|
||||
async head(messageId: string): Promise<Buffer> {
|
||||
return this.timed(async () => {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
const reply = await this.command(`HEAD ${id}`)
|
||||
if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply)
|
||||
await this.expect(reply, ['221'])
|
||||
return this.rawBody()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Several bodies, pipelined: every BODY is sent at once and the replies read in order, so one round trip
|
||||
* covers them all (one at a time, a connection waited ~0.1–2 s per article: 3–10 articles/s on 6; LOG.md
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
/**
|
||||
* Spotnet's category tree, from Spotweb's SpotCategories (English names).
|
||||
* A spot has a head category (0–3) and subcategories per type letter:
|
||||
* `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
|
||||
* `d` genre, `z` kind (movie, series, book, …).
|
||||
*/
|
||||
export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
|
||||
|
||||
/** What each type letter means per head category. */
|
||||
export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
|
||||
{ a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Genre', z: 'Type' },
|
||||
]
|
||||
|
||||
const NAMES: Record<string, Record<number, string>>[] = [
|
||||
{
|
||||
a: {
|
||||
0: 'DivX',
|
||||
1: 'WMV',
|
||||
2: 'MPG',
|
||||
3: 'DVD5',
|
||||
4: 'HD other',
|
||||
5: 'ePub',
|
||||
6: 'Blu-ray',
|
||||
7: 'HD-DVD',
|
||||
8: 'WMVHD',
|
||||
9: 'x264',
|
||||
10: 'DVD9',
|
||||
11: 'PDF',
|
||||
12: 'Bitmap',
|
||||
13: 'Vector',
|
||||
14: '3D',
|
||||
15: 'UHD',
|
||||
},
|
||||
b: {
|
||||
0: 'CAM',
|
||||
1: '(S)VCD',
|
||||
2: 'Promo',
|
||||
3: 'Retail',
|
||||
4: 'TV',
|
||||
6: 'Satellite',
|
||||
7: 'R5',
|
||||
8: 'Telecine',
|
||||
9: 'Telesync',
|
||||
10: 'Scan',
|
||||
11: 'WEB-DL',
|
||||
12: 'WEBRip',
|
||||
13: 'HDRip',
|
||||
},
|
||||
c: {
|
||||
0: 'No subtitles',
|
||||
1: 'Dutch subtitles (external)',
|
||||
2: 'Dutch subtitles (builtin)',
|
||||
3: 'English subtitles (external)',
|
||||
4: 'English subtitles (builtin)',
|
||||
6: 'Dutch subtitles (available)',
|
||||
7: 'English subtitles (available)',
|
||||
10: 'English audio/written',
|
||||
11: 'Dutch audio/written',
|
||||
12: 'German audio/written',
|
||||
13: 'French audio/written',
|
||||
14: 'Spanish audio/written',
|
||||
15: 'Asian audio/written',
|
||||
},
|
||||
d: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Animation',
|
||||
3: 'Cabaret',
|
||||
4: 'Comedy',
|
||||
5: 'Crime',
|
||||
6: 'Documentary',
|
||||
7: 'Drama',
|
||||
8: 'Family',
|
||||
9: 'Fantasy',
|
||||
10: 'Arthouse',
|
||||
11: 'Television',
|
||||
12: 'Horror',
|
||||
13: 'Music',
|
||||
14: 'Musical',
|
||||
15: 'Mystery',
|
||||
16: 'Romance',
|
||||
17: 'Science Fiction',
|
||||
18: 'Sport',
|
||||
19: 'Short movie',
|
||||
20: 'Thriller',
|
||||
21: 'War',
|
||||
22: 'Western',
|
||||
23: 'Erotica (hetero)',
|
||||
24: 'Erotica (gay male)',
|
||||
25: 'Erotica (gay female)',
|
||||
26: 'Erotica (bi)',
|
||||
28: 'Asian',
|
||||
29: 'Anime',
|
||||
30: 'Cover',
|
||||
31: 'Comicbook',
|
||||
32: 'Cartoons',
|
||||
33: 'Youth',
|
||||
34: 'Business',
|
||||
35: 'Computer',
|
||||
36: 'Hobby',
|
||||
37: 'Cooking',
|
||||
38: 'Handwork',
|
||||
39: 'Craftwork',
|
||||
40: 'Health',
|
||||
41: 'History',
|
||||
42: 'Psychology',
|
||||
43: 'Newspaper',
|
||||
44: 'Magazine',
|
||||
45: 'Science',
|
||||
46: 'Female',
|
||||
47: 'Religion',
|
||||
48: 'Roman',
|
||||
49: 'Biography',
|
||||
50: 'Detective',
|
||||
51: 'Animals',
|
||||
52: 'Humor',
|
||||
53: 'Travel',
|
||||
54: 'True story',
|
||||
55: 'Non-fiction',
|
||||
56: 'Politics',
|
||||
57: 'Poetry',
|
||||
58: 'Fairy tale',
|
||||
59: 'Technical',
|
||||
60: 'Art',
|
||||
72: 'Bi',
|
||||
73: 'Lesbian',
|
||||
74: 'Homo',
|
||||
75: 'Hetero',
|
||||
76: 'Amature',
|
||||
77: 'Group',
|
||||
78: 'POV',
|
||||
79: 'Solo',
|
||||
80: 'Young',
|
||||
81: 'Soft',
|
||||
82: 'Fetish',
|
||||
83: 'Old',
|
||||
84: 'Fat',
|
||||
85: 'SM',
|
||||
86: 'Rough',
|
||||
87: 'Dark',
|
||||
88: 'Hentai',
|
||||
89: 'Outside',
|
||||
},
|
||||
z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'MP3',
|
||||
1: 'WMA',
|
||||
2: 'WAV',
|
||||
3: 'OGG',
|
||||
4: 'EAC',
|
||||
5: 'DTS',
|
||||
6: 'AAC',
|
||||
7: 'APE',
|
||||
8: 'FLAC',
|
||||
},
|
||||
b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
|
||||
c: {
|
||||
0: 'Variable',
|
||||
1: '< 96kbit',
|
||||
2: '96kbit',
|
||||
3: '128kbit',
|
||||
4: '160kbit',
|
||||
5: '192kbit',
|
||||
6: '256kbit',
|
||||
7: '320kbit',
|
||||
8: 'Lossless',
|
||||
9: 'Other',
|
||||
},
|
||||
d: {
|
||||
0: 'Blues',
|
||||
1: 'Compilation',
|
||||
2: 'Cabaret',
|
||||
3: 'Dance',
|
||||
4: 'Diverse',
|
||||
5: 'Hardcore',
|
||||
6: 'World',
|
||||
7: 'Jazz',
|
||||
8: 'Youth',
|
||||
9: 'Classical',
|
||||
10: 'Kleinkunst',
|
||||
11: 'Dutch',
|
||||
12: 'New Age',
|
||||
13: 'Pop',
|
||||
14: 'RnB',
|
||||
15: 'Hiphop',
|
||||
16: 'Reggae',
|
||||
17: 'Religious',
|
||||
18: 'Rock',
|
||||
19: 'Soundtracks',
|
||||
20: 'Other',
|
||||
21: 'Hardstyle',
|
||||
22: 'Asian',
|
||||
23: 'Disco',
|
||||
24: 'Classics',
|
||||
25: 'Metal',
|
||||
26: 'Country',
|
||||
27: 'Dubstep',
|
||||
28: 'Nederhop',
|
||||
29: 'DnB',
|
||||
30: 'Electro',
|
||||
31: 'Folk',
|
||||
32: 'Soul',
|
||||
33: 'Trance',
|
||||
34: 'Balkan',
|
||||
35: 'Techno',
|
||||
36: 'Ambient',
|
||||
37: 'Latin',
|
||||
38: 'Live',
|
||||
},
|
||||
z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'Playstation',
|
||||
4: 'Playstation 2',
|
||||
5: 'PSP',
|
||||
6: 'Xbox',
|
||||
7: 'Xbox 360',
|
||||
8: 'Gameboy Advance',
|
||||
9: 'Gamecube',
|
||||
10: 'Nintendo DS',
|
||||
11: 'Nintento Wii',
|
||||
12: 'Playstation 3',
|
||||
13: 'Windows Phone',
|
||||
14: 'iOS',
|
||||
15: 'Android',
|
||||
16: 'Nintendo 3DS',
|
||||
17: 'Playstation 4',
|
||||
18: 'XBox 1',
|
||||
},
|
||||
b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
|
||||
c: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Strategy',
|
||||
3: 'Roleplaying',
|
||||
4: 'Simulation',
|
||||
5: 'Race',
|
||||
6: 'Flying',
|
||||
7: 'Shooter',
|
||||
8: 'Platform',
|
||||
9: 'Sport',
|
||||
10: 'Child/youth',
|
||||
11: 'Puzzle',
|
||||
12: 'Other',
|
||||
13: 'Boardgame',
|
||||
14: 'Cards',
|
||||
15: 'Education',
|
||||
16: 'Music',
|
||||
17: 'Family',
|
||||
},
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'OS/2',
|
||||
4: 'Windows Phone',
|
||||
5: 'Navigation systems',
|
||||
6: 'iOS',
|
||||
7: 'Android',
|
||||
},
|
||||
b: {
|
||||
0: 'Audio',
|
||||
1: 'Video',
|
||||
2: 'Graphics',
|
||||
3: 'CD/DVD Tools',
|
||||
4: 'Media players',
|
||||
5: 'Rippers & Encoders',
|
||||
6: 'Plugins',
|
||||
7: 'Database tools',
|
||||
8: 'Email software',
|
||||
9: 'Photo',
|
||||
10: 'Screensavers',
|
||||
11: 'Skin software',
|
||||
12: 'Drivers',
|
||||
13: 'Browsers',
|
||||
14: 'Download managers',
|
||||
15: 'Download',
|
||||
16: 'Usenet software',
|
||||
17: 'RSS Readers',
|
||||
18: 'FTP software',
|
||||
19: 'Firewalls',
|
||||
20: 'Antivirus software',
|
||||
21: 'Antispyware software',
|
||||
22: 'Optimization software',
|
||||
23: 'Security software',
|
||||
24: 'System software',
|
||||
25: 'Other',
|
||||
26: 'Educational',
|
||||
27: 'Office',
|
||||
28: 'Internet',
|
||||
29: 'Communication',
|
||||
30: 'Development',
|
||||
31: 'Spotnet',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
export interface SpotCategoryLabels {
|
||||
head: string | null
|
||||
/** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
|
||||
subcats: { type: string; values: string[] }[]
|
||||
}
|
||||
|
||||
/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
|
||||
export function spotCategoryLabels(
|
||||
category: number | null,
|
||||
subcats: Record<string, number[]> | null
|
||||
): SpotCategoryLabels {
|
||||
if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
|
||||
const labels: SpotCategoryLabels['subcats'] = []
|
||||
for (const letter of ['z', 'a', 'b', 'c', 'd']) {
|
||||
const values = [
|
||||
...new Set(
|
||||
(subcats?.[letter] ?? [])
|
||||
.map((n) => NAMES[category][letter]?.[n])
|
||||
.filter((v): v is string => Boolean(v))
|
||||
),
|
||||
]
|
||||
if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
|
||||
}
|
||||
return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { createHash, createPublicKey, verify } from 'node:crypto'
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader.
|
||||
*
|
||||
* From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]>
|
||||
* Subject: Title | Tag
|
||||
*/
|
||||
export interface SpotHeader {
|
||||
messageId: string
|
||||
poster: string
|
||||
title: string
|
||||
tag: string
|
||||
/** 0 image/video, 1 sound, 2 games, 3 applications. */
|
||||
category: number
|
||||
keyId: number
|
||||
/** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */
|
||||
subcats: Record<string, number[]>
|
||||
size: number
|
||||
/** Unix seconds. */
|
||||
stamp: number
|
||||
/** Everything after the @. */
|
||||
header: string
|
||||
headerSign: string
|
||||
selfSignedPubKey: string
|
||||
userSignature?: string
|
||||
}
|
||||
|
||||
/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */
|
||||
export function spotUnprepareBase64(input: string): string {
|
||||
const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input
|
||||
return padded.replace(/-s/g, '/').replace(/-p/g, '+')
|
||||
}
|
||||
|
||||
/** Parses the From/Subject of a spot. Null when it isn't one. */
|
||||
export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null {
|
||||
const lt = from.indexOf('<')
|
||||
if (lt === -1) return null
|
||||
const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined)
|
||||
const at = address.lastIndexOf('@')
|
||||
if (at === -1) return null
|
||||
const header = address.slice(at + 1)
|
||||
const [pubKey, userSignature] = address.slice(0, at).split('.')
|
||||
|
||||
const fields = header.split('.')
|
||||
if (fields.length < 6) return null
|
||||
const category = Number.parseInt(fields[0][0], 10) - 1
|
||||
const keyId = Number.parseInt(fields[0][1], 10)
|
||||
if (!(category >= 0) || !(keyId >= 0)) return null
|
||||
|
||||
// a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits.
|
||||
const subcats: Record<string, number[]> = {}
|
||||
for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) {
|
||||
if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10))
|
||||
}
|
||||
|
||||
// Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field.
|
||||
const parts = subject.split('|')
|
||||
let title: string
|
||||
let tag = ''
|
||||
if (keyId !== 1) {
|
||||
title = parts[0].trim()
|
||||
tag = parts.length > 1 ? parts[1].trim() : ''
|
||||
} else {
|
||||
tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim()
|
||||
title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : ''
|
||||
}
|
||||
|
||||
const parsedDate = date ? Date.parse(date) : Number.NaN
|
||||
return {
|
||||
messageId: messageId.replace(/^<|>$/g, ''),
|
||||
poster: from.slice(0, lt).trim(),
|
||||
title,
|
||||
tag,
|
||||
category,
|
||||
keyId,
|
||||
subcats,
|
||||
size: Number.parseInt(fields[1], 10) || 0,
|
||||
stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000),
|
||||
header,
|
||||
headerSign: fields[fields.length - 1],
|
||||
selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''),
|
||||
userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */
|
||||
export const SPOTNET_KEYS: Record<number, string> = {
|
||||
2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P',
|
||||
3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat',
|
||||
4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9',
|
||||
}
|
||||
|
||||
const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url')
|
||||
|
||||
/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */
|
||||
export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean {
|
||||
try {
|
||||
const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' })
|
||||
const data = typeof message === 'string' ? toBytes(message) : message
|
||||
return verify('sha1', data, key, Buffer.from(signatureB64, 'base64'))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Spots are signed over their ISO-8859-1 bytes. */
|
||||
function toBytes(text: string): Buffer {
|
||||
return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8')
|
||||
}
|
||||
|
||||
/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('<msgid>') starts with 0000. */
|
||||
function hashcash(messageId: string): boolean {
|
||||
return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000')
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies a spot header like Spotweb does: key 7 (self-signed) needs a
|
||||
* hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also
|
||||
* carry a user signature must have it match the message id.
|
||||
*/
|
||||
export function verifySpotHeader(spot: SpotHeader, keys: Record<number, string> = SPOTNET_KEYS): boolean {
|
||||
if (!spot.title || !spot.poster) return false
|
||||
// Old key 1 spots from before 2011 weren't signed.
|
||||
if (spot.keyId === 1 && spot.stamp <= 1293870080) return true
|
||||
if (!spot.headerSign) return false
|
||||
|
||||
const signature = spotUnprepareBase64(spot.headerSign)
|
||||
const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster
|
||||
let verified: boolean
|
||||
if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) {
|
||||
// Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb.
|
||||
verified = hashcash(spot.messageId)
|
||||
} else {
|
||||
verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId])
|
||||
}
|
||||
if (verified && spot.userSignature && spot.selfSignedPubKey) {
|
||||
verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey)
|
||||
}
|
||||
return verified
|
||||
}
|
||||
|
||||
/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */
|
||||
export function spotterId(pubKeyB64: string): string {
|
||||
if (!pubKeyB64) return ''
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0)
|
||||
return crc.toString('base64').replace(/[/+=]/g, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Moderation: key-2 spots titled "delete|dispose|remove <message-id>".
|
||||
* From Spotnet's key they are moderator actions; with the poster's own key
|
||||
* (a spotter id) a personal dispose of their own spot.
|
||||
*/
|
||||
export function moderationTarget(spot: Pick<SpotHeader, 'keyId' | 'title'>): string | null {
|
||||
if (spot.keyId !== 2) return null
|
||||
const [command, target] = spot.title.split(' ')
|
||||
if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null
|
||||
return target.replace(/^<|>$/g, '')
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Spotnet (free.pt) spots, independent of AdonisJS: header parsing and
|
||||
* signature checks after Spotweb, the spot XML and its compressed NZB.
|
||||
* See CONCEPT.md (D29).
|
||||
*/
|
||||
export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
|
||||
export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js'
|
||||
export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js'
|
||||
export { passwordFromSpot } from './password.js'
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Archive passwords that spotters put in the title or description instead of
|
||||
* the NZB's `<meta type="password">`:
|
||||
* `Title{{secret}}` (the newznab/SABnzbd convention),
|
||||
* `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
|
||||
* a few words between keyword and separator (`Het wachtwoord voor de nzb is: dw0067`,
|
||||
* `ww in nzb: X`, `Wachtwoord inbegrepen: wysiwyg`),
|
||||
* the value on the next line (`WW:` / `tijdloos`),
|
||||
* `wachtwoord is arn2`, `… voorzien van een wachtwoord. Deze is: X`,
|
||||
* `WW FC0401` (abbreviation without separator: with a digit in it, or as the whole line).
|
||||
* The value must be the rest of its line and a single token, so sentences
|
||||
* such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
|
||||
* "met een password beschermd" don't count (LOG.md 2026-10-08 (10)).
|
||||
*/
|
||||
const KEYWORD = String.raw`(?:wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw)(?![a-z])`
|
||||
const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}(?:\s+[a-zà-ÿ]+){0,3}?\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
|
||||
const IS_VALUE = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}\s+is\s+(\S+)\s*$`, 'i')
|
||||
/** "Deze is voorzien van een wachtwoord. Deze is: X": the value in the next sentence. */
|
||||
const NEXT_SENTENCE = /(?:^|[.!?]\s*)(?:deze|dit|het|die|this|it)\s+is\s*[:=]\s*(\S+)\s*$/i
|
||||
const MENTIONS = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}`, 'i')
|
||||
const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
|
||||
/** "WW carrie" as a line of its own; "WW II" is the war. */
|
||||
const WHOLE_LINE_ABBREVIATION = /^(?:ww|pw)\s+(\S+)$/i
|
||||
const NOT_PASSWORD_TOKEN = /^(?:i{1,3}|iv|v|[12])$/i
|
||||
const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
|
||||
const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
|
||||
|
||||
/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */
|
||||
function lines(description: string): string[] {
|
||||
return description
|
||||
.replace(/\[br\]|<br\s*\/?>/gi, '\n')
|
||||
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '')
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
}
|
||||
|
||||
function clean(value: string): string | null {
|
||||
const token = value
|
||||
.trim()
|
||||
// "Wachtwoord : yakowyke !!"
|
||||
.replace(/^[!*~#]+\s*|\s*[!*~#]+$/g, '')
|
||||
.replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
|
||||
if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
|
||||
return token
|
||||
}
|
||||
|
||||
export function passwordFromSpot(title: string, description?: string | null): string | null {
|
||||
const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
|
||||
if (braces && clean(braces[1])) return clean(braces[1])
|
||||
const all = lines(description ?? '')
|
||||
for (const [i, line] of all.entries()) {
|
||||
if (!line || NO_PASSWORD.test(line)) continue
|
||||
const separated = WITH_SEPARATOR.exec(line)
|
||||
if (separated) {
|
||||
const value = separated[1].trim() ? clean(separated[1]) : nextLineValue(all, i)
|
||||
if (value) return value
|
||||
continue
|
||||
}
|
||||
const is = IS_VALUE.exec(line)
|
||||
if (is) {
|
||||
const value = clean(is[1])
|
||||
if (value) return value
|
||||
}
|
||||
const sentence = MENTIONS.test(line) ? NEXT_SENTENCE.exec(line) : null
|
||||
if (sentence) {
|
||||
const value = clean(sentence[1])
|
||||
if (value) return value
|
||||
}
|
||||
const whole = WHOLE_LINE_ABBREVIATION.exec(line)
|
||||
const abbreviated = whole ?? ABBREVIATION.exec(line)
|
||||
if (abbreviated && (whole ? !NOT_PASSWORD_TOKEN.test(abbreviated[1]) : /\d/.test(abbreviated[1]))) {
|
||||
const value = clean(abbreviated[1])
|
||||
if (value) return value
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** `WW:` with the value alone on the next non-empty line. */
|
||||
function nextLineValue(all: string[], i: number): string | null {
|
||||
const next = all.slice(i + 1).find((line) => line !== '')
|
||||
return next ? clean(next) : null
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { inflateRawSync } from 'node:zlib'
|
||||
|
||||
export interface SpotXml {
|
||||
title: string
|
||||
category: number | null
|
||||
subcats: string[]
|
||||
size: number
|
||||
website: string
|
||||
/** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
|
||||
description: string
|
||||
tag: string
|
||||
/** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
|
||||
image: SpotImage | null
|
||||
/** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
|
||||
nzbSegments: string[]
|
||||
created: number | null
|
||||
}
|
||||
|
||||
export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
|
||||
function text(value: string | undefined): string {
|
||||
if (value === undefined) return ''
|
||||
const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
|
||||
return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
}
|
||||
return ENTITIES[entity.toLowerCase()] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i').exec(xml)?.[1]
|
||||
|
||||
/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */
|
||||
export function parseSpotXml(xml: string): SpotXml | null {
|
||||
const posting = element(xml, 'Posting')
|
||||
if (!posting) return null
|
||||
const categoryXml = element(posting, 'Category') ?? ''
|
||||
const key = Number.parseInt(element(posting, 'Key') ?? '', 10)
|
||||
const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
|
||||
const nzb = element(posting, 'NZB') ?? ''
|
||||
return {
|
||||
description: text(element(posting, 'Description')).trim(),
|
||||
tag: text(element(posting, 'Tag')).trim(),
|
||||
image: parseImage(posting),
|
||||
title: text(element(posting, 'Title')).trim(),
|
||||
// Newer keys number categories from 1.
|
||||
category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
|
||||
subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()),
|
||||
size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0,
|
||||
website: text(element(posting, 'Website')).trim(),
|
||||
nzbSegments: [...nzb.matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean),
|
||||
created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null,
|
||||
}
|
||||
}
|
||||
|
||||
function parseImage(posting: string): SpotImage | null {
|
||||
const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
|
||||
if (!match) return null
|
||||
const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
|
||||
if (segments.length > 0) {
|
||||
const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
|
||||
return { segments, width: size('Width'), height: size('Height') }
|
||||
}
|
||||
const url = text(match[2]).trim()
|
||||
return /^https?:\/\//i.test(url) ? { url } : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
|
||||
* the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
|
||||
*/
|
||||
export function decodeSpotText(latin1: string): string {
|
||||
const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
|
||||
return utf8.includes('\uFFFD') ? latin1 : utf8
|
||||
}
|
||||
|
||||
/**
|
||||
* Header lines of a raw HEAD response; X-XML (split over many lines) joined.
|
||||
* Folded headers are unfolded: old (key 1) spots put the From address on a
|
||||
* continuation line.
|
||||
*/
|
||||
export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
|
||||
const headers: Record<string, string> = {}
|
||||
let xml = ''
|
||||
let last: string | null = null
|
||||
let lastIsFirst = false
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
if (/^[ \t]/.test(line) && last) {
|
||||
if (last === 'x-xml') xml += line
|
||||
else if (lastIsFirst) headers[last] += line
|
||||
continue
|
||||
}
|
||||
const colon = line.indexOf(':')
|
||||
if (colon <= 0) {
|
||||
last = null
|
||||
continue
|
||||
}
|
||||
const name = line.slice(0, colon).toLowerCase()
|
||||
const value = line.slice(colon + 1).replace(/^ /, '')
|
||||
last = name
|
||||
lastIsFirst = !(name in headers)
|
||||
if (name === 'x-xml') xml += value
|
||||
else headers[name] ??= value
|
||||
}
|
||||
return { headers, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Spot binaries (the NZB, images): article bodies joined without line
|
||||
* breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw
|
||||
* deflate (PHP gzinflate).
|
||||
*/
|
||||
export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer {
|
||||
const joined = bodies
|
||||
.map((body) =>
|
||||
body
|
||||
.toString('latin1')
|
||||
.split('\r\n')
|
||||
// NNTP dot-stuffing.
|
||||
.map((line) => (line.startsWith('..') ? line.slice(1) : line))
|
||||
.join('')
|
||||
)
|
||||
.join('')
|
||||
const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1')
|
||||
return compressed ? inflateRawSync(unescaped) : unescaped
|
||||
}
|
||||
Reference in new issue
Block a user