Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/spotnet/categories.ts

@@ -0,0 +1,333 @@
+/**
+ * Spotnet's category tree, from Spotweb's SpotCategories (English names).
+ * A spot has a head category (0–3) and subcategories per type letter:
+ * `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
+ * `d` genre, `z` kind (movie, series, book, …).
+ */
+export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
+
+/** What each type letter means per head category. */
+export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
+  { a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
+  { a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
+  { a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
+  { a: 'Platform', b: 'Genre', z: 'Type' },
+]
+
+const NAMES: Record<string, Record<number, string>>[] = [
+  {
+    a: {
+      0: 'DivX',
+      1: 'WMV',
+      2: 'MPG',
+      3: 'DVD5',
+      4: 'HD other',
+      5: 'ePub',
+      6: 'Blu-ray',
+      7: 'HD-DVD',
+      8: 'WMVHD',
+      9: 'x264',
+      10: 'DVD9',
+      11: 'PDF',
+      12: 'Bitmap',
+      13: 'Vector',
+      14: '3D',
+      15: 'UHD',
+    },
+    b: {
+      0: 'CAM',
+      1: '(S)VCD',
+      2: 'Promo',
+      3: 'Retail',
+      4: 'TV',
+      6: 'Satellite',
+      7: 'R5',
+      8: 'Telecine',
+      9: 'Telesync',
+      10: 'Scan',
+      11: 'WEB-DL',
+      12: 'WEBRip',
+      13: 'HDRip',
+    },
+    c: {
+      0: 'No subtitles',
+      1: 'Dutch subtitles (external)',
+      2: 'Dutch subtitles (builtin)',
+      3: 'English subtitles (external)',
+      4: 'English subtitles (builtin)',
+      6: 'Dutch subtitles (available)',
+      7: 'English subtitles (available)',
+      10: 'English audio/written',
+      11: 'Dutch audio/written',
+      12: 'German audio/written',
+      13: 'French audio/written',
+      14: 'Spanish audio/written',
+      15: 'Asian audio/written',
+    },
+    d: {
+      0: 'Action',
+      1: 'Adventure',
+      2: 'Animation',
+      3: 'Cabaret',
+      4: 'Comedy',
+      5: 'Crime',
+      6: 'Documentary',
+      7: 'Drama',
+      8: 'Family',
+      9: 'Fantasy',
+      10: 'Arthouse',
+      11: 'Television',
+      12: 'Horror',
+      13: 'Music',
+      14: 'Musical',
+      15: 'Mystery',
+      16: 'Romance',
+      17: 'Science Fiction',
+      18: 'Sport',
+      19: 'Short movie',
+      20: 'Thriller',
+      21: 'War',
+      22: 'Western',
+      23: 'Erotica (hetero)',
+      24: 'Erotica (gay male)',
+      25: 'Erotica (gay female)',
+      26: 'Erotica (bi)',
+      28: 'Asian',
+      29: 'Anime',
+      30: 'Cover',
+      31: 'Comicbook',
+      32: 'Cartoons',
+      33: 'Youth',
+      34: 'Business',
+      35: 'Computer',
+      36: 'Hobby',
+      37: 'Cooking',
+      38: 'Handwork',
+      39: 'Craftwork',
+      40: 'Health',
+      41: 'History',
+      42: 'Psychology',
+      43: 'Newspaper',
+      44: 'Magazine',
+      45: 'Science',
+      46: 'Female',
+      47: 'Religion',
+      48: 'Roman',
+      49: 'Biography',
+      50: 'Detective',
+      51: 'Animals',
+      52: 'Humor',
+      53: 'Travel',
+      54: 'True story',
+      55: 'Non-fiction',
+      56: 'Politics',
+      57: 'Poetry',
+      58: 'Fairy tale',
+      59: 'Technical',
+      60: 'Art',
+      72: 'Bi',
+      73: 'Lesbian',
+      74: 'Homo',
+      75: 'Hetero',
+      76: 'Amature',
+      77: 'Group',
+      78: 'POV',
+      79: 'Solo',
+      80: 'Young',
+      81: 'Soft',
+      82: 'Fetish',
+      83: 'Old',
+      84: 'Fat',
+      85: 'SM',
+      86: 'Rough',
+      87: 'Dark',
+      88: 'Hentai',
+      89: 'Outside',
+    },
+    z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
+  },
+  {
+    a: {
+      0: 'MP3',
+      1: 'WMA',
+      2: 'WAV',
+      3: 'OGG',
+      4: 'EAC',
+      5: 'DTS',
+      6: 'AAC',
+      7: 'APE',
+      8: 'FLAC',
+    },
+    b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
+    c: {
+      0: 'Variable',
+      1: '< 96kbit',
+      2: '96kbit',
+      3: '128kbit',
+      4: '160kbit',
+      5: '192kbit',
+      6: '256kbit',
+      7: '320kbit',
+      8: 'Lossless',
+      9: 'Other',
+    },
+    d: {
+      0: 'Blues',
+      1: 'Compilation',
+      2: 'Cabaret',
+      3: 'Dance',
+      4: 'Diverse',
+      5: 'Hardcore',
+      6: 'World',
+      7: 'Jazz',
+      8: 'Youth',
+      9: 'Classical',
+      10: 'Kleinkunst',
+      11: 'Dutch',
+      12: 'New Age',
+      13: 'Pop',
+      14: 'RnB',
+      15: 'Hiphop',
+      16: 'Reggae',
+      17: 'Religious',
+      18: 'Rock',
+      19: 'Soundtracks',
+      20: 'Other',
+      21: 'Hardstyle',
+      22: 'Asian',
+      23: 'Disco',
+      24: 'Classics',
+      25: 'Metal',
+      26: 'Country',
+      27: 'Dubstep',
+      28: 'Nederhop',
+      29: 'DnB',
+      30: 'Electro',
+      31: 'Folk',
+      32: 'Soul',
+      33: 'Trance',
+      34: 'Balkan',
+      35: 'Techno',
+      36: 'Ambient',
+      37: 'Latin',
+      38: 'Live',
+    },
+    z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
+  },
+  {
+    a: {
+      0: 'Windows',
+      1: 'Macintosh',
+      2: 'Linux',
+      3: 'Playstation',
+      4: 'Playstation 2',
+      5: 'PSP',
+      6: 'Xbox',
+      7: 'Xbox 360',
+      8: 'Gameboy Advance',
+      9: 'Gamecube',
+      10: 'Nintendo DS',
+      11: 'Nintento Wii',
+      12: 'Playstation 3',
+      13: 'Windows Phone',
+      14: 'iOS',
+      15: 'Android',
+      16: 'Nintendo 3DS',
+      17: 'Playstation 4',
+      18: 'XBox 1',
+    },
+    b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
+    c: {
+      0: 'Action',
+      1: 'Adventure',
+      2: 'Strategy',
+      3: 'Roleplaying',
+      4: 'Simulation',
+      5: 'Race',
+      6: 'Flying',
+      7: 'Shooter',
+      8: 'Platform',
+      9: 'Sport',
+      10: 'Child/youth',
+      11: 'Puzzle',
+      12: 'Other',
+      13: 'Boardgame',
+      14: 'Cards',
+      15: 'Education',
+      16: 'Music',
+      17: 'Family',
+    },
+  },
+  {
+    a: {
+      0: 'Windows',
+      1: 'Macintosh',
+      2: 'Linux',
+      3: 'OS/2',
+      4: 'Windows Phone',
+      5: 'Navigation systems',
+      6: 'iOS',
+      7: 'Android',
+    },
+    b: {
+      0: 'Audio',
+      1: 'Video',
+      2: 'Graphics',
+      3: 'CD/DVD Tools',
+      4: 'Media players',
+      5: 'Rippers &amp; Encoders',
+      6: 'Plugins',
+      7: 'Database tools',
+      8: 'Email software',
+      9: 'Photo',
+      10: 'Screensavers',
+      11: 'Skin software',
+      12: 'Drivers',
+      13: 'Browsers',
+      14: 'Download managers',
+      15: 'Download',
+      16: 'Usenet software',
+      17: 'RSS Readers',
+      18: 'FTP software',
+      19: 'Firewalls',
+      20: 'Antivirus software',
+      21: 'Antispyware software',
+      22: 'Optimization software',
+      23: 'Security software',
+      24: 'System software',
+      25: 'Other',
+      26: 'Educational',
+      27: 'Office',
+      28: 'Internet',
+      29: 'Communication',
+      30: 'Development',
+      31: 'Spotnet',
+    },
+  },
+]
+
+export interface SpotCategoryLabels {
+  head: string | null
+  /** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
+  subcats: { type: string; values: string[] }[]
+}
+
+/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
+export function spotCategoryLabels(
+  category: number | null,
+  subcats: Record<string, number[]> | null
+): SpotCategoryLabels {
+  if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
+  const labels: SpotCategoryLabels['subcats'] = []
+  for (const letter of ['z', 'a', 'b', 'c', 'd']) {
+    const values = [
+      ...new Set(
+        (subcats?.[letter] ?? [])
+          .map((n) => NAMES[category][letter]?.[n])
+          .filter((v): v is string => Boolean(v))
+      ),
+    ]
+    if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
+  }
+  return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
+}

lib/spotnet/index.ts

@@ -4,4 +4,6 @@
  * See CONCEPT.md (D29).
  */
 export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
-export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js'
+export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js'
+export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js'
+export { passwordFromSpot } from './password.js'

lib/spotnet/password.ts

@@ -0,0 +1,50 @@
+/**
+ * Archive passwords that spotters put in the title or description instead of
+ * the NZB's `<meta type="password">`:
+ *   `Title{{secret}}` (the newznab/SABnzbd convention),
+ *   `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
+ *   `WW FC0401` (abbreviation without separator: only with a digit in it).
+ * The value must be the rest of its line and a single token, so sentences
+ * such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
+ * "met een password beschermd" don't count.
+ */
+const KEYWORD = String.raw`wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw`
+const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])(?:${KEYWORD})\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
+const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
+const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
+const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
+
+/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */
+function lines(description: string): string[] {
+  return description
+    .replace(/\[br\]|<br\s*\/?>/gi, '\n')
+    .replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+}
+
+function clean(value: string): string | null {
+  const token = value.trim().replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
+  if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
+  return token
+}
+
+export function passwordFromSpot(title: string, description?: string | null): string | null {
+  const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
+  if (braces && clean(braces[1])) return clean(braces[1])
+  for (const line of lines(description ?? '')) {
+    if (!line || NO_PASSWORD.test(line)) continue
+    const separated = WITH_SEPARATOR.exec(line)
+    if (separated) {
+      const value = clean(separated[1])
+      if (value) return value
+      continue
+    }
+    const abbreviated = ABBREVIATION.exec(line)
+    if (abbreviated && /\d/.test(abbreviated[1])) {
+      const value = clean(abbreviated[1])
+      if (value) return value
+    }
+  }
+  return null
+}

lib/spotnet/xml.ts

@@ -6,18 +6,25 @@ export interface SpotXml {
   subcats: string[]
   size: number
   website: string
+  /** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
+  description: string
+  tag: string
+  /** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
+  image: SpotImage | null
   /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
   nzbSegments: string[]
   created: number | null
 }
 
+export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
+
 const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
 
+/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
 function text(value: string | undefined): string {
   if (value === undefined) return ''
   const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
-  if (cdata) return cdata[1]
-  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+  return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
     if (entity[0] === '#') {
       const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
       return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
@@ -37,6 +44,9 @@ export function parseSpotXml(xml: string): SpotXml | null {
   const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
   const nzb = element(posting, 'NZB') ?? ''
   return {
+    description: text(element(posting, 'Description')).trim(),
+    tag: text(element(posting, 'Tag')).trim(),
+    image: parseImage(posting),
     title: text(element(posting, 'Title')).trim(),
     // Newer keys number categories from 1.
     category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
@@ -48,15 +58,52 @@ export function parseSpotXml(xml: string): SpotXml | null {
   }
 }
 
-/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
+function parseImage(posting: string): SpotImage | null {
+  const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
+  if (!match) return null
+  const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
+  if (segments.length > 0) {
+    const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
+    return { segments, width: size('Width'), height: size('Height') }
+  }
+  const url = text(match[2]).trim()
+  return /^https?:\/\//i.test(url) ? { url } : null
+}
+
+/**
+ * The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
+ * the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
+ */
+export function decodeSpotText(latin1: string): string {
+  const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
+  return utf8.includes('\uFFFD') ? latin1 : utf8
+}
+
+/**
+ * Header lines of a raw HEAD response; X-XML (split over many lines) joined.
+ * Folded headers are unfolded: old (key 1) spots put the From address on a
+ * continuation line.
+ */
 export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
   const headers: Record<string, string> = {}
   let xml = ''
+  let last: string | null = null
+  let lastIsFirst = false
   for (const line of raw.split(/\r?\n/)) {
+    if (/^[ \t]/.test(line) && last) {
+      if (last === 'x-xml') xml += line
+      else if (lastIsFirst) headers[last] += line
+      continue
+    }
     const colon = line.indexOf(':')
-    if (colon <= 0) continue
+    if (colon <= 0) {
+      last = null
+      continue
+    }
     const name = line.slice(0, colon).toLowerCase()
     const value = line.slice(colon + 1).replace(/^ /, '')
+    last = name
+    lastIsFirst = !(name in headers)
     if (name === 'x-xml') xml += value
     else headers[name] ??= value
   }