Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
@@ -0,0 +1,13 @@
|
||||
import { sevenZipVolume } from '../sevenzip/index.js'
|
||||
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
@@ -2,3 +2,6 @@
|
||||
* Content recognition by magic bytes, independent of AdonisJS.
|
||||
*/
|
||||
export { sniffContentType, type ContentType } from './sniff.js'
|
||||
export { isArchiveStart } from './archive_names.js'
|
||||
export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
|
||||
export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'
|
||||
@@ -0,0 +1,76 @@
|
||||
import { createZstdDecompress } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Captures of niconico videos (`<unixtime>_nicovideo_jp_watch_sm<id>.tar.zst`,
|
||||
* one poster, ~190k files on homelab). The tar's first entry after its
|
||||
* directory is `server_response.metadata`: the site's API response for the
|
||||
* watch page, 30–70 KB, so the first article of the file holds it.
|
||||
*/
|
||||
export const NICOVIDEO_FILE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)\.tar\.zst$/i
|
||||
export const NICOVIDEO_RELEASE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)$/i
|
||||
|
||||
export interface NicovideoInfo {
|
||||
id: string
|
||||
title: string
|
||||
owner: string | null
|
||||
registeredAt: string | null
|
||||
tags: string[]
|
||||
}
|
||||
|
||||
/** As much of a zstd stream as `data` (a prefix of it) decompresses to. */
|
||||
export function zstdPrefix(data: Buffer, maxBytes = 4 * 1024 * 1024): Promise<Buffer> {
|
||||
return new Promise((resolve) => {
|
||||
const out: Buffer[] = []
|
||||
let length = 0
|
||||
const z = createZstdDecompress()
|
||||
const done = () => resolve(Buffer.concat(out))
|
||||
z.on('data', (chunk: Buffer) => {
|
||||
out.push(chunk)
|
||||
length += chunk.length
|
||||
if (length >= maxBytes) z.destroy()
|
||||
})
|
||||
z.on('error', done)
|
||||
z.on('close', done)
|
||||
z.on('end', done)
|
||||
z.end(data)
|
||||
})
|
||||
}
|
||||
|
||||
/** Entries at the start of a (possibly truncated) ustar archive. */
|
||||
export function* tarEntries(tar: Buffer): Generator<{ name: string; data: Buffer }> {
|
||||
let offset = 0
|
||||
while (offset + 512 <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + 512)
|
||||
const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '')
|
||||
if (!name) return
|
||||
const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '')
|
||||
const size = Number.parseInt(header.subarray(124, 136).toString('latin1').replace(/\0.*$/s, '').trim() || '0', 8) || 0
|
||||
yield { name: prefix ? `${prefix}/${name}` : name, data: tar.subarray(offset + 512, offset + 512 + size) }
|
||||
offset += 512 + Math.ceil(size / 512) * 512
|
||||
}
|
||||
}
|
||||
|
||||
/** Title, uploader and tags from the start of a capture, or null when they aren't in it. */
|
||||
export function nicovideoInfo(tar: Buffer): NicovideoInfo | null {
|
||||
for (const entry of tarEntries(tar)) {
|
||||
if (!entry.name.endsWith('/server_response.metadata')) continue
|
||||
try {
|
||||
const text = entry.data.toString('utf8')
|
||||
const json = JSON.parse(text.slice(text.indexOf('{')))
|
||||
const video = json?.data?.response?.video
|
||||
if (!video?.title) return null
|
||||
const owner = json.data.response.owner?.nickname ?? json.data.response.channel?.name ?? json.data.metadata?.jsonLds?.[0]?.author?.name ?? null
|
||||
const tags = (json.data.response.tag?.items ?? []).map((t: any) => String(t.name)).filter(Boolean)
|
||||
return { id: String(video.id ?? ''), title: String(video.title).trim(), owner: owner ? String(owner).trim() : null, registeredAt: video.registeredAt ?? null, tags }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Release name for a capture: `<title> - <uploader> (nicovideo sm123)`. */
|
||||
export function nicovideoReleaseName(info: NicovideoInfo, fallbackId: string): string {
|
||||
const id = info.id || fallbackId
|
||||
return `${info.title}${info.owner ? ` - ${info.owner}` : ''} (nicovideo ${id})`
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* SFV checksum lists: one `name CRC32` per line, `;` comments. Scene
|
||||
* releases (music in particular) post one with every album; it names every
|
||||
* file that belongs to it.
|
||||
*/
|
||||
export interface SfvEntry {
|
||||
name: string
|
||||
crc: string
|
||||
}
|
||||
|
||||
export function parseSfv(text: string): SfvEntry[] {
|
||||
const entries: SfvEntry[] = []
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const line = raw.trim()
|
||||
if (!line || line.startsWith(';')) continue
|
||||
const match = /^(.+?)\s+([0-9a-f]{8})$/i.exec(line)
|
||||
if (match) entries.push({ name: match[1].replace(/^"|"$/g, '').split(/[\\/]/).pop()!, crc: match[2].toLowerCase() })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* The album name an SFV gives a release: its stem without the leading
|
||||
* track number scene groups put on the playlist/SFV/cover files
|
||||
* (`00-artist-album-web-2026-grp.sfv` → `artist-album-web-2026-grp`).
|
||||
*/
|
||||
export function sfvReleaseName(sfvName: string): string {
|
||||
return sfvName.replace(/\.sfv$/i, '').replace(/^0{1,3}-/, '')
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* Recognises a file type from its first bytes, independent of AdonisJS.
|
||||
* `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
|
||||
* "nzbe", 16 random bytes (salt/IV), then ciphertext.
|
||||
* `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
|
||||
* (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
|
||||
*/
|
||||
export type ContentType =
|
||||
| 'nzbe'
|
||||
@@ -16,6 +18,7 @@ export type ContentType =
|
||||
| 'matroska'
|
||||
| 'mp4'
|
||||
| 'rclone'
|
||||
| 'yencbin'
|
||||
| 'iso'
|
||||
| 'udf'
|
||||
| 'pdf'
|
||||
@@ -29,7 +32,11 @@ export type ContentType =
|
||||
| 'ogg'
|
||||
| 'bzip2'
|
||||
|
||||
const SIGNATURES: [ContentType, number, number[]][] = [
|
||||
/** Magic bytes at an offset; null matches any byte. */
|
||||
const SIGNATURES: [ContentType, number, (number | null)[]][] = [
|
||||
// Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
|
||||
// the bytes they share.
|
||||
['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
|
||||
['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
|
||||
['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
|
||||
['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
|
||||
@@ -53,7 +60,7 @@ const SIGNATURES: [ContentType, number, number[]][] = [
|
||||
|
||||
export function sniffContentType(data: Buffer): ContentType | null {
|
||||
for (const [type, offset, bytes] of SIGNATURES) {
|
||||
if (data.length >= offset + bytes.length && bytes.every((b, i) => data[offset + i] === b)) return type
|
||||
if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
|
||||
}
|
||||
// Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
|
||||
// ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
|
||||
|
||||
+13
-1
@@ -13,6 +13,8 @@ export interface NewznabItem {
|
||||
poster: string | null
|
||||
groups: string[]
|
||||
password: boolean
|
||||
/** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
|
||||
description?: string
|
||||
season?: string
|
||||
episode?: string
|
||||
}
|
||||
@@ -26,6 +28,16 @@ export interface NewznabServer {
|
||||
}
|
||||
|
||||
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
|
||||
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
|
||||
export function ubbToText(ubb: string): string {
|
||||
return ubb
|
||||
.replace(/<br\s*\/?>|\[br\]/gi, '\n')
|
||||
.replace(/<\/?[a-z][^>]*>/gi, '')
|
||||
.replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
|
||||
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function errorXml(code: number, description: string): string {
|
||||
return create({ version: '1.0', encoding: 'UTF-8' })
|
||||
.ele('error', { code: String(code), description })
|
||||
@@ -74,7 +86,7 @@ export function resultsXml(server: NewznabServer, items: NewznabItem[], offset:
|
||||
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
|
||||
element.ele('pubDate').txt(pubDate)
|
||||
element.ele('category').txt(categoryLabel(item.category))
|
||||
element.ele('description').txt(item.title)
|
||||
element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
|
||||
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
|
||||
|
||||
const attr = (name: string, value: string | number) =>
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* The article number where a group's posts reach `target` (ms since epoch),
|
||||
* by binary search between `first` and `last`. Article numbers only roughly
|
||||
* follow posting time, so the answer is within `precision` articles of the
|
||||
* first one posted at or after `target`.
|
||||
*
|
||||
* `dateOf(n)` returns the posting time of the first article at or after `n`
|
||||
* that still exists, or null when there is none nearby (expired or removed);
|
||||
* a gap counts as older than `target`.
|
||||
*/
|
||||
export async function articleAtDate(
|
||||
first: bigint,
|
||||
last: bigint,
|
||||
target: number,
|
||||
dateOf: (n: bigint) => Promise<number | null>,
|
||||
precision = 10_000n
|
||||
): Promise<bigint> {
|
||||
let lo = first
|
||||
let hi = last
|
||||
while (hi - lo > precision) {
|
||||
const mid = lo + (hi - lo) / 2n
|
||||
const date = await dateOf(mid)
|
||||
if (date === null || date < target) lo = mid
|
||||
else hi = mid
|
||||
}
|
||||
return lo
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
|
||||
/**
|
||||
* Splits a byte stream into messages that end in a delimiter, which the
|
||||
* consumer can change per message: `next(delimiter)` returns everything up
|
||||
* to the next `delimiter` (excluded). A replacement for nntp-js's
|
||||
* findByteSequence, which has two bugs:
|
||||
*
|
||||
* - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
|
||||
* without articles) never ends: the status line takes the CRLF that
|
||||
* `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
|
||||
* follows one that ended with CRLF, that CRLF counts for both.
|
||||
* - A delimiter split over two socket chunks left its first bytes in the
|
||||
* returned data.
|
||||
*
|
||||
* Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
|
||||
*/
|
||||
export async function* delimitedReader(
|
||||
source: AsyncIterable<Uint8Array>,
|
||||
initial: Uint8Array
|
||||
): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
|
||||
let delimiter = Buffer.from(initial)
|
||||
let parts: Buffer[] = []
|
||||
let length = 0
|
||||
// Bytes at the start of the buffer, already searched without a match.
|
||||
let searched = 0
|
||||
// Leading bytes that belong to the previous delimiter (a shared CRLF).
|
||||
let skip = 0
|
||||
|
||||
for await (const chunk of source) {
|
||||
parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
|
||||
length += chunk.byteLength
|
||||
|
||||
while (true) {
|
||||
const from = Math.max(0, searched - (delimiter.length - 1))
|
||||
const found = indexOf(parts, length, delimiter, from)
|
||||
if (found === -1) {
|
||||
searched = length
|
||||
break
|
||||
}
|
||||
const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
|
||||
const message = all.subarray(Math.min(skip, found), found)
|
||||
const rest = all.subarray(found + delimiter.length)
|
||||
const previous = delimiter
|
||||
|
||||
if (message.length === 0 && delimiter.equals(CRLF)) {
|
||||
// A stray empty line: keep looking for the same delimiter.
|
||||
} else {
|
||||
delimiter = Buffer.from((yield message) ?? delimiter)
|
||||
}
|
||||
|
||||
const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
|
||||
parts = shareCrlf ? [CRLF, rest] : [rest]
|
||||
length = rest.length + (shareCrlf ? 2 : 0)
|
||||
skip = shareCrlf ? 2 : 0
|
||||
searched = 0
|
||||
}
|
||||
}
|
||||
|
||||
if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
|
||||
}
|
||||
|
||||
const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
|
||||
const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
|
||||
|
||||
/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
|
||||
function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
|
||||
if (parts.length === 1) return parts[0].indexOf(needle, from)
|
||||
// Collect the parts that cover [from, length).
|
||||
let start = length
|
||||
let i = parts.length
|
||||
while (i > 0 && start > from) start -= parts[--i].length
|
||||
const window = Buffer.concat(parts.slice(i), length - start)
|
||||
const found = window.indexOf(needle, from - start)
|
||||
return found === -1 ? -1 : start + found
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { NNTP } from 'nntp-js'
|
||||
import { delimitedReader } from './delimited_reader.js'
|
||||
|
||||
const CRLF = new Uint8Array([13, 10])
|
||||
|
||||
/**
|
||||
* nntp-js with its reply reader replaced by `delimitedReader` (see there for
|
||||
* the bugs it fixes; the empty-reply one hung XOVER of empty ranges forever).
|
||||
* The reader is created in `connect`, so that is overridden: the same steps
|
||||
* as nntp-js 1.0.4, with the other reader.
|
||||
*/
|
||||
const proto = NNTP.prototype as any
|
||||
if (!proto.__delimitedReader) {
|
||||
proto.__delimitedReader = true
|
||||
proto.connect = async function (this: any, readermode = false, timeout: number | undefined = undefined, socket?: any) {
|
||||
try {
|
||||
this.sock = socket ?? this._createSocket(timeout)
|
||||
// The first reply is always the one-line welcome.
|
||||
this.byteReader = delimitedReader(this.sock, CRLF)
|
||||
this.welcome = new TextDecoder('utf-8').decode(await this._getresp())
|
||||
this.caps = undefined
|
||||
this.connected = true
|
||||
await this.getcapabilities()
|
||||
this.readermodeAfterauth = false
|
||||
if (readermode && !this.caps.READER) {
|
||||
await this._setreadermode()
|
||||
if (!this.readermodeAfterauth) {
|
||||
this.caps = undefined
|
||||
await this.getcapabilities()
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.sock?.end()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { NNTP }
|
||||
+6
-9
@@ -1,4 +1,5 @@
|
||||
import { create } from 'xmlbuilder2'
|
||||
import type { Segment } from './segments.js'
|
||||
|
||||
export interface NzbFile {
|
||||
/** Name the downloader should use; the real name when it is known. */
|
||||
@@ -7,8 +8,7 @@ export interface NzbFile {
|
||||
/** Posting time in milliseconds. */
|
||||
date: number
|
||||
groups: string[]
|
||||
/** Segment number → message-id (without angle brackets) and article size. */
|
||||
segments: Record<string, { id: string; size: number }>
|
||||
segments: Segment[]
|
||||
}
|
||||
|
||||
export interface NzbOptions {
|
||||
@@ -34,20 +34,17 @@ export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string {
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const numbers = Object.keys(file.segments)
|
||||
.map(Number)
|
||||
.sort((a, b) => a - b)
|
||||
const sorted = [...file.segments].sort((a, b) => a.number - b.number)
|
||||
const element = root.ele('file', {
|
||||
poster: file.poster,
|
||||
date: String(Math.floor(file.date / 1000)),
|
||||
subject: `"${file.name}" yEnc (1/${numbers.length})`,
|
||||
subject: `"${file.name}" yEnc (1/${sorted.length})`,
|
||||
})
|
||||
const groups = element.ele('groups')
|
||||
for (const group of file.groups) groups.ele('group').txt(group)
|
||||
const segments = element.ele('segments')
|
||||
for (const number of numbers) {
|
||||
const segment = file.segments[String(number)]
|
||||
segments.ele('segment', { bytes: String(segment.size), number: String(number) }).txt(segment.id)
|
||||
for (const segment of sorted) {
|
||||
segments.ele('segment', { bytes: String(segment.size), number: String(segment.number) }).txt(segment.id)
|
||||
}
|
||||
}
|
||||
return root.end({ prettyPrint: true })
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
*/
|
||||
export { buildNzb, type NzbFile, type NzbOptions } from './build.js'
|
||||
export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js'
|
||||
export type { Segment } from './segments.js'
|
||||
@@ -0,0 +1,33 @@
|
||||
import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib'
|
||||
|
||||
/** One article of a posted file: its number, message-id (without <>) and size in bytes. */
|
||||
export interface Segment {
|
||||
number: number
|
||||
id: string
|
||||
size: number
|
||||
}
|
||||
|
||||
const VERSION = 1
|
||||
|
||||
/**
|
||||
* Segment lists as stored in `files.segments` (CONCEPT.md D31): a version
|
||||
* byte, then zstd (level 1) of a JSON line with the sizes (and the numbers,
|
||||
* unless they are 1..N) followed by one message-id per line. About 20 bytes
|
||||
* per segment, against ~38 for the pglz-compressed JSON object it replaces:
|
||||
* message-ids are mostly random, so this is close to what's possible.
|
||||
*/
|
||||
export function packSegments(segments: Segment[]): Buffer {
|
||||
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
||||
const sequential = sorted.every((s, i) => s.number === i + 1)
|
||||
const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) })
|
||||
const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8')
|
||||
const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } })
|
||||
return Buffer.concat([Buffer.from([VERSION]), packed])
|
||||
}
|
||||
|
||||
export function unpackSegments(data: Buffer): Segment[] {
|
||||
if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`)
|
||||
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
||||
const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] }
|
||||
return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size }))
|
||||
}
|
||||
+2
-1
@@ -19,7 +19,8 @@ export interface RarEntry {
|
||||
}
|
||||
|
||||
export interface RarInfo {
|
||||
format: 'rar4' | 'rar5'
|
||||
/** 7z listings take the same shape (see lib/sevenzip). */
|
||||
format: 'rar4' | 'rar5' | '7z'
|
||||
/** Part of a multi-volume set. */
|
||||
volume: boolean
|
||||
/** First volume of the set (null when the format version does not say). */
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/**
|
||||
* Grouping of files into releases (downloads), independent of AdonisJS.
|
||||
*/
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
|
||||
@@ -197,11 +197,23 @@ export function par2ReleaseName(fileNames: string[]): string | undefined {
|
||||
* volumes after the first, the first entry is the file that continues from
|
||||
* the previous volume, so all volumes of a single-file archive agree.
|
||||
*/
|
||||
/**
|
||||
* Folder and file names that say nothing about what a release is: disc
|
||||
* structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers.
|
||||
* Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i
|
||||
|
||||
export function isGenericName(name: string): boolean {
|
||||
return GENERIC_NAMES.test(name.trim())
|
||||
}
|
||||
|
||||
export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
|
||||
if (entries.length === 0) return null
|
||||
const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
const nested = entries.some((e) => /[\\/]/.test(e.name))
|
||||
if (nested && tops.size === 1) {
|
||||
// A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead.
|
||||
if (nested && tops.size === 1 && !isGenericName([...tops][0])) {
|
||||
const [top] = tops
|
||||
return { key: hash('rar-dir', top), name: top, source: 'rar' }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* 7-Zip archives, independent of AdonisJS. A 7z archive starts with a
|
||||
* 32-byte signature header that points to the archive's header (the file
|
||||
* list) at its very end, often LZMA-compressed ("encoded") with its packed
|
||||
* data just before it. So the list needs the first 32 bytes and the tail:
|
||||
* for a split archive (.7z.001, .002, …), the start of the first volume and
|
||||
* the end of the last. Listing itself is left to libarchive (bsdtar).
|
||||
*/
|
||||
|
||||
const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
|
||||
const SIGNATURE_HEADER_SIZE = 32
|
||||
/** Header property ids (7zFormat.txt). */
|
||||
const K_ENCODED_HEADER = 0x17
|
||||
const K_PACK_INFO = 0x06
|
||||
|
||||
export interface SevenZipStart {
|
||||
/** Where the archive header starts, from the start of the archive. */
|
||||
headerStart: number
|
||||
headerSize: number
|
||||
headerCrc: number
|
||||
/** The archive's total size (the header is its last part). */
|
||||
archiveSize: number
|
||||
}
|
||||
|
||||
/** Reads the signature header; null when `data` isn't a 7z start or its CRC fails. */
|
||||
export function parseSevenZipStart(data: Buffer): SevenZipStart | null {
|
||||
if (data.length < SIGNATURE_HEADER_SIZE || !data.subarray(0, 6).equals(SIGNATURE)) return null
|
||||
if ((crc32(data.subarray(12, 32)) >>> 0) !== data.readUInt32LE(8)) return null
|
||||
const offset = Number(data.readBigUInt64LE(12))
|
||||
const size = Number(data.readBigUInt64LE(20))
|
||||
const headerStart = SIGNATURE_HEADER_SIZE + offset
|
||||
return { headerStart, headerSize: size, headerCrc: data.readUInt32LE(28), archiveSize: headerStart + size }
|
||||
}
|
||||
|
||||
/** Whether `header` (the bytes at headerStart) is the archive header its signature header describes. */
|
||||
export function checkHeader(start: SevenZipStart, header: Buffer): boolean {
|
||||
return header.length === start.headerSize && (crc32(header) >>> 0) === start.headerCrc
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the bytes needed for listing start: the header itself, or for an
|
||||
* encoded header the packed (compressed) header stream it points to.
|
||||
*/
|
||||
export function listingStart(start: SevenZipStart, header: Buffer): number {
|
||||
if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return start.headerStart
|
||||
const [packPos] = readNumber(header, 2)
|
||||
return SIGNATURE_HEADER_SIZE + packPos
|
||||
}
|
||||
|
||||
/**
|
||||
* 7z's variable-length number: the leading 1-bits of the first byte count
|
||||
* the little-endian bytes that follow; its remaining bits are the high part.
|
||||
*/
|
||||
export function readNumber(data: Buffer, pos: number): [value: number, next: number] {
|
||||
const first = data[pos]
|
||||
let mask = 0x80
|
||||
let value = 0
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * 2 ** (8 * i)
|
||||
return [value, pos + 1 + i]
|
||||
}
|
||||
value += data[pos + 1 + i] * 2 ** (8 * i)
|
||||
mask >>= 1
|
||||
}
|
||||
return [value, pos + 9]
|
||||
}
|
||||
|
||||
export interface SevenZipEntry {
|
||||
name: string
|
||||
size: number
|
||||
directory: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Entries from `bsdtar -tv` output, e.g.
|
||||
* `-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release/movie.mkv`.
|
||||
*/
|
||||
export function parseBsdtarListing(stdout: string): SevenZipEntry[] {
|
||||
const entries: SevenZipEntry[] = []
|
||||
for (const line of stdout.split('\n')) {
|
||||
const match = /^([-dlbcps])\S*\s+\d+\s+\S+\s+\S+\s+(\d+)\s+\w{3}\s+\d{1,2}\s+(?:\d{1,2}:\d{2}|\d{4})\s(.+)$/.exec(line)
|
||||
if (!match) continue
|
||||
const directory = match[1] === 'd'
|
||||
entries.push({ name: directory ? match[3].replace(/\/$/, '') : match[3], size: Number(match[2]), directory })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */
|
||||
export function sevenZipVolume(name: string): number | null {
|
||||
const split = /\.7z\.(\d{1,4})$/i.exec(name)
|
||||
if (split) return Number(split[1])
|
||||
return /\.7z$/i.test(name) ? 0 : null
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
/**
|
||||
* Spotnet's category tree, from Spotweb's SpotCategories (English names).
|
||||
* A spot has a head category (0–3) and subcategories per type letter:
|
||||
* `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
|
||||
* `d` genre, `z` kind (movie, series, book, …).
|
||||
*/
|
||||
export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
|
||||
|
||||
/** What each type letter means per head category. */
|
||||
export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
|
||||
{ a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Genre', z: 'Type' },
|
||||
]
|
||||
|
||||
const NAMES: Record<string, Record<number, string>>[] = [
|
||||
{
|
||||
a: {
|
||||
0: 'DivX',
|
||||
1: 'WMV',
|
||||
2: 'MPG',
|
||||
3: 'DVD5',
|
||||
4: 'HD other',
|
||||
5: 'ePub',
|
||||
6: 'Blu-ray',
|
||||
7: 'HD-DVD',
|
||||
8: 'WMVHD',
|
||||
9: 'x264',
|
||||
10: 'DVD9',
|
||||
11: 'PDF',
|
||||
12: 'Bitmap',
|
||||
13: 'Vector',
|
||||
14: '3D',
|
||||
15: 'UHD',
|
||||
},
|
||||
b: {
|
||||
0: 'CAM',
|
||||
1: '(S)VCD',
|
||||
2: 'Promo',
|
||||
3: 'Retail',
|
||||
4: 'TV',
|
||||
6: 'Satellite',
|
||||
7: 'R5',
|
||||
8: 'Telecine',
|
||||
9: 'Telesync',
|
||||
10: 'Scan',
|
||||
11: 'WEB-DL',
|
||||
12: 'WEBRip',
|
||||
13: 'HDRip',
|
||||
},
|
||||
c: {
|
||||
0: 'No subtitles',
|
||||
1: 'Dutch subtitles (external)',
|
||||
2: 'Dutch subtitles (builtin)',
|
||||
3: 'English subtitles (external)',
|
||||
4: 'English subtitles (builtin)',
|
||||
6: 'Dutch subtitles (available)',
|
||||
7: 'English subtitles (available)',
|
||||
10: 'English audio/written',
|
||||
11: 'Dutch audio/written',
|
||||
12: 'German audio/written',
|
||||
13: 'French audio/written',
|
||||
14: 'Spanish audio/written',
|
||||
15: 'Asian audio/written',
|
||||
},
|
||||
d: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Animation',
|
||||
3: 'Cabaret',
|
||||
4: 'Comedy',
|
||||
5: 'Crime',
|
||||
6: 'Documentary',
|
||||
7: 'Drama',
|
||||
8: 'Family',
|
||||
9: 'Fantasy',
|
||||
10: 'Arthouse',
|
||||
11: 'Television',
|
||||
12: 'Horror',
|
||||
13: 'Music',
|
||||
14: 'Musical',
|
||||
15: 'Mystery',
|
||||
16: 'Romance',
|
||||
17: 'Science Fiction',
|
||||
18: 'Sport',
|
||||
19: 'Short movie',
|
||||
20: 'Thriller',
|
||||
21: 'War',
|
||||
22: 'Western',
|
||||
23: 'Erotica (hetero)',
|
||||
24: 'Erotica (gay male)',
|
||||
25: 'Erotica (gay female)',
|
||||
26: 'Erotica (bi)',
|
||||
28: 'Asian',
|
||||
29: 'Anime',
|
||||
30: 'Cover',
|
||||
31: 'Comicbook',
|
||||
32: 'Cartoons',
|
||||
33: 'Youth',
|
||||
34: 'Business',
|
||||
35: 'Computer',
|
||||
36: 'Hobby',
|
||||
37: 'Cooking',
|
||||
38: 'Handwork',
|
||||
39: 'Craftwork',
|
||||
40: 'Health',
|
||||
41: 'History',
|
||||
42: 'Psychology',
|
||||
43: 'Newspaper',
|
||||
44: 'Magazine',
|
||||
45: 'Science',
|
||||
46: 'Female',
|
||||
47: 'Religion',
|
||||
48: 'Roman',
|
||||
49: 'Biography',
|
||||
50: 'Detective',
|
||||
51: 'Animals',
|
||||
52: 'Humor',
|
||||
53: 'Travel',
|
||||
54: 'True story',
|
||||
55: 'Non-fiction',
|
||||
56: 'Politics',
|
||||
57: 'Poetry',
|
||||
58: 'Fairy tale',
|
||||
59: 'Technical',
|
||||
60: 'Art',
|
||||
72: 'Bi',
|
||||
73: 'Lesbian',
|
||||
74: 'Homo',
|
||||
75: 'Hetero',
|
||||
76: 'Amature',
|
||||
77: 'Group',
|
||||
78: 'POV',
|
||||
79: 'Solo',
|
||||
80: 'Young',
|
||||
81: 'Soft',
|
||||
82: 'Fetish',
|
||||
83: 'Old',
|
||||
84: 'Fat',
|
||||
85: 'SM',
|
||||
86: 'Rough',
|
||||
87: 'Dark',
|
||||
88: 'Hentai',
|
||||
89: 'Outside',
|
||||
},
|
||||
z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'MP3',
|
||||
1: 'WMA',
|
||||
2: 'WAV',
|
||||
3: 'OGG',
|
||||
4: 'EAC',
|
||||
5: 'DTS',
|
||||
6: 'AAC',
|
||||
7: 'APE',
|
||||
8: 'FLAC',
|
||||
},
|
||||
b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
|
||||
c: {
|
||||
0: 'Variable',
|
||||
1: '< 96kbit',
|
||||
2: '96kbit',
|
||||
3: '128kbit',
|
||||
4: '160kbit',
|
||||
5: '192kbit',
|
||||
6: '256kbit',
|
||||
7: '320kbit',
|
||||
8: 'Lossless',
|
||||
9: 'Other',
|
||||
},
|
||||
d: {
|
||||
0: 'Blues',
|
||||
1: 'Compilation',
|
||||
2: 'Cabaret',
|
||||
3: 'Dance',
|
||||
4: 'Diverse',
|
||||
5: 'Hardcore',
|
||||
6: 'World',
|
||||
7: 'Jazz',
|
||||
8: 'Youth',
|
||||
9: 'Classical',
|
||||
10: 'Kleinkunst',
|
||||
11: 'Dutch',
|
||||
12: 'New Age',
|
||||
13: 'Pop',
|
||||
14: 'RnB',
|
||||
15: 'Hiphop',
|
||||
16: 'Reggae',
|
||||
17: 'Religious',
|
||||
18: 'Rock',
|
||||
19: 'Soundtracks',
|
||||
20: 'Other',
|
||||
21: 'Hardstyle',
|
||||
22: 'Asian',
|
||||
23: 'Disco',
|
||||
24: 'Classics',
|
||||
25: 'Metal',
|
||||
26: 'Country',
|
||||
27: 'Dubstep',
|
||||
28: 'Nederhop',
|
||||
29: 'DnB',
|
||||
30: 'Electro',
|
||||
31: 'Folk',
|
||||
32: 'Soul',
|
||||
33: 'Trance',
|
||||
34: 'Balkan',
|
||||
35: 'Techno',
|
||||
36: 'Ambient',
|
||||
37: 'Latin',
|
||||
38: 'Live',
|
||||
},
|
||||
z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'Playstation',
|
||||
4: 'Playstation 2',
|
||||
5: 'PSP',
|
||||
6: 'Xbox',
|
||||
7: 'Xbox 360',
|
||||
8: 'Gameboy Advance',
|
||||
9: 'Gamecube',
|
||||
10: 'Nintendo DS',
|
||||
11: 'Nintento Wii',
|
||||
12: 'Playstation 3',
|
||||
13: 'Windows Phone',
|
||||
14: 'iOS',
|
||||
15: 'Android',
|
||||
16: 'Nintendo 3DS',
|
||||
17: 'Playstation 4',
|
||||
18: 'XBox 1',
|
||||
},
|
||||
b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
|
||||
c: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Strategy',
|
||||
3: 'Roleplaying',
|
||||
4: 'Simulation',
|
||||
5: 'Race',
|
||||
6: 'Flying',
|
||||
7: 'Shooter',
|
||||
8: 'Platform',
|
||||
9: 'Sport',
|
||||
10: 'Child/youth',
|
||||
11: 'Puzzle',
|
||||
12: 'Other',
|
||||
13: 'Boardgame',
|
||||
14: 'Cards',
|
||||
15: 'Education',
|
||||
16: 'Music',
|
||||
17: 'Family',
|
||||
},
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'OS/2',
|
||||
4: 'Windows Phone',
|
||||
5: 'Navigation systems',
|
||||
6: 'iOS',
|
||||
7: 'Android',
|
||||
},
|
||||
b: {
|
||||
0: 'Audio',
|
||||
1: 'Video',
|
||||
2: 'Graphics',
|
||||
3: 'CD/DVD Tools',
|
||||
4: 'Media players',
|
||||
5: 'Rippers & Encoders',
|
||||
6: 'Plugins',
|
||||
7: 'Database tools',
|
||||
8: 'Email software',
|
||||
9: 'Photo',
|
||||
10: 'Screensavers',
|
||||
11: 'Skin software',
|
||||
12: 'Drivers',
|
||||
13: 'Browsers',
|
||||
14: 'Download managers',
|
||||
15: 'Download',
|
||||
16: 'Usenet software',
|
||||
17: 'RSS Readers',
|
||||
18: 'FTP software',
|
||||
19: 'Firewalls',
|
||||
20: 'Antivirus software',
|
||||
21: 'Antispyware software',
|
||||
22: 'Optimization software',
|
||||
23: 'Security software',
|
||||
24: 'System software',
|
||||
25: 'Other',
|
||||
26: 'Educational',
|
||||
27: 'Office',
|
||||
28: 'Internet',
|
||||
29: 'Communication',
|
||||
30: 'Development',
|
||||
31: 'Spotnet',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
export interface SpotCategoryLabels {
|
||||
head: string | null
|
||||
/** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
|
||||
subcats: { type: string; values: string[] }[]
|
||||
}
|
||||
|
||||
/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
|
||||
export function spotCategoryLabels(
|
||||
category: number | null,
|
||||
subcats: Record<string, number[]> | null
|
||||
): SpotCategoryLabels {
|
||||
if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
|
||||
const labels: SpotCategoryLabels['subcats'] = []
|
||||
for (const letter of ['z', 'a', 'b', 'c', 'd']) {
|
||||
const values = [
|
||||
...new Set(
|
||||
(subcats?.[letter] ?? [])
|
||||
.map((n) => NAMES[category][letter]?.[n])
|
||||
.filter((v): v is string => Boolean(v))
|
||||
),
|
||||
]
|
||||
if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
|
||||
}
|
||||
return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
|
||||
}
|
||||
@@ -4,4 +4,6 @@
|
||||
* See CONCEPT.md (D29).
|
||||
*/
|
||||
export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
|
||||
export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js'
|
||||
export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js'
|
||||
export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js'
|
||||
export { passwordFromSpot } from './password.js'
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Archive passwords that spotters put in the title or description instead of
|
||||
* the NZB's `<meta type="password">`:
|
||||
* `Title{{secret}}` (the newznab/SABnzbd convention),
|
||||
* `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
|
||||
* `WW FC0401` (abbreviation without separator: only with a digit in it).
|
||||
* The value must be the rest of its line and a single token, so sentences
|
||||
* such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
|
||||
* "met een password beschermd" don't count.
|
||||
*/
|
||||
const KEYWORD = String.raw`wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw`
|
||||
const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])(?:${KEYWORD})\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
|
||||
const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
|
||||
const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
|
||||
const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
|
||||
|
||||
/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */
|
||||
function lines(description: string): string[] {
|
||||
return description
|
||||
.replace(/\[br\]|<br\s*\/?>/gi, '\n')
|
||||
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '')
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
}
|
||||
|
||||
function clean(value: string): string | null {
|
||||
const token = value.trim().replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
|
||||
if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
|
||||
return token
|
||||
}
|
||||
|
||||
export function passwordFromSpot(title: string, description?: string | null): string | null {
|
||||
const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
|
||||
if (braces && clean(braces[1])) return clean(braces[1])
|
||||
for (const line of lines(description ?? '')) {
|
||||
if (!line || NO_PASSWORD.test(line)) continue
|
||||
const separated = WITH_SEPARATOR.exec(line)
|
||||
if (separated) {
|
||||
const value = clean(separated[1])
|
||||
if (value) return value
|
||||
continue
|
||||
}
|
||||
const abbreviated = ABBREVIATION.exec(line)
|
||||
if (abbreviated && /\d/.test(abbreviated[1])) {
|
||||
const value = clean(abbreviated[1])
|
||||
if (value) return value
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
+51
-4
@@ -6,18 +6,25 @@ export interface SpotXml {
|
||||
subcats: string[]
|
||||
size: number
|
||||
website: string
|
||||
/** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
|
||||
description: string
|
||||
tag: string
|
||||
/** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
|
||||
image: SpotImage | null
|
||||
/** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
|
||||
nzbSegments: string[]
|
||||
created: number | null
|
||||
}
|
||||
|
||||
export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
|
||||
function text(value: string | undefined): string {
|
||||
if (value === undefined) return ''
|
||||
const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
|
||||
if (cdata) return cdata[1]
|
||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
@@ -37,6 +44,9 @@ export function parseSpotXml(xml: string): SpotXml | null {
|
||||
const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
|
||||
const nzb = element(posting, 'NZB') ?? ''
|
||||
return {
|
||||
description: text(element(posting, 'Description')).trim(),
|
||||
tag: text(element(posting, 'Tag')).trim(),
|
||||
image: parseImage(posting),
|
||||
title: text(element(posting, 'Title')).trim(),
|
||||
// Newer keys number categories from 1.
|
||||
category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
|
||||
@@ -48,15 +58,52 @@ export function parseSpotXml(xml: string): SpotXml | null {
|
||||
}
|
||||
}
|
||||
|
||||
/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
|
||||
function parseImage(posting: string): SpotImage | null {
|
||||
const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
|
||||
if (!match) return null
|
||||
const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
|
||||
if (segments.length > 0) {
|
||||
const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
|
||||
return { segments, width: size('Width'), height: size('Height') }
|
||||
}
|
||||
const url = text(match[2]).trim()
|
||||
return /^https?:\/\//i.test(url) ? { url } : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
|
||||
* the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
|
||||
*/
|
||||
export function decodeSpotText(latin1: string): string {
|
||||
const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
|
||||
return utf8.includes('\uFFFD') ? latin1 : utf8
|
||||
}
|
||||
|
||||
/**
|
||||
* Header lines of a raw HEAD response; X-XML (split over many lines) joined.
|
||||
* Folded headers are unfolded: old (key 1) spots put the From address on a
|
||||
* continuation line.
|
||||
*/
|
||||
export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
|
||||
const headers: Record<string, string> = {}
|
||||
let xml = ''
|
||||
let last: string | null = null
|
||||
let lastIsFirst = false
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
if (/^[ \t]/.test(line) && last) {
|
||||
if (last === 'x-xml') xml += line
|
||||
else if (lastIsFirst) headers[last] += line
|
||||
continue
|
||||
}
|
||||
const colon = line.indexOf(':')
|
||||
if (colon <= 0) continue
|
||||
if (colon <= 0) {
|
||||
last = null
|
||||
continue
|
||||
}
|
||||
const name = line.slice(0, colon).toLowerCase()
|
||||
const value = line.slice(colon + 1).replace(/^ /, '')
|
||||
last = name
|
||||
lastIsFirst = !(name in headers)
|
||||
if (name === 'x-xml') xml += value
|
||||
else headers[name] ??= value
|
||||
}
|
||||
|
||||
@@ -28,3 +28,20 @@ export function weakFileKey(name: string, size: number, totalParts: number): str
|
||||
const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
|
||||
return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file by its counters, for uploads that give every article
|
||||
* its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`,
|
||||
* part count and posting day stay the same across the articles of one file.
|
||||
* The day keeps different uploads with the same shape apart; collisions that
|
||||
* remain are caught by the conflict check (these files are weak identities).
|
||||
* Null when the subject has no file counter.
|
||||
*/
|
||||
export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
|
||||
if (!counter || segment.totalParts < 2) return null
|
||||
const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
|
||||
return createHash('sha1')
|
||||
.update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
|
||||
.digest('hex')
|
||||
}
|
||||
@@ -2,4 +2,4 @@
|
||||
* Conventions of binary posts (subject lines, file identity), independent of AdonisJS.
|
||||
*/
|
||||
export { parseSubject, type ParsedSubject } from './parse.js'
|
||||
export { fileKey, weakFileKey, looksLikeFilename } from './file_key.js'
|
||||
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
||||
Reference in new issue
Block a user