Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
34 lines
1.6 KiB
TypeScript
34 lines
1.6 KiB
TypeScript
import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib'
|
|
|
|
/** One article of a posted file: its number, message-id (without <>) and size in bytes. */
|
|
export interface Segment {
|
|
number: number
|
|
id: string
|
|
size: number
|
|
}
|
|
|
|
const VERSION = 1
|
|
|
|
/**
|
|
* Segment lists as stored in `files.segments` (CONCEPT.md D31): a version
|
|
* byte, then zstd (level 1) of a JSON line with the sizes (and the numbers,
|
|
* unless they are 1..N) followed by one message-id per line. About 20 bytes
|
|
* per segment, against ~38 for the pglz-compressed JSON object it replaces:
|
|
* message-ids are mostly random, so this is close to what's possible.
|
|
*/
|
|
export function packSegments(segments: Segment[]): Buffer {
|
|
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
|
const sequential = sorted.every((s, i) => s.number === i + 1)
|
|
const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) })
|
|
const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8')
|
|
const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } })
|
|
return Buffer.concat([Buffer.from([VERSION]), packed])
|
|
}
|
|
|
|
export function unpackSegments(data: Buffer): Segment[] {
|
|
if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`)
|
|
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
|
const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] }
|
|
return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size }))
|
|
}
|