Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
3.1 KiB
TypeScript
77 lines
3.1 KiB
TypeScript
import { createZstdDecompress } from 'node:zlib'
|
||
|
||
/**
|
||
* Captures of niconico videos (`<unixtime>_nicovideo_jp_watch_sm<id>.tar.zst`,
|
||
* one poster, ~190k files on homelab). The tar's first entry after its
|
||
* directory is `server_response.metadata`: the site's API response for the
|
||
* watch page, 30–70 KB, so the first article of the file holds it.
|
||
*/
|
||
export const NICOVIDEO_FILE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)\.tar\.zst$/i
|
||
export const NICOVIDEO_RELEASE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)$/i
|
||
|
||
export interface NicovideoInfo {
|
||
id: string
|
||
title: string
|
||
owner: string | null
|
||
registeredAt: string | null
|
||
tags: string[]
|
||
}
|
||
|
||
/** As much of a zstd stream as `data` (a prefix of it) decompresses to. */
|
||
export function zstdPrefix(data: Buffer, maxBytes = 4 * 1024 * 1024): Promise<Buffer> {
|
||
return new Promise((resolve) => {
|
||
const out: Buffer[] = []
|
||
let length = 0
|
||
const z = createZstdDecompress()
|
||
const done = () => resolve(Buffer.concat(out))
|
||
z.on('data', (chunk: Buffer) => {
|
||
out.push(chunk)
|
||
length += chunk.length
|
||
if (length >= maxBytes) z.destroy()
|
||
})
|
||
z.on('error', done)
|
||
z.on('close', done)
|
||
z.on('end', done)
|
||
z.end(data)
|
||
})
|
||
}
|
||
|
||
/** Entries at the start of a (possibly truncated) ustar archive. */
|
||
export function* tarEntries(tar: Buffer): Generator<{ name: string; data: Buffer }> {
|
||
let offset = 0
|
||
while (offset + 512 <= tar.length) {
|
||
const header = tar.subarray(offset, offset + 512)
|
||
const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '')
|
||
if (!name) return
|
||
const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '')
|
||
const size = Number.parseInt(header.subarray(124, 136).toString('latin1').replace(/\0.*$/s, '').trim() || '0', 8) || 0
|
||
yield { name: prefix ? `${prefix}/${name}` : name, data: tar.subarray(offset + 512, offset + 512 + size) }
|
||
offset += 512 + Math.ceil(size / 512) * 512
|
||
}
|
||
}
|
||
|
||
/** Title, uploader and tags from the start of a capture, or null when they aren't in it. */
|
||
export function nicovideoInfo(tar: Buffer): NicovideoInfo | null {
|
||
for (const entry of tarEntries(tar)) {
|
||
if (!entry.name.endsWith('/server_response.metadata')) continue
|
||
try {
|
||
const text = entry.data.toString('utf8')
|
||
const json = JSON.parse(text.slice(text.indexOf('{')))
|
||
const video = json?.data?.response?.video
|
||
if (!video?.title) return null
|
||
const owner = json.data.response.owner?.nickname ?? json.data.response.channel?.name ?? json.data.metadata?.jsonLds?.[0]?.author?.name ?? null
|
||
const tags = (json.data.response.tag?.items ?? []).map((t: any) => String(t.name)).filter(Boolean)
|
||
return { id: String(video.id ?? ''), title: String(video.title).trim(), owner: owner ? String(owner).trim() : null, registeredAt: video.registeredAt ?? null, tags }
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
return null
|
||
}
|
||
|
||
/** Release name for a capture: `<title> - <uploader> (nicovideo sm123)`. */
|
||
export function nicovideoReleaseName(info: NicovideoInfo, fallbackId: string): string {
|
||
const id = info.id || fallbackId
|
||
return `${info.title}${info.owner ? ` - ${info.owner}` : ''} (nicovideo ${id})`
|
||
}
|