Rebuild: fresh AdonisJS 7 API app on Postgres

The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
@@ -1,40 +0,0 @@
-import { sevenZipVolume } from '../sevenzip/index.js'
-
-/**
- * Files whose first segment says what an archive holds: named first RAR
- * volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
- * uploaders use for archive volumes too (sniffed by content).
- */
-export function isArchiveStart(name: string): boolean {
-  if (/\.bin$/i.test(name)) return true
-  const volume = sevenZipVolume(name)
-  if (volume === 0 || volume === 1) return true
-  return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
-}
-
-/**
- * A later RAR volume (.part02.rar, .r00) or a par2 recovery volume
- * (.vol03+04.par2): the volume number is readable even under a random stem.
- * The set is listed from its first volume and named from its index .par2,
- * so the first segment of these says nothing new (LOG.md 2026-10-04).
- */
-export function isLaterVolume(name: string): boolean {
-  const part = /\.part(\d+)\.rar$/i.exec(name)
-  if (part) return Number(part[1]) > 1
-  return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name)
-}
-
-/**
- * The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of
- * `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for
- * other names.
- */
-export function rarVolume(name: string): { base: string; number: number } | null {
-  const part = /^(.*)\.part(\d+)\.rar$/i.exec(name)
-  if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) }
-  const old = /^(.*)\.r(\d{2,3})$/i.exec(name)
-  if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 }
-  const first = /^(.*)\.rar$/i.exec(name)
-  if (first) return { base: first[1].toLowerCase(), number: 1 }
-  return null
-}
@@ -1,7 +0,0 @@
-/**
- * Content recognition by magic bytes, independent of AdonisJS.
- */
-export { sniffContentType, type ContentType } from './sniff.js'
-export { isArchiveStart, isLaterVolume, rarVolume } from './archive_names.js'
-export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
-export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'
@@ -1,76 +0,0 @@
-import { createZstdDecompress } from 'node:zlib'
-
-/**
- * Captures of niconico videos (`<unixtime>_nicovideo_jp_watch_sm<id>.tar.zst`,
- * one poster, ~190k files on homelab). The tar's first entry after its
- * directory is `server_response.metadata`: the site's API response for the
- * watch page, 30–70 KB, so the first article of the file holds it.
- */
-export const NICOVIDEO_FILE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)\.tar\.zst$/i
-export const NICOVIDEO_RELEASE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)$/i
-
-export interface NicovideoInfo {
-  id: string
-  title: string
-  owner: string | null
-  registeredAt: string | null
-  tags: string[]
-}
-
-/** As much of a zstd stream as `data` (a prefix of it) decompresses to. */
-export function zstdPrefix(data: Buffer, maxBytes = 4 * 1024 * 1024): Promise<Buffer> {
-  return new Promise((resolve) => {
-    const out: Buffer[] = []
-    let length = 0
-    const z = createZstdDecompress()
-    const done = () => resolve(Buffer.concat(out))
-    z.on('data', (chunk: Buffer) => {
-      out.push(chunk)
-      length += chunk.length
-      if (length >= maxBytes) z.destroy()
-    })
-    z.on('error', done)
-    z.on('close', done)
-    z.on('end', done)
-    z.end(data)
-  })
-}
-
-/** Entries at the start of a (possibly truncated) ustar archive. */
-export function* tarEntries(tar: Buffer): Generator<{ name: string; data: Buffer }> {
-  let offset = 0
-  while (offset + 512 <= tar.length) {
-    const header = tar.subarray(offset, offset + 512)
-    const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '')
-    if (!name) return
-    const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '')
-    const size = Number.parseInt(header.subarray(124, 136).toString('latin1').replace(/\0.*$/s, '').trim() || '0', 8) || 0
-    yield { name: prefix ? `${prefix}/${name}` : name, data: tar.subarray(offset + 512, offset + 512 + size) }
-    offset += 512 + Math.ceil(size / 512) * 512
-  }
-}
-
-/** Title, uploader and tags from the start of a capture, or null when they aren't in it. */
-export function nicovideoInfo(tar: Buffer): NicovideoInfo | null {
-  for (const entry of tarEntries(tar)) {
-    if (!entry.name.endsWith('/server_response.metadata')) continue
-    try {
-      const text = entry.data.toString('utf8')
-      const json = JSON.parse(text.slice(text.indexOf('{')))
-      const video = json?.data?.response?.video
-      if (!video?.title) return null
-      const owner = json.data.response.owner?.nickname ?? json.data.response.channel?.name ?? json.data.metadata?.jsonLds?.[0]?.author?.name ?? null
-      const tags = (json.data.response.tag?.items ?? []).map((t: any) => String(t.name)).filter(Boolean)
-      return { id: String(video.id ?? ''), title: String(video.title).trim(), owner: owner ? String(owner).trim() : null, registeredAt: video.registeredAt ?? null, tags }
-    } catch {
-      return null
-    }
-  }
-  return null
-}
-
-/** Release name for a capture: `<title> - <uploader> (nicovideo sm123)`. */
-export function nicovideoReleaseName(info: NicovideoInfo, fallbackId: string): string {
-  const id = info.id || fallbackId
-  return `${info.title}${info.owner ? ` - ${info.owner}` : ''} (nicovideo ${id})`
-}
@@ -1,205 +0,0 @@
-import { createHash } from 'node:crypto'
-import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
-import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
-import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
-import { extractEntry } from '../rar/extract.js'
-import { walkRar } from '../rar/walk.js'
-import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
-import { listSevenZip } from '../sevenzip/list.js'
-import { sniffContentType, type ContentType } from './sniff.js'
-import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'
-
-/**
- * Content inspection's download-and-parse half (CONCEPT.md D21, D41),
- * without a database: what the indexer needs to know about a file, from its
- * segments. The indexer prepares a task from the database, the probe runs
- * here or on a node (articles through the seed API's relay), and the
- * indexer applies the result. Tasks and results are plain JSON.
- */
-
-export interface TaskSegment {
-  number: number
-  id: string
-  /** Article size in bytes. */
-  size?: number
-}
-
-/** Inspect one file: its first segment, and more where its kind needs it. */
-export interface InspectTask {
-  kind: 'inspect'
-  fileId: number
-  /** real_name, else filename. */
-  name: string
-  segments: TaskSegment[]
-  /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
-  sevenZip: { volumes: number; last: TaskSegment[] } | null
-}
-
-export interface InspectProbe {
-  /** The whole file's size, from the yEnc header. */
-  size: number
-  contentType: ContentType | null
-  md5_16k: string | null
-  /** The file name in the yEnc header. */
-  yencName: string
-  nicovideo?: NicovideoInfo | null
-  /** A whole small SFV (latin1) or NZB (utf8). */
-  text?: string | null
-  par2?: Par2Set | null
-  archive?: RarInfo | null
-  /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
-  archiveNzbs?: { entryName: string; xml: string }[]
-  archiveNzbEntries?: number
-}
-
-/** Walk a RAR set's headers for its complete listing (D40). */
-export interface WalkTask {
-  kind: 'walk'
-  releaseId: number
-  volumes: TaskSegment[][]
-  maxSegments: number
-}
-
-export interface WalkProbe {
-  entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
-  complete: boolean
-  stopped?: string
-  headersEncrypted: boolean
-  segments: number
-}
-
-export type ContentTask = InspectTask | WalkTask
-
-/** Small posted files (NZBs, SFVs) that are read completely. */
-const WHOLE_MAX_PARTS = 40
-const WHOLE_MAX_BYTES = 25 * 1024 * 1024
-/** Par2 index files are small; recovery volumes can be large and are not needed. */
-const PAR2_MAX_PARTS = 20
-const FIRST_CHUNK = 16 * 1024
-
-/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
-export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
-  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
-  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
-}
-
-/** A "no such article" answer: 430 from the news server, 410 from the seed relay. */
-export function isMissingArticle(error: unknown): boolean {
-  return /(^|[^0-9])(430|410)([^0-9]|$)/.test(String((error as any)?.message ?? error)) && !/timed out|no answer/i.test(String((error as any)?.message))
-}
-
-/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
-export function decodeBody(body: Buffer): DecodedSegment {
-  try {
-    return decodeSegment(body)
-  } catch (error) {
-    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
-    return decodeSegment(body, { lenientSize: true })
-  }
-}
-
-/**
- * All of a (small) file whose first segment is already decoded; null when it is too big.
- * The first segment isn't downloaded again (it was, for every par2 set, NZB and SFV).
- */
-export async function readWhole(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
-  const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
-  if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
-  if (segments.length === 1) return first.data
-  const assembled = YencFile.fromSegment(first)
-  assembled.add(first)
-  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id), { lenientSize: true }))
-  return assembled.getBuffer()
-}
-
-/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
-export async function readPar2(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
-  if (!isPar2(first.data)) return null
-  if (segments.length > PAR2_MAX_PARTS) return null
-  if (segments.length === 1) return parsePar2(first.data)
-  const assembled = YencFile.fromSegment(first)
-  assembled.add(first)
-  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id)))
-  return parsePar2(assembled.getBuffer())
-}
-
-export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
-  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
-}
-
-/** NZBs that can be read from a single, unencrypted archive. */
-function hasNzbEntries(info: RarInfo): boolean {
-  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
-}
-
-export async function probeInspection(
-  task: InspectTask,
-  fetch: BodyFetcher,
-  warn: (message: string) => void = () => {}
-): Promise<InspectProbe> {
-  let body: Buffer
-  try {
-    body = await fetch(task.segments[0].id)
-  } catch (error: any) {
-    // The server no longer has it (430; 410 through the seed relay), typically a takedown. A final
-    // answer, and an expensive one: Eweka takes ~450 ms per missing article, so it isn't retried
-    // (LOG.md 2026-10-06 (14)).
-    if (isMissingArticle(error)) return { size: 0, contentType: 'missing', md5_16k: null, yencName: '' }
-    throw error
-  }
-  const segment = decodeBody(body)
-  const size = segment.meta.begin.size
-  const contentType = sniffContentType(segment.data)
-  const probe: InspectProbe = {
-    size,
-    contentType,
-    md5_16k: chunkMd5(segment.data, size) ?? null,
-    yencName: segment.meta.begin.name,
-  }
-  // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
-  if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
-  if (NICOVIDEO_FILE.test(task.name)) {
-    probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
-    return probe
-  }
-  if (/\.sfv$/i.test(task.name)) {
-    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('latin1') ?? null
-    return probe
-  }
-  if (contentType === 'nzb') {
-    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('utf8') ?? null
-    return probe
-  }
-  probe.par2 = await readPar2(segment, task.segments, fetch)
-  if (probe.par2) return probe
-  if (contentType === '7z') {
-    probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
-    return probe
-  }
-  const info = parseRarHeaders(segment.data)
-  probe.archive = info
-  if (info && hasNzbEntries(info)) {
-    probe.archiveNzbs = []
-    const archive = await readWhole(segment, task.segments, fetch)
-    const archived = archive && parseRarHeaders(archive)
-    const entries = archived ? archived.entries.filter(isNzbEntry) : []
-    probe.archiveNzbEntries = entries.length
-    for (const entry of entries) {
-      const xml = await extractEntry(archive!, entry, undefined, warn)
-      if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
-    }
-  }
-  return probe
-}
-
-export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
-  const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
-  const walked = await walkRar(reader)
-  return {
-    entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
-    complete: walked.complete,
-    stopped: walked.stopped,
-    headersEncrypted: walked.headersEncrypted,
-    segments: reader.fetched,
-  }
-}
@@ -1,29 +0,0 @@
-/**
- * SFV checksum lists: one `name CRC32` per line, `;` comments. Scene
- * releases (music in particular) post one with every album; it names every
- * file that belongs to it.
- */
-export interface SfvEntry {
-  name: string
-  crc: string
-}
-
-export function parseSfv(text: string): SfvEntry[] {
-  const entries: SfvEntry[] = []
-  for (const raw of text.split(/\r?\n/)) {
-    const line = raw.trim()
-    if (!line || line.startsWith(';')) continue
-    const match = /^(.+?)\s+([0-9a-f]{8})$/i.exec(line)
-    if (match) entries.push({ name: match[1].replace(/^"|"$/g, '').split(/[\\/]/).pop()!, crc: match[2].toLowerCase() })
-  }
-  return entries
-}
-
-/**
- * The album name an SFV gives a release: its stem without the leading
- * track number scene groups put on the playlist/SFV/cover files
- * (`00-artist-album-web-2026-grp.sfv` → `artist-album-web-2026-grp`).
- */
-export function sfvReleaseName(sfvName: string): string {
-  return sfvName.replace(/\.sfv$/i, '').replace(/^0{1,3}-/, '')
-}
@@ -1,81 +0,0 @@
-/**
- * Recognises a file type from its first bytes, independent of AdonisJS.
- * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
- * "nzbe", 16 random bytes (salt/IV), then ciphertext.
- * `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
- * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
- */
-export type ContentType =
-  /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */
-  | 'missing'
-  | 'nzbe'
-  | 'par2'
-  | 'rar4'
-  | 'rar5'
-  | '7z'
-  | 'zip'
-  | 'zstd'
-  | 'gzip'
-  | 'xz'
-  | 'matroska'
-  | 'mp4'
-  | 'rclone'
-  | 'yencbin'
-  | 'iso'
-  | 'udf'
-  | 'pdf'
-  | 'nzb'
-  | 'jpeg'
-  | 'png'
-  | 'mp3'
-  | 'flac'
-  | 'avi'
-  | 'wav'
-  | 'ogg'
-  | 'bzip2'
-
-/** Magic bytes at an offset; null matches any byte. */
-const SIGNATURES: [ContentType, number, (number | null)[]][] = [
-  // Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
-  // the bytes they share.
-  ['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
-  ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
-  ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
-  ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
-  ['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]],
-  ['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
-  ['zip', 0, [0x50, 0x4b, 0x03, 0x04]],
-  ['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]],
-  ['gzip', 0, [0x1f, 0x8b]],
-  ['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]],
-  ['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]],
-  ['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box
-  ['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt
-  ['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]],
-  ['jpeg', 0, [0xff, 0xd8, 0xff]],
-  ['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]],
-  ['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag
-  ['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC"
-  ['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS"
-  ['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh"
-]
-
-export function sniffContentType(data: Buffer): ContentType | null {
-  for (const [type, offset, bytes] of SIGNATURES) {
-    if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
-  }
-  // Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
-  // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
-  if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso'
-  if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' &&
-      /^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf'
-  // RIFF container: AVI or WAV by form type.
-  if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') {
-    const form = data.subarray(8, 12).toString('latin1')
-    if (form === 'AVI ') return 'avi'
-    if (form === 'WAVE') return 'wav'
-  }
-  const head = data.subarray(0, 512).toString('latin1')
-  if (/^\s*(<\?xml[^>]*>\s*)?(<!DOCTYPE nzb|<nzb[\s>])/.test(head)) return 'nzb'
-  return null
-}