daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: fresh AdonisJS 7 API app on Postgres

Browse Source 
The rebuild branch starts over (REBUILD.md): the API starter kit, with
Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a
shared one in development) and the #lib alias. Phase 1's code is
imported from the current app module by module.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
264 files changed
+777 -24132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/rar/extract.ts
-48
View File
@@ -1,48 +0,0 @@
import { execFile } from 'node:child_process'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import { storedEntryData, type RarEntry } from './parse.js'
const run = promisify(execFile)
/** Tools that print one archive entry to stdout, tried in order. */
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
]
/**
* The unpacked data of one entry of a complete (single-volume) archive:
* copied out when stored, else unpacked with bsdtar (libarchive, in the
* Docker image) or unrar. Null when that is not possible.
*/
export async function extractEntry(
archive: Buffer,
entry: RarEntry,
maxBytes = 64 * 1024 * 1024,
warn: (message: string) => void = () => {}
): Promise<Buffer | null> {
const stored = storedEntryData(archive, entry)
if (stored) return stored
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
if (entry.size !== null && entry.size > maxBytes) return null
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
try {
const file = path.join(dir, 'archive.rar')
await writeFile(file, archive)
for (const [tool, args] of EXTRACTORS) {
try {
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
if (stdout.length > 0) return stdout
} catch (error: any) {
if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
}
}
return null
} finally {
await rm(dir, { recursive: true, force: true })
}
}
lib/rar/index.ts
-8
View File
@@ -1,8 +0,0 @@
/**
* RAR (4.x and 5.0) header parsing, independent of AdonisJS. Lists archive
* contents from the first segment of a first volume. See CONCEPT.md (D19).
*/
export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
export { walkRar, type VolumeReader, type RarWalk } from './walk.js'
export { SegmentReader, type BodyFetcher } from './segment_reader.js'
export { extractEntry } from './extract.js'
lib/rar/parse.ts
-351
View File
@@ -1,351 +0,0 @@
import { crc32 } from 'node:zlib'
export interface RarEntry {
name: string
/** Unpacked size in bytes (null when the archiver did not record it). */
size: number | null
packedSize: number
directory: boolean
/** The file's data is password protected. */
encrypted: boolean
/** Continues from the previous volume. */
splitBefore: boolean
/** Continues in the next volume. */
splitAfter: boolean
/** Stored without compression, so the data can be copied out as is. */
stored: boolean
/** Offset of the entry's (packed) data in the parsed buffer. */
dataOffset: number
}
export interface RarInfo {
/** 7z listings take the same shape (see lib/sevenzip). */
format: 'rar4' | 'rar5' | '7z'
/** Part of a multi-volume set. */
volume: boolean
/** First volume of the set (null when the format version does not say). */
firstVolume: boolean | null
solid: boolean
/** File names are encrypted too; nothing can be listed without the password. */
headersEncrypted: boolean
entries: RarEntry[]
/** The buffer ended before the end-of-archive header (normal for one segment). */
truncated: boolean
/**
* Offset of the first block that was not read: where the next header
* starts (possibly beyond the buffer, after an entry's data), or the
* header the buffer cut off. After the end-of-archive header, its end.
*/
next: number
}
const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
return null
}
/**
* Lists a RAR archive from its headers only, so the first segment of a first
* volume is enough. Returns null when the data is not a RAR archive.
*
* With `continueAs`, `data` starts at a header in the middle of a volume (no
* signature), e.g. where a previous parse said the next header is.
*/
export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
if (continueAs === 'rar4') return parseRar4(data, 0)
if (continueAs === 'rar5') return parseRar5(data, 0)
const format = rarFormat(data)
if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
return null
}
// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
const R4_MAIN = 0x73
const R4_FILE = 0x74
const R4_END = 0x7b
const R4_LONG_BLOCK = 0x8000
function parseRar4(data: Buffer, start: number): RarInfo {
const info: RarInfo = {
format: 'rar4',
volume: false,
firstVolume: null,
solid: false,
headersEncrypted: false,
entries: [],
truncated: true,
next: start,
}
let p = start
while (p + 7 <= data.length) {
const headCrc = data.readUInt16LE(p)
const type = data[p + 2]
const flags = data.readUInt16LE(p + 3)
const size = data.readUInt16LE(p + 5)
if (size < 7 || p + size > data.length) break
if (type === R4_END) {
info.truncated = false
p += size
break
}
let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
if (type === R4_MAIN) {
info.volume = Boolean(flags & 0x0001)
info.solid = Boolean(flags & 0x0008)
info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
if (flags & 0x0080) {
info.headersEncrypted = true
break
}
} else if (type === R4_FILE) {
if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
const large = Boolean(flags & 0x0100)
let packed = data.readUInt32LE(p + 7)
let unpacked = data.readUInt32LE(p + 11)
const nameSize = data.readUInt16LE(p + 26)
let nameStart = p + 32
if (large) {
packed += data.readUInt32LE(p + 32) * 2 ** 32
unpacked += data.readUInt32LE(p + 36) * 2 ** 32
nameStart = p + 40
}
const rawName = data.subarray(nameStart, nameStart + nameSize)
info.entries.push({
name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
// 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
size: unpacked === 0xffffffff && !large ? null : unpacked,
packedSize: packed,
directory: (flags & 0x00e0) === 0x00e0,
encrypted: Boolean(flags & 0x0004),
splitBefore: Boolean(flags & 0x0001),
splitAfter: Boolean(flags & 0x0002),
stored: data[p + 25] === 0x30,
dataOffset: p + size,
})
next = p + size + packed
}
if (next <= p) break
p = next
}
// Archivers before RAR 3.0 don't set the first-volume flag; a first
// volume is one whose first file does not continue from a previous one.
if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
info.firstVolume = true
}
info.next = p
return info
}
/**
* RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
* compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
*/
export function decodeRar4Name(raw: Buffer): string {
const zero = raw.indexOf(0)
if (zero === -1) return raw.toString('utf8')
const ascii = raw.subarray(0, zero)
const enc = raw.subarray(zero + 1)
if (enc.length === 0) return ascii.toString('latin1')
const out: number[] = []
const highByte = enc[0]
let encPos = 1
let flags = 0
let flagBits = 0
while (encPos < enc.length) {
if (flagBits === 0) {
flags = enc[encPos++]
flagBits = 8
}
switch (flags >> 6) {
case 0:
out.push(enc[encPos++])
break
case 1:
out.push(enc[encPos++] + (highByte << 8))
break
case 2:
out.push(enc[encPos] + (enc[encPos + 1] << 8))
encPos += 2
break
case 3: {
let length = enc[encPos++]
if (length & 0x80) {
const correction = enc[encPos++]
for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
}
} else {
for (length += 2; length > 0 && out.length < ascii.length; length--) {
out.push(ascii[out.length])
}
}
break
}
}
flags = (flags << 2) & 0xff
flagBits -= 2
}
return String.fromCharCode(...out)
}
// --- RAR 5.0 ----------------------------------------------------------------
const R5_MAIN = 1
const R5_FILE = 2
const R5_ENCRYPTION = 4
const R5_END = 5
/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
function vint(data: Buffer, offset: number): { value: number; next: number } | null {
let value = 0
let shift = 1
for (let i = offset; i < data.length && i < offset + 10; i++) {
value += (data[i] & 0x7f) * shift
shift *= 128
if (!(data[i] & 0x80)) return { value, next: i + 1 }
}
return null
}
function parseRar5(data: Buffer, start: number): RarInfo {
const info: RarInfo = {
format: 'rar5',
volume: false,
firstVolume: null,
solid: false,
headersEncrypted: false,
entries: [],
truncated: true,
next: start,
}
let p = start
while (p + 5 <= data.length) {
const headCrc = data.readUInt32LE(p)
const headerSize = vint(data, p + 4)
if (!headerSize || headerSize.value === 0) break
const headerEnd = headerSize.next + headerSize.value
if (headerEnd > data.length) break
if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
const type = vint(data, headerSize.next)
const flags = type && vint(data, type.next)
if (!type || !flags) break
let cursor = flags.next
let extraSize = 0
let dataSize = 0
if (flags.value & 0x0001) {
const v = vint(data, cursor)
if (!v) break
extraSize = v.value
cursor = v.next
}
if (flags.value & 0x0002) {
const v = vint(data, cursor)
if (!v) break
dataSize = v.value
cursor = v.next
}
if (type.value === R5_END) {
info.truncated = false
p = headerEnd
break
}
if (type.value === R5_ENCRYPTION) {
info.headersEncrypted = true
break
}
if (type.value === R5_MAIN) {
const archiveFlags = vint(data, cursor)
if (!archiveFlags) break
info.volume = Boolean(archiveFlags.value & 0x0001)
// A volume number is only stored for volumes after the first.
info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
info.solid = Boolean(archiveFlags.value & 0x0004)
} else if (type.value === R5_FILE) {
const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
if (!entry) break
info.entries.push(entry)
}
const next = headerEnd + dataSize
if (next <= p) break
p = next
}
info.next = p
return info
}
function parseRar5File(
data: Buffer,
start: number,
extraStart: number,
headerEnd: number,
headerFlags: number,
dataSize: number
): RarEntry | null {
const fileFlags = vint(data, start)
const unpacked = fileFlags && vint(data, fileFlags.next)
const attributes = unpacked && vint(data, unpacked.next)
if (!fileFlags || !unpacked || !attributes) return null
let cursor = attributes.next
if (fileFlags.value & 0x0002) cursor += 4 // mtime
if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
const compression = vint(data, cursor)
const hostOs = compression && vint(data, compression.next)
const nameLength = hostOs && vint(data, hostOs.next)
if (!compression || !hostOs || !nameLength) return null
const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
// Compression information: bits 7-9 are the method, 0 is "store".
const method = Math.floor(compression.value / 128) & 0x7
// Extra area records: size, type, data. Type 1 is file encryption.
let encrypted = false
let e = extraStart
while (e < headerEnd) {
const recordSize = vint(data, e)
const recordType = recordSize && vint(data, recordSize.next)
if (!recordSize || !recordType || recordSize.value === 0) break
if (recordType.value === 0x01) encrypted = true
e = recordSize.next + recordSize.value
}
return {
name,
size: fileFlags.value & 0x0008 ? null : unpacked.value,
packedSize: dataSize,
directory: Boolean(fileFlags.value & 0x0001),
encrypted,
splitBefore: Boolean(headerFlags & 0x0008),
splitAfter: Boolean(headerFlags & 0x0010),
stored: method === 0,
dataOffset: headerEnd,
}
}
/**
* The data of an entry stored without compression, copied out of the
* archive buffer. Null when it is compressed, encrypted, split over volumes
* or not completely in the buffer.
*/
export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
if (entry.size === null || entry.size !== entry.packedSize) return null
const end = entry.dataOffset + entry.packedSize
if (end > data.length) return null
return data.subarray(entry.dataOffset, end)
}
lib/rar/segment_reader.ts
-82
View File
@@ -1,82 +0,0 @@
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
import type { VolumeReader } from './walk.js'
/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
export type BodyFetcher = (messageId: string) => Promise<Buffer>
/**
* Reads volumes (each a list of segment message ids, in order) a segment at
* a time, finding the segment for an offset from the part size (yEnc
* posters use one size per set) and correcting from the `=ypart` offsets
* when a guess is off. Stops reading after `maxSegments` downloads.
*/
export class SegmentReader implements VolumeReader {
fetched = 0
private partSize = 0
private readonly sizes = new Map<number, number>()
private readonly cache = new Map<string, DecodedSegment>()
constructor(
private readonly files: { id: string }[][],
private readonly maxSegments: number,
private readonly fetchBody: BodyFetcher
) {}
get volumes() {
return this.files.length
}
size(v: number) {
return this.sizes.get(v) ?? null
}
async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
const count = this.files[v].length
let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
for (let tries = 0; tries < 4; tries++) {
const segment = await this.segment(v, index)
if (!segment) return null
const end = segment.offset + segment.data.length
if (offset >= segment.offset && offset < end) {
const data = segment.data.subarray(offset - segment.offset)
if (!more || index + 1 >= count) return data
const next = await this.segment(v, index + 1)
return next ? Buffer.concat([data, next.data]) : null
}
const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
if (moved === index) return null
index = moved
}
return null
}
private async segment(v: number, index: number): Promise<DecodedSegment | null> {
const key = `${v}:${index}`
const cached = this.cache.get(key)
if (cached) return cached
const part = this.files[v][index]
if (!part?.id || this.fetched >= this.maxSegments) return null
this.fetched++
let body: Buffer
try {
body = await this.fetchBody(part.id)
} catch {
// A missing article (or the relay failing) ends the walk with what was found.
return null
}
let segment: DecodedSegment
try {
segment = decodeSegment(body)
} catch (error) {
if (!(error instanceof YencError) || error instanceof YencCrcError) return null
segment = decodeSegment(body, { lenientSize: true })
}
this.sizes.set(v, segment.meta.begin.size)
if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
// Walks move forward: the last few segments are enough.
if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
this.cache.set(key, segment)
return segment
}
}
lib/rar/walk.ts
-77
View File
@@ -1,77 +0,0 @@
import { parseRarHeaders, rarFormat, type RarEntry, type RarInfo } from './parse.js'
/** Reads the volumes of a RAR set, a segment at a time. */
export interface VolumeReader {
readonly volumes: number
/**
* Bytes of volume `v` from `offset` to the end of the segment that holds
* it; with `more`, the following segment too (for a header cut off by the
* segment end). Null when that can't be read (missing article, budget).
*/
read(v: number, offset: number, more: boolean): Promise<Buffer | null>
/** Size of volume `v`, known once something of it was read. */
size(v: number): number | null
}
export interface RarWalk {
format: 'rar4' | 'rar5' | null
/** One entry per name, from its first header. */
entries: RarEntry[]
/** Every volume was walked to its end. */
complete: boolean
headersEncrypted: boolean
/** Why the walk ended early. */
stopped?: 'unreadable' | 'invalid' | 'encrypted'
}
/**
* Lists a RAR set completely by hopping from header to header (CONCEPT.md
* D40). Each header says how much data follows it, so the next header's
* offset is known and only the segment holding it is read: one segment per
* entry and one per volume, instead of the whole set. Split files repeat
* their header at the start of every volume they continue in.
*/
export async function walkRar(reader: VolumeReader): Promise<RarWalk> {
const seen = new Map<string, RarEntry>()
const result = (stopped?: RarWalk['stopped']): RarWalk => ({
format,
entries: [...seen.values()],
complete: !stopped,
headersEncrypted: stopped === 'encrypted',
stopped,
})
let format: 'rar4' | 'rar5' | null = null
let v = 0
let offset = 0
while (v < reader.volumes) {
let info: RarInfo | null = null
let reached = -1
for (const more of [false, true]) {
const data = await reader.read(v, offset, more)
if (!data) return result('unreadable')
if (offset === 0) format = rarFormat(data) ?? format
info = offset === 0 ? parseRarHeaders(data) : format && parseRarHeaders(data, format)
if (!info) return result('invalid')
// Stopped inside the data: a header the segment end cut off (read on), or a bad one.
if (!info.truncated || info.next >= data.length || info.headersEncrypted) break
// Read on and got further: the next header is cut off at the end of the following segment.
if (more && info.next > reached) break
if (more) return result('invalid')
reached = info.next
}
if (info!.headersEncrypted) return result('encrypted')
for (const entry of info!.entries) if (!seen.has(entry.name)) seen.set(entry.name, entry)
const next = offset + info!.next
const size = reader.size(v)
// The end-of-archive header closes every volume; data running to the end does too.
if (!info!.truncated || size === null || next >= size) {
v++
offset = 0
} else {
offset = next
}
}
return result()
}