daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: fresh AdonisJS 7 API app on Postgres

Browse Source 
The rebuild branch starts over (REBUILD.md): the API starter kit, with
Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a
shared one in development) and the #lib alias. Phase 1's code is
imported from the current app module by module.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
264 files changed
+777 -24132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/sevenzip/index.ts
-164
View File
@@ -1,164 +0,0 @@
import { crc32 } from 'node:zlib'
/**
* 7-Zip archives, independent of AdonisJS. A 7z archive starts with a
* 32-byte signature header that points to the archive's header (the file
* list) at its very end, often LZMA-compressed ("encoded") with its packed
* data just before it. So the list needs the first 32 bytes and the tail:
* for a split archive (.7z.001, .002, …), the start of the first volume and
* the end of the last. Listing itself is left to libarchive (bsdtar).
*/
const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
const SIGNATURE_HEADER_SIZE = 32
/** Header property ids (7zFormat.txt). */
const K_END = 0x00
const K_PACK_INFO = 0x06
const K_UNPACK_INFO = 0x07
const K_SIZE = 0x09
const K_CRC = 0x0a
const K_FOLDER = 0x0b
const K_ENCODED_HEADER = 0x17
export interface SevenZipStart {
/** Where the archive header starts, from the start of the archive. */
headerStart: number
headerSize: number
headerCrc: number
/** The archive's total size (the header is its last part). */
archiveSize: number
}
/** Reads the signature header; null when `data` isn't a 7z start or its CRC fails. */
export function parseSevenZipStart(data: Buffer): SevenZipStart | null {
if (data.length < SIGNATURE_HEADER_SIZE || !data.subarray(0, 6).equals(SIGNATURE)) return null
if ((crc32(data.subarray(12, 32)) >>> 0) !== data.readUInt32LE(8)) return null
const offset = Number(data.readBigUInt64LE(12))
const size = Number(data.readBigUInt64LE(20))
const headerStart = SIGNATURE_HEADER_SIZE + offset
return { headerStart, headerSize: size, headerCrc: data.readUInt32LE(28), archiveSize: headerStart + size }
}
/** Whether `header` (the bytes at headerStart) is the archive header its signature header describes. */
export function checkHeader(start: SevenZipStart, header: Buffer): boolean {
return header.length === start.headerSize && (crc32(header) >>> 0) === start.headerCrc
}
/**
* Where the bytes needed for listing start: the header itself, or for an
* encoded header the packed (compressed) header stream it points to.
*/
export function listingStart(start: SevenZipStart, header: Buffer): number {
if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return start.headerStart
const [packPos] = readNumber(header, 2)
return SIGNATURE_HEADER_SIZE + packPos
}
/** 7zAES, the coder of password-protected streams. */
const AES_CODER = Buffer.from([0x06, 0xf1, 0x07, 0x01])
/**
* Whether an encoded header (`header`, the bytes at headerStart) is
* encrypted (7z -mhe): its coders include 7zAES. Then nothing can be listed
* without the password, and neither the packed header nor bsdtar is needed.
* Null when the header isn't encoded or doesn't read as expected.
*/
export function encodedHeaderEncrypted(header: Buffer): boolean | null {
try {
if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return null
let pos = 2
let numPackStreams: number
;[, pos] = readNumber(header, pos) // packPos
;[numPackStreams, pos] = readNumber(header, pos)
for (let id = header[pos++]; id !== K_END; id = header[pos++]) {
if (id === K_SIZE) {
for (let i = 0; i < numPackStreams; i++) [, pos] = readNumber(header, pos)
} else if (id === K_CRC) {
const allDefined = header[pos++]
let defined = numPackStreams
if (!allDefined) {
const bits = header.subarray(pos, pos + Math.ceil(numPackStreams / 8))
pos += bits.length
defined = [...bits].reduce((sum, byte) => sum + byte.toString(2).replaceAll('0', '').length, 0)
}
pos += 4 * defined
} else {
return null
}
if (pos >= header.length) return null
}
if (header[pos++] !== K_UNPACK_INFO || header[pos++] !== K_FOLDER) return null
let numFolders: number
;[numFolders, pos] = readNumber(header, pos)
if (numFolders < 1 || header[pos++] !== 0) return null // folders stored elsewhere
let numCoders: number
;[numCoders, pos] = readNumber(header, pos)
if (!(numCoders >= 1) || pos >= header.length) return null
for (let c = 0; c < numCoders; c++) {
const flags = header[pos++]
const idSize = flags & 0x0f
if (header.subarray(pos, pos + idSize).equals(AES_CODER)) return true
pos += idSize
if (flags & 0x10) {
;[, pos] = readNumber(header, pos)
;[, pos] = readNumber(header, pos)
}
if (flags & 0x20) {
let size: number
;[size, pos] = readNumber(header, pos)
pos += size
}
if (pos > header.length) return null
}
return false
} catch {
return null
}
}
/**
* 7z's variable-length number: the leading 1-bits of the first byte count
* the little-endian bytes that follow; its remaining bits are the high part.
*/
export function readNumber(data: Buffer, pos: number): [value: number, next: number] {
const first = data[pos]
let mask = 0x80
let value = 0
for (let i = 0; i < 8; i++) {
if ((first & mask) === 0) {
value += (first & (mask - 1)) * 2 ** (8 * i)
return [value, pos + 1 + i]
}
value += data[pos + 1 + i] * 2 ** (8 * i)
mask >>= 1
}
return [value, pos + 9]
}
export interface SevenZipEntry {
name: string
size: number
directory: boolean
}
/**
* Entries from `bsdtar -tv` output, e.g.
* `-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release/movie.mkv`.
*/
export function parseBsdtarListing(stdout: string): SevenZipEntry[] {
const entries: SevenZipEntry[] = []
for (const line of stdout.split('\n')) {
const match = /^([-dlbcps])\S*\s+\d+\s+\S+\s+\S+\s+(\d+)\s+\w{3}\s+\d{1,2}\s+(?:\d{1,2}:\d{2}|\d{4})\s(.+)$/.exec(line)
if (!match) continue
const directory = match[1] === 'd'
entries.push({ name: directory ? match[3].replace(/\/$/, '') : match[3], size: Number(match[2]), directory })
}
return entries
}
/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */
export function sevenZipVolume(name: string): number | null {
const split = /\.7z\.(\d{1,4})$/i.exec(name)
if (split) return Number(split[1])
return /\.7z$/i.test(name) ? 0 : null
}
lib/sevenzip/list.ts
-171
View File
@@ -1,171 +0,0 @@
import { execFile } from 'node:child_process'
import { mkdtemp, open, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import { decodeSegment, type DecodedSegment } from '../yenc/index.js'
import type { RarInfo } from '../rar/parse.js'
import type { BodyFetcher } from '../rar/segment_reader.js'
import {
checkHeader,
encodedHeaderEncrypted,
listingStart,
parseBsdtarListing,
parseSevenZipStart,
type SevenZipStart,
} from './index.js'
const run = promisify(execFile)
/** The tail read for a listing: header plus packed header; more is not a header. */
const MAX_TAIL_BYTES = 16 * 1024 * 1024
/**
* Lists a 7z archive, also a split one (.7z.001, .002, …), from the first
* segment of the first volume (`firstData`, decoded) and the last segment(s)
* of the last one (CONCEPT.md D19). Those bytes are put at their place in a
* sparse file the size of the whole archive, which bsdtar (libarchive)
* lists: it only reads the start and the header at the end. Null when it
* isn't a 7z start, or the tail doesn't hold the header its start points to.
*/
export async function listSevenZip(
firstData: Buffer,
volumeCount: number,
lastVolume: { number: number; id: string }[],
fetch: BodyFetcher,
warn: (message: string) => void = () => {}
): Promise<RarInfo | null> {
const start = parseSevenZipStart(firstData)
if (!start) return null
const split = volumeCount > 1
if (start.headerSize === 0) return sevenZipInfo([], false, false, split)
const tail = await readTail(lastVolume, start, fetch)
if (!tail) return null
// Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
if ('headersEncrypted' in tail) return sevenZipInfo([], true, true, split)
const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
try {
const archive = path.join(dir, 'archive.7z')
const handle = await open(archive, 'w')
try {
await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
await handle.write(tail.data, 0, tail.data.length, tail.from)
await handle.truncate(start.archiveSize)
} finally {
await handle.close()
}
return await bsdtar(archive, split, warn)
} finally {
await rm(dir, { recursive: true, force: true })
}
}
/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
async function bsdtar(archive: string, split: boolean, warn: (message: string) => void): Promise<RarInfo | null> {
let stdout = ''
try {
stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
} catch (error: any) {
if (/header is encrypted/i.test(String(error.stderr))) return sevenZipInfo([], true, true, split)
warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
return null
}
const entries = parseBsdtarListing(stdout)
const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
let encrypted = false
if (sample) {
try {
await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
} catch (error: any) {
// Without the data (it's all zeros here), anything else fails too: only this message counts.
encrypted = /content is encrypted/i.test(String(error.stderr))
}
}
return sevenZipInfo(entries, false, encrypted, split)
}
export function sevenZipInfo(
entries: { name: string; size: number; directory: boolean }[],
headersEncrypted: boolean,
encrypted: boolean,
split: boolean
): RarInfo {
return {
format: '7z',
volume: split,
firstVolume: true,
solid: false,
headersEncrypted,
truncated: false,
next: 0,
entries: entries.map((e) => ({
name: e.name,
size: e.size,
packedSize: 0,
directory: e.directory,
encrypted: encrypted && !e.directory,
splitBefore: false,
splitAfter: false,
stored: false,
dataOffset: 0,
})),
}
}
/**
* The archive's tail from the last volume: the header (checked against its
* CRC) and, for an encoded header, the packed stream before it. `from` is
* its position in the whole archive.
*/
async function readTail(
last: { number: number; id: string }[],
start: SevenZipStart,
fetch: BodyFetcher
): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
const ids = new Map(last.map((s) => [s.number, s]))
const numbers = [...ids.keys()]
if (numbers.length === 0) return null
const segments = new Map<number, DecodedSegment>()
const segment = async (n: number) => {
if (!segments.has(n)) {
const id = ids.get(n)?.id
if (!id) throw new Error(`part ${n} of the last 7z volume is missing`)
segments.set(n, decodeSegment(await fetch(id), { lenientSize: true }))
}
return segments.get(n)!
}
const lastNumber = numbers[numbers.length - 1]
const lastSegment = await segment(lastNumber)
const volumeSize = lastSegment.meta.begin.size
const volumeStart = start.archiveSize - volumeSize
const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
// Bytes [from, to) of the whole archive, from the last volume's parts.
const read = async (from: number, to: number): Promise<Buffer | null> => {
if (from < volumeStart) return null
const a = from - volumeStart
const b = to - volumeStart
const pieces: Buffer[] = []
for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
const s = await segment(n)
const offset = (s.meta.part?.begin ?? 1) - 1
if (offset !== (n - 1) * partSize) return null
pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
}
const data = Buffer.concat(pieces)
return data.length === b - a ? data : null
}
const header = await read(start.headerStart, start.archiveSize)
if (!header || !checkHeader(start, header)) return null
if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
const from = Math.min(listingStart(start, header), start.headerStart)
if (start.archiveSize - from > MAX_TAIL_BYTES) return null
const data = await read(from, start.archiveSize)
return data ? { from, data } : null
}