daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: fresh AdonisJS 7 API app on Postgres

Browse Source 
The rebuild branch starts over (REBUILD.md): the API starter kit, with
Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a
shared one in development) and the #lib alias. Phase 1's code is
imported from the current app module by module.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
264 files changed
+777 -24132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/nntp/date_search.ts
-27
View File
@@ -1,27 +0,0 @@
/**
* The article number where a group's posts reach `target` (ms since epoch),
* by binary search between `first` and `last`. Article numbers only roughly
* follow posting time, so the answer is within `precision` articles of the
* first one posted at or after `target`.
*
* `dateOf(n)` returns the posting time of the first article at or after `n`
* that still exists, or null when there is none nearby (expired or removed);
* a gap counts as older than `target`.
*/
export async function articleAtDate(
first: bigint,
last: bigint,
target: number,
dateOf: (n: bigint) => Promise<number | null>,
precision = 10_000n
): Promise<bigint> {
let lo = first
let hi = last
while (hi - lo > precision) {
const mid = lo + (hi - lo) / 2n
const date = await dateOf(mid)
if (date === null || date < target) lo = mid
else hi = mid
}
return lo
}
lib/nntp/delimited_reader.ts
-76
View File
@@ -1,76 +0,0 @@
const CRLF = Buffer.from('\r\n')
/**
* Splits a byte stream into messages that end in a delimiter, which the
* consumer can change per message: `next(delimiter)` returns everything up
* to the next `delimiter` (excluded). A replacement for nntp-js's
* findByteSequence, which has two bugs:
*
* - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
* without articles) never ends: the status line takes the CRLF that
* `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
* follows one that ended with CRLF, that CRLF counts for both.
* - A delimiter split over two socket chunks left its first bytes in the
* returned data.
*
* Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
*/
export async function* delimitedReader(
source: AsyncIterable<Uint8Array>,
initial: Uint8Array
): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
let delimiter = Buffer.from(initial)
let parts: Buffer[] = []
let length = 0
// Bytes at the start of the buffer, already searched without a match.
let searched = 0
// Leading bytes that belong to the previous delimiter (a shared CRLF).
let skip = 0
for await (const chunk of source) {
parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
length += chunk.byteLength
while (true) {
const from = Math.max(0, searched - (delimiter.length - 1))
const found = indexOf(parts, length, delimiter, from)
if (found === -1) {
searched = length
break
}
const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
const message = all.subarray(Math.min(skip, found), found)
const rest = all.subarray(found + delimiter.length)
const previous = delimiter
if (message.length === 0 && delimiter.equals(CRLF)) {
// A stray empty line: keep looking for the same delimiter.
} else {
delimiter = Buffer.from((yield message) ?? delimiter)
}
const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
parts = shareCrlf ? [CRLF, rest] : [rest]
length = rest.length + (shareCrlf ? 2 : 0)
skip = shareCrlf ? 2 : 0
searched = 0
}
}
if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
}
const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
if (parts.length === 1) return parts[0].indexOf(needle, from)
// Collect the parts that cover [from, length).
let start = length
let i = parts.length
while (i > 0 && start > from) start -= parts[--i].length
const window = Buffer.concat(parts.slice(i), length - start)
const found = window.indexOf(needle, from - start)
return found === -1 ? -1 : start + found
}
lib/nntp/group.ts
-23
View File
@@ -1,23 +0,0 @@
export interface GroupInfo {
count: bigint
first: bigint
last: bigint
name: string
}
/**
* Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1).
*
* nntp-js's own `group()` destructures this reply without skipping the status
* code, so its `first`/`last` are off by one field. Pass it `result.resp`.
*/
export function parseGroupResponse(resp: string): GroupInfo {
const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim())
if (!match) throw new Error(`Unexpected GROUP response: ${resp}`)
return {
count: BigInt(match[1]),
first: BigInt(match[2]),
last: BigInt(match[3]),
name: match[4],
}
}
lib/nntp/nntp_js.ts
-39
View File
@@ -1,39 +0,0 @@
import { NNTP } from 'nntp-js'
import { delimitedReader } from './delimited_reader.js'
const CRLF = new Uint8Array([13, 10])
/**
* nntp-js with its reply reader replaced by `delimitedReader` (see there for
* the bugs it fixes; the empty-reply one hung XOVER of empty ranges forever).
* The reader is created in `connect`, so that is overridden: the same steps
* as nntp-js 1.0.4, with the other reader.
*/
const proto = NNTP.prototype as any
if (!proto.__delimitedReader) {
proto.__delimitedReader = true
proto.connect = async function (this: any, readermode = false, timeout: number | undefined = undefined, socket?: any) {
try {
this.sock = socket ?? this._createSocket(timeout)
// The first reply is always the one-line welcome.
this.byteReader = delimitedReader(this.sock, CRLF)
this.welcome = new TextDecoder('utf-8').decode(await this._getresp())
this.caps = undefined
this.connected = true
await this.getcapabilities()
this.readermodeAfterauth = false
if (readermode && !this.caps.READER) {
await this._setreadermode()
if (!this.readermodeAfterauth) {
this.caps = undefined
await this.getcapabilities()
}
}
} catch (error) {
this.sock?.end()
throw error
}
}
}
export { NNTP }
lib/nntp/xover_client.ts
-238
View File
@@ -1,238 +0,0 @@
import { promisify } from 'node:util'
import net from 'node:net'
import tls from 'node:tls'
import { deflate, deflateSync, unzipSync } from 'node:zlib'
const deflateAsync = promisify(deflate)
export interface XoverClientOptions {
host: string
port: number
user?: string
password?: string
secure?: boolean
/** Hostname the TLS certificate must match, when connecting through a tunnel (default: host). */
servername?: string
/** A reply with no data for this long fails, and the connection with it (inactivity, not duration). */
timeoutMs?: number
/** Ask for gzip-compressed overview data (XFEATURE COMPRESS GZIP), when the server has it. */
compress?: boolean
}
export type Overview = [number, Record<string, string>]
/** The overview fields in RFC 3977 order, named as nntp-js names them (segmentFromHeader reads these). */
const FIELDS = ['subject', 'from', 'date', 'message-id', 'references', ':bytes', ':lines', 'xref']
const TERMINATOR = Buffer.from('\r\n.\r\n')
const DOT_LINE = Buffer.from('.\r\n')
/**
* A minimal NNTP client for seeding nodes (CONCEPT.md D37): one connection,
* GROUP and XOVER only, with gzip-compressed overviews. nntp-js can't read
* those: after "224 … [COMPRESS=GZIP]" the lines come as one zlib stream,
* followed by a plain "." line; an empty range is a 420 without body.
* About six times less traffic per header (LOG.md 2026-10-05).
*/
export class XoverClient {
private chunks: Buffer[] = []
private length = 0
private wake?: () => void
private failure?: Error
compressed = false
private constructor(
private sock: net.Socket,
private timeoutMs: number
) {
sock.on('data', (chunk: Buffer) => {
this.chunks.push(chunk)
this.length += chunk.length
this.wake?.()
})
const fail = (error: Error) => {
this.failure ??= error
this.wake?.()
}
sock.on('error', fail)
sock.on('close', () => fail(new Error('NNTP connection closed')))
sock.on('timeout', () => {
fail(new Error('NNTP reply timed out'))
sock.destroy()
})
}
static async connect(options: XoverClientOptions): Promise<XoverClient> {
const sock = options.secure
? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host })
: net.connect({ host: options.host, port: options.port })
const client = new XoverClient(sock, options.timeoutMs ?? 120_000)
sock.setTimeout(client.timeoutMs)
try {
await client.expect(await client.line(), ['200', '201'])
if (options.user) {
const reply = await client.command(`AUTHINFO USER ${options.user}`)
if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281'])
else await client.expect(reply, ['281'])
}
if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
} catch (error) {
client.close()
throw error
}
// Idle between commands is fine (a node waiting for the seed API); only replies are timed.
sock.setTimeout(0)
return client
}
async group(name: string): Promise<{ count: number; low: number; high: number }> {
return this.timed(() => this.selectGroup(name))
}
/** Overviews of articles `from`..`to` in the current group; an empty range gives []. */
async xover(from: number, to: number): Promise<Overview[]> {
return this.timed(() => this.overviews(from, to))
}
/**
* Overviews of `from`..`to` as one zlib/gzip stream, as the server sent it when it
* compresses (plain replies are deflated here): the seed API passes these to nodes
* without unpacking them (CONCEPT.md D38). `parseOverviews(unzipSync(raw).toString())`.
*/
async xoverRaw(from: number, to: number): Promise<Buffer> {
return this.timed(async () => {
const reply = await this.command(`XOVER ${from}-${to}`)
if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0))
await this.expect(reply, ['224'])
if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw
// Plain replies (the relay's default): compressed off the event loop, at the fastest level.
return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
})
}
/** A reply that stalls for timeoutMs fails the connection; idle time between commands doesn't count. */
private async timed<T>(exchange: () => Promise<T>): Promise<T> {
this.sock.setTimeout(this.timeoutMs)
try {
return await exchange()
} finally {
this.sock.setTimeout(0)
}
}
private async selectGroup(name: string): Promise<{ count: number; low: number; high: number }> {
const reply = await this.command(`GROUP ${name}`)
await this.expect(reply, ['211'])
const [, count, low, high] = reply.split(' ')
return { count: Number(count), low: Number(low), high: Number(high) }
}
private async overviews(from: number, to: number): Promise<Overview[]> {
const reply = await this.command(`XOVER ${from}-${to}`)
if (reply.startsWith('420') || reply.startsWith('423')) return []
await this.expect(reply, ['224'])
const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody()
return parseOverviews(data)
}
close() {
try {
this.sock.end('QUIT\r\n')
} catch {
// already closed
}
this.sock.destroy()
}
private async command(line: string): Promise<string> {
this.sock.write(`${line}\r\n`)
return this.line()
}
private async expect(reply: string, codes: string[]) {
if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`)
}
private buffer(): Buffer {
if (this.chunks.length > 1) this.chunks = [Buffer.concat(this.chunks, this.length)]
return this.chunks[0] ?? Buffer.alloc(0)
}
private take(n: number): Buffer {
const all = this.buffer()
const out = all.subarray(0, n)
const rest = all.subarray(n)
this.chunks = rest.length > 0 ? [rest] : []
this.length = rest.length
return out
}
private async more() {
if (this.failure) throw this.failure
await new Promise<void>((resolve) => (this.wake = resolve))
this.wake = undefined
if (this.failure && this.length === 0) throw this.failure
}
private async line(): Promise<string> {
while (true) {
const at = this.buffer().indexOf('\r\n')
if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1')
await this.more()
}
}
/** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
private async plainBody(): Promise<string> {
let searched = 0
while (true) {
const all = this.buffer()
// An empty body is just ".\r\n".
if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
this.take(3)
return ''
}
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.')
searched = all.length
await this.more()
}
}
private async compressedBody(): Promise<string> {
return (await this.compressedRaw()).text
}
/** A zlib/gzip stream followed by a "." line: complete once it ends in ".\r\n" and inflates. */
private async compressedRaw(): Promise<{ raw: Buffer; text: string }> {
while (true) {
const all = this.buffer()
if (all.length > DOT_LINE.length && all.subarray(all.length - DOT_LINE.length).equals(DOT_LINE)) {
const raw = all.subarray(0, all.length - DOT_LINE.length)
try {
const text = unzipSync(raw).toString('utf8')
this.take(all.length)
return { raw: Buffer.from(raw), text }
} catch {
// ".\r\n" inside the compressed data: wait for more.
}
}
await this.more()
}
}
}
/** Overview lines (tab-separated, after the article number) as nntp-js returns them. */
export function parseOverviews(data: string): Overview[] {
const out: Overview[] = []
for (const line of data.split('\r\n')) {
if (!line) continue
const [number, ...tokens] = line.split('\t')
const fields: Record<string, string> = {}
for (let i = 0; i < FIELDS.length && i < tokens.length; i++) {
// Full fields after the defaults carry their name: "Xref: host group:number".
fields[FIELDS[i]] = i >= 7 ? tokens[i].replace(/^[^:\s]+:\s*/, '') : tokens[i]
}
out.push([Number.parseInt(number, 10), fields])
}
return out
}
lib/nntp/xref.ts
-13
View File
@@ -1,13 +0,0 @@
/**
* Returns the group names from an Xref header,
* e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`.
*/
export function parseXrefGroups(xref: string | undefined): string[] {
if (!xref) return []
return xref
.trim()
.split(/\s+/)
.slice(1)
.map((entry) => entry.slice(0, entry.lastIndexOf(':')))
.filter(Boolean)
}