Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
3.0 KiB
TypeScript
77 lines
3.0 KiB
TypeScript
const CRLF = Buffer.from('\r\n')
|
|
|
|
/**
|
|
* Splits a byte stream into messages that end in a delimiter, which the
|
|
* consumer can change per message: `next(delimiter)` returns everything up
|
|
* to the next `delimiter` (excluded). A replacement for nntp-js's
|
|
* findByteSequence, which has two bugs:
|
|
*
|
|
* - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
|
|
* without articles) never ends: the status line takes the CRLF that
|
|
* `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
|
|
* follows one that ended with CRLF, that CRLF counts for both.
|
|
* - A delimiter split over two socket chunks left its first bytes in the
|
|
* returned data.
|
|
*
|
|
* Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
|
|
*/
|
|
export async function* delimitedReader(
|
|
source: AsyncIterable<Uint8Array>,
|
|
initial: Uint8Array
|
|
): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
|
|
let delimiter = Buffer.from(initial)
|
|
let parts: Buffer[] = []
|
|
let length = 0
|
|
// Bytes at the start of the buffer, already searched without a match.
|
|
let searched = 0
|
|
// Leading bytes that belong to the previous delimiter (a shared CRLF).
|
|
let skip = 0
|
|
|
|
for await (const chunk of source) {
|
|
parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
|
|
length += chunk.byteLength
|
|
|
|
while (true) {
|
|
const from = Math.max(0, searched - (delimiter.length - 1))
|
|
const found = indexOf(parts, length, delimiter, from)
|
|
if (found === -1) {
|
|
searched = length
|
|
break
|
|
}
|
|
const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
|
|
const message = all.subarray(Math.min(skip, found), found)
|
|
const rest = all.subarray(found + delimiter.length)
|
|
const previous = delimiter
|
|
|
|
if (message.length === 0 && delimiter.equals(CRLF)) {
|
|
// A stray empty line: keep looking for the same delimiter.
|
|
} else {
|
|
delimiter = Buffer.from((yield message) ?? delimiter)
|
|
}
|
|
|
|
const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
|
|
parts = shareCrlf ? [CRLF, rest] : [rest]
|
|
length = rest.length + (shareCrlf ? 2 : 0)
|
|
skip = shareCrlf ? 2 : 0
|
|
searched = 0
|
|
}
|
|
}
|
|
|
|
if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
|
|
}
|
|
|
|
const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
|
|
const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
|
|
|
|
/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
|
|
function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
|
|
if (parts.length === 1) return parts[0].indexOf(needle, from)
|
|
// Collect the parts that cover [from, length).
|
|
let start = length
|
|
let i = parts.length
|
|
while (i > 0 && start > from) start -= parts[--i].length
|
|
const window = Buffer.concat(parts.slice(i), length - start)
|
|
const found = window.indexOf(needle, from - start)
|
|
return found === -1 ? -1 : start + found
|
|
}
|