Complete archive listings by walking RAR headers, at low priority

Inspection lists only the entries whose headers are in the first segment
of a first volume. A walk hops from header to header, fetching one segment
per entry and volume, in its own queue (one job per collection worker) on
NNTP connections inspection isn't waiting for. content:walk-archives queues
releases inspected earlier (CONCEPT D40).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/rar/walk.ts

@@ -0,0 +1,77 @@
+import { parseRarHeaders, rarFormat, type RarEntry, type RarInfo } from './parse.js'
+
+/** Reads the volumes of a RAR set, a segment at a time. */
+export interface VolumeReader {
+  readonly volumes: number
+  /**
+   * Bytes of volume `v` from `offset` to the end of the segment that holds
+   * it; with `more`, the following segment too (for a header cut off by the
+   * segment end). Null when that can't be read (missing article, budget).
+   */
+  read(v: number, offset: number, more: boolean): Promise<Buffer | null>
+  /** Size of volume `v`, known once something of it was read. */
+  size(v: number): number | null
+}
+
+export interface RarWalk {
+  format: 'rar4' | 'rar5' | null
+  /** One entry per name, from its first header. */
+  entries: RarEntry[]
+  /** Every volume was walked to its end. */
+  complete: boolean
+  headersEncrypted: boolean
+  /** Why the walk ended early. */
+  stopped?: 'unreadable' | 'invalid' | 'encrypted'
+}
+
+/**
+ * Lists a RAR set completely by hopping from header to header (CONCEPT.md
+ * D40). Each header says how much data follows it, so the next header's
+ * offset is known and only the segment holding it is read: one segment per
+ * entry and one per volume, instead of the whole set. Split files repeat
+ * their header at the start of every volume they continue in.
+ */
+export async function walkRar(reader: VolumeReader): Promise<RarWalk> {
+  const seen = new Map<string, RarEntry>()
+  const result = (stopped?: RarWalk['stopped']): RarWalk => ({
+    format,
+    entries: [...seen.values()],
+    complete: !stopped,
+    headersEncrypted: stopped === 'encrypted',
+    stopped,
+  })
+  let format: 'rar4' | 'rar5' | null = null
+  let v = 0
+  let offset = 0
+
+  while (v < reader.volumes) {
+    let info: RarInfo | null = null
+    let reached = -1
+    for (const more of [false, true]) {
+      const data = await reader.read(v, offset, more)
+      if (!data) return result('unreadable')
+      if (offset === 0) format = rarFormat(data) ?? format
+      info = offset === 0 ? parseRarHeaders(data) : format && parseRarHeaders(data, format)
+      if (!info) return result('invalid')
+      // Stopped inside the data: a header the segment end cut off (read on), or a bad one.
+      if (!info.truncated || info.next >= data.length || info.headersEncrypted) break
+      // Read on and got further: the next header is cut off at the end of the following segment.
+      if (more && info.next > reached) break
+      if (more) return result('invalid')
+      reached = info.next
+    }
+    if (info!.headersEncrypted) return result('encrypted')
+    for (const entry of info!.entries) if (!seen.has(entry.name)) seen.set(entry.name, entry)
+
+    const next = offset + info!.next
+    const size = reader.size(v)
+    // The end-of-archive header closes every volume; data running to the end does too.
+    if (!info!.truncated || size === null || next >= size) {
+      v++
+      offset = 0
+    } else {
+      offset = next
+    }
+  }
+  return result()
+}