Complete archive listings by walking RAR headers, at low priority

Inspection lists only the entries whose headers are in the first segment
of a first volume. A walk hops from header to header, fetching one segment
per entry and volume, in its own queue (one job per collection worker) on
NNTP connections inspection isn't waiting for. content:walk-archives queues
releases inspected earlier (CONCEPT D40).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/content/archive_names.ts

@@ -23,3 +23,18 @@ export function isLaterVolume(name: string): boolean {
   if (part) return Number(part[1]) > 1
   return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name)
 }
+
+/**
+ * The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of
+ * `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for
+ * other names.
+ */
+export function rarVolume(name: string): { base: string; number: number } | null {
+  const part = /^(.*)\.part(\d+)\.rar$/i.exec(name)
+  if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) }
+  const old = /^(.*)\.r(\d{2,3})$/i.exec(name)
+  if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 }
+  const first = /^(.*)\.rar$/i.exec(name)
+  if (first) return { base: first[1].toLowerCase(), number: 1 }
+  return null
+}

lib/content/index.ts

@@ -2,6 +2,6 @@
  * Content recognition by magic bytes, independent of AdonisJS.
  */
 export { sniffContentType, type ContentType } from './sniff.js'
-export { isArchiveStart, isLaterVolume } from './archive_names.js'
+export { isArchiveStart, isLaterVolume, rarVolume } from './archive_names.js'
 export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
 export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'

lib/rar/index.ts

@@ -3,3 +3,4 @@
  * contents from the first segment of a first volume. See CONCEPT.md (D19).
  */
 export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
+export { walkRar, type VolumeReader, type RarWalk } from './walk.js'

lib/rar/parse.ts

@@ -31,6 +31,12 @@ export interface RarInfo {
   entries: RarEntry[]
   /** The buffer ended before the end-of-archive header (normal for one segment). */
   truncated: boolean
+  /**
+   * Offset of the first block that was not read: where the next header
+   * starts (possibly beyond the buffer, after an entry's data), or the
+   * header the buffer cut off. After the end-of-archive header, its end.
+   */
+  next: number
 }
 
 const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
@@ -45,11 +51,16 @@ export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
 /**
  * Lists a RAR archive from its headers only, so the first segment of a first
  * volume is enough. Returns null when the data is not a RAR archive.
+ *
+ * With `continueAs`, `data` starts at a header in the middle of a volume (no
+ * signature), e.g. where a previous parse said the next header is.
  */
-export function parseRarHeaders(data: Buffer): RarInfo | null {
+export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
+  if (continueAs === 'rar4') return parseRar4(data, 0)
+  if (continueAs === 'rar5') return parseRar5(data, 0)
   const format = rarFormat(data)
-  if (format === 'rar4') return parseRar4(data)
-  if (format === 'rar5') return parseRar5(data)
+  if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
+  if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
   return null
 }
 
@@ -60,7 +71,7 @@ const R4_FILE = 0x74
 const R4_END = 0x7b
 const R4_LONG_BLOCK = 0x8000
 
-function parseRar4(data: Buffer): RarInfo {
+function parseRar4(data: Buffer, start: number): RarInfo {
   const info: RarInfo = {
     format: 'rar4',
     volume: false,
@@ -69,8 +80,9 @@ function parseRar4(data: Buffer): RarInfo {
     headersEncrypted: false,
     entries: [],
     truncated: true,
+    next: start,
   }
-  let p = RAR4_MARKER.length
+  let p = start
 
   while (p + 7 <= data.length) {
     const headCrc = data.readUInt16LE(p)
@@ -81,6 +93,7 @@ function parseRar4(data: Buffer): RarInfo {
 
     if (type === R4_END) {
       info.truncated = false
+      p += size
       break
     }
 
@@ -130,6 +143,7 @@ function parseRar4(data: Buffer): RarInfo {
   if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
     info.firstVolume = true
   }
+  info.next = p
   return info
 }
 
@@ -205,7 +219,7 @@ function vint(data: Buffer, offset: number): { value: number; next: number } | n
   return null
 }
 
-function parseRar5(data: Buffer): RarInfo {
+function parseRar5(data: Buffer, start: number): RarInfo {
   const info: RarInfo = {
     format: 'rar5',
     volume: false,
@@ -214,8 +228,9 @@ function parseRar5(data: Buffer): RarInfo {
     headersEncrypted: false,
     entries: [],
     truncated: true,
+    next: start,
   }
-  let p = RAR5_MARKER.length
+  let p = start
 
   while (p + 5 <= data.length) {
     const headCrc = data.readUInt32LE(p)
@@ -246,6 +261,7 @@ function parseRar5(data: Buffer): RarInfo {
 
     if (type.value === R5_END) {
       info.truncated = false
+      p = headerEnd
       break
     }
     if (type.value === R5_ENCRYPTION) {
@@ -270,6 +286,7 @@ function parseRar5(data: Buffer): RarInfo {
     if (next <= p) break
     p = next
   }
+  info.next = p
   return info
 }
 

lib/rar/walk.ts

@@ -0,0 +1,77 @@
+import { parseRarHeaders, rarFormat, type RarEntry, type RarInfo } from './parse.js'
+
+/** Reads the volumes of a RAR set, a segment at a time. */
+export interface VolumeReader {
+  readonly volumes: number
+  /**
+   * Bytes of volume `v` from `offset` to the end of the segment that holds
+   * it; with `more`, the following segment too (for a header cut off by the
+   * segment end). Null when that can't be read (missing article, budget).
+   */
+  read(v: number, offset: number, more: boolean): Promise<Buffer | null>
+  /** Size of volume `v`, known once something of it was read. */
+  size(v: number): number | null
+}
+
+export interface RarWalk {
+  format: 'rar4' | 'rar5' | null
+  /** One entry per name, from its first header. */
+  entries: RarEntry[]
+  /** Every volume was walked to its end. */
+  complete: boolean
+  headersEncrypted: boolean
+  /** Why the walk ended early. */
+  stopped?: 'unreadable' | 'invalid' | 'encrypted'
+}
+
+/**
+ * Lists a RAR set completely by hopping from header to header (CONCEPT.md
+ * D40). Each header says how much data follows it, so the next header's
+ * offset is known and only the segment holding it is read: one segment per
+ * entry and one per volume, instead of the whole set. Split files repeat
+ * their header at the start of every volume they continue in.
+ */
+export async function walkRar(reader: VolumeReader): Promise<RarWalk> {
+  const seen = new Map<string, RarEntry>()
+  const result = (stopped?: RarWalk['stopped']): RarWalk => ({
+    format,
+    entries: [...seen.values()],
+    complete: !stopped,
+    headersEncrypted: stopped === 'encrypted',
+    stopped,
+  })
+  let format: 'rar4' | 'rar5' | null = null
+  let v = 0
+  let offset = 0
+
+  while (v < reader.volumes) {
+    let info: RarInfo | null = null
+    let reached = -1
+    for (const more of [false, true]) {
+      const data = await reader.read(v, offset, more)
+      if (!data) return result('unreadable')
+      if (offset === 0) format = rarFormat(data) ?? format
+      info = offset === 0 ? parseRarHeaders(data) : format && parseRarHeaders(data, format)
+      if (!info) return result('invalid')
+      // Stopped inside the data: a header the segment end cut off (read on), or a bad one.
+      if (!info.truncated || info.next >= data.length || info.headersEncrypted) break
+      // Read on and got further: the next header is cut off at the end of the following segment.
+      if (more && info.next > reached) break
+      if (more) return result('invalid')
+      reached = info.next
+    }
+    if (info!.headersEncrypted) return result('encrypted')
+    for (const entry of info!.entries) if (!seen.has(entry.name)) seen.set(entry.name, entry)
+
+    const next = offset + info!.next
+    const size = reader.size(v)
+    // The end-of-archive header closes every volume; data running to the end does too.
+    if (!info!.truncated || size === null || next >= size) {
+      v++
+      offset = 0
+    } else {
+      offset = next
+    }
+  }
+  return result()
+}