Tiny-archive rule; the ladder's discard by index; the namer in chunks

The tiny-archive rule (dropTinyArchives, both indexers): a settled single
archive of one article and 2-100 kB under a release-like name, spaces
allowed, outside the exempt groups and without an e-book, subtitle,
crack, cover or sidecar word: the 2015-17 exe-stub flood in miniature,
1.66M catalog rows (LOG.md (131), (133)).

PendingLadder.discard scanned every waiting row against the dropped
signatures and classes, 9+ minutes a sweep inside the claim, stalling
every lane: now driven from the dropped classes and signatures into the
indexes. The namer feeds the indexer the span's overview text chunk by
chunk (one string of it topped V8's limit) and gives up spans over 3M
articles. Migration 7000: a partial index on the taken probe candidates
for the prober's start.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/pending_ladder.ts

@@ -375,21 +375,28 @@ class PendingLadder {
     )
   }
 
-  /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
+  /**
+   * Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
+   * many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the
+   * waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)).
+   */
   async discard(limit = 10_000): Promise<number> {
     const { samplePercent } = await RuleSets.current()
-    const { rows } = await db.rawQuery(
-      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
-       WHERE p.status = ? AND (EXISTS (
-         SELECT 1 FROM pending_signatures s
-         WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
-         OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
-       LIMIT ?`,
+    const { rows: byClass } = await db.rawQuery(
+      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c
+       JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass
+       WHERE c.dropped LIMIT ?`,
       [PENDING.waiting, limit]
     )
-    const keys = rows
-      .map((r: any) => r.release_key as string)
-      .filter((key: string) => !explored(key, samplePercent))
+    const { rows: bySignature } = await db.rawQuery(
+      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s
+       JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))
+       WHERE s.dropped LIMIT ?`,
+      [PENDING.waiting, limit]
+    )
+    const keys = [
+      ...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)),
+    ].filter((key: string) => !explored(key, samplePercent))
     if (keys.length === 0) return 0
     const { rows: gone } = await db.rawQuery(
       `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)

app/services/pending_namer.ts

@@ -30,6 +30,13 @@ interface NamedRow {
 /** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
 const MAX_MOVES = 3
 
+/**
+ * The widest article span the namer reads again (a busy group posts 50M articles a day, a large upload among
+ * them spans a million or two): one wider is a name the edge saw again much later ("v-lgpc.3ds"), and its span
+ * of segments would not fit the indexer's memory (LOG.md (133)).
+ */
+const MAX_SPAN = 3_000_000
+
 /**
  * The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
  * read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
@@ -94,6 +101,10 @@ export class PendingNamer {
 
   /** One named release into the catalog. */
   async move(row: NamedRow): Promise<void> {
+    if (row.last_article - row.first_article > MAX_SPAN)
+      throw new Error(
+        `its articles ${row.first_article}–${row.last_article} span more than ${MAX_SPAN}`
+      )
     const group = await db.from('groups').where('id', row.group_id).firstOrFail()
     const range = await db
       .from('ranges')
@@ -103,19 +114,20 @@ export class PendingNamer {
       .orderBy('id')
       .first()
     if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
-    const text = await this.overviewText(group.name, row)
     const rules = await RuleSets.current()
     const ix = new RangeIndexer({
       group: group.name,
       start: row.first_article,
       end: row.last_article,
       rules: { ...rules, samplePercent: 0 },
-      idleChunks: 0,
+      // No file given up before the end: the span is read chunk by chunk (one text of it topped V8's string).
+      idleChunks: Number.MAX_SAFE_INTEGER,
       edgeArticles: 0,
       // The edge wrote it partial when its file is short of parts on the server: so must the re-read.
       partialFiles: true,
     })
-    ix.addText(text, row.last_article + 1)
+    for await (const piece of this.overviewPieces(group.name, row))
+      ix.addText(piece.text, piece.next)
     ix.finish()
     const batch = ix.take()
     const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
@@ -160,50 +172,20 @@ export class PendingNamer {
     })
   }
 
-  /** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
-  private async overviewText(groupName: string, row: NamedRow): Promise<string> {
-    if (this.corpusDir) {
-      const stored = await db
-        .from('ranges')
-        .where('group_id', row.group_id)
-        .whereNotNull('stored_at')
-        .where('start_id', '<=', row.last_article)
-        .where('end_id', '>=', row.first_article)
-        .orderBy('start_id')
-      const parts: string[] = []
-      let covered = true
-      for (const r of stored) {
-        const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
-        const size = coordinatorConfig.chunkArticles
-        for (
-          let from =
-            range.start +
-            Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
-          from <= Math.min(row.last_article, range.end);
-          from += size
-        ) {
-          const raw = await Corpus.read(
-            range,
-            { from, to: Math.min(from + size - 1, range.end) },
-            this.corpusDir
-          )
-          if (!raw) {
-            covered = false
-            break
-          }
-          parts.push((await inflate(raw)).toString('latin1'))
-        }
-        if (!covered) break
-      }
-      if (
-        covered &&
-        parts.length > 0 &&
-        stored.length > 0 &&
-        Number(stored[0].start_id) <= row.first_article &&
-        Number(stored[stored.length - 1].end_id) >= row.last_article
-      ) {
-        return parts.join('')
-      }
+  /**
+   * The overview lines of the release's article span, a chunk of articles at a time with the first article after
+   * it: the stored corpus chunks when its ranges are stored and contiguous, else XOVER in windows of a chunk.
+   */
+  private async *overviewPieces(
+    groupName: string,
+    row: NamedRow
+  ): AsyncGenerator<{ text: string; next: number }> {
+    const size = coordinatorConfig.chunkArticles
+    const stored = await this.storedChunks(groupName, row)
+    if (stored) {
+      for (const { raw, next } of stored)
+        yield { text: (await inflate(raw)).toString('latin1'), next }
+      return
     }
     this.client ??= await XoverClient.connect({
       host: env.get('NNTP_HOST')!,
@@ -217,13 +199,55 @@ export class PendingNamer {
     this.usedAt = Date.now()
     try {
       await this.client.group(groupName)
-      // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
-      const raw = await this.client.xoverRaw(row.first_article, row.last_article)
-      return (await inflate(raw)).toString('latin1')
+      for (let from = row.first_article; from <= row.last_article; from += size) {
+        const to = Math.min(from + size - 1, row.last_article)
+        // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
+        const raw = await this.client.xoverRaw(from, to)
+        yield { text: (await inflate(raw)).toString('latin1'), next: to + 1 }
+      }
     } catch (error) {
       this.client.close()
       this.client = null
       throw error
     }
   }
+
+  /** The span's stored corpus chunks (compressed, with the first article after each), or null when any is missing. */
+  private async storedChunks(
+    groupName: string,
+    row: NamedRow
+  ): Promise<{ raw: Buffer; next: number }[] | null> {
+    if (!this.corpusDir) return null
+    const stored = await db
+      .from('ranges')
+      .where('group_id', row.group_id)
+      .whereNotNull('stored_at')
+      .where('start_id', '<=', row.last_article)
+      .where('end_id', '>=', row.first_article)
+      .orderBy('start_id')
+    if (
+      stored.length === 0 ||
+      Number(stored[0].start_id) > row.first_article ||
+      Number(stored[stored.length - 1].end_id) < row.last_article
+    )
+      return null
+    const size = coordinatorConfig.chunkArticles
+    const chunks: { raw: Buffer; next: number }[] = []
+    for (const r of stored) {
+      const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
+      for (
+        let from =
+          range.start +
+          Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
+        from <= Math.min(row.last_article, range.end);
+        from += size
+      ) {
+        const to = Math.min(from + size - 1, range.end)
+        const raw = await Corpus.read(range, { from, to }, this.corpusDir)
+        if (!raw) return null
+        chunks.push({ raw, next: to + 1 })
+      }
+    }
+    return chunks
+  }
 }