Rebuild: first end-to-end runs; prefetch per chunk, executable rule, wire fixes

End to end on the workstation (coordinator:run, the API, edge:node, real
NNTP): a.b.e-book.french and a.b.cd.image.playstation2.dvdiso ranges.

- Prefetcher: every connection takes the next missing chunk (leased
  ranges first, in order); one connection per range (~26k lines/s) kept
  the worker waiting
- executable rule: releases of bare executables (malware floods named
  after films and software) are dropped; they were 92% of the catalog rows
  of the test ranges, 21% of the segments
- fragments per file (details once) and gzipped batch bodies; the JSON
  limit (1 MB by default) raised to 256 MB, empty strings kept (a rule
  count's key is '')
- release_chunks.batch_from is a bigint (boneless article numbers)
- tests get their own database; edge:bench --store times BatchStore:
  1.3 s for a boneless range (1.24 ms per release chunk)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -69,7 +69,7 @@ class BatchStore {
         const rows = releases.slice(i, i + 500).map((r) => ({
           release_id: ids.get(r.key)!,
           range_id: rangeId,
-          batch_seq: batch.from,
+          batch_from: batch.from,
           files: r.files,
           segments: r.segments,
           payload: r.payload,
@@ -100,7 +100,13 @@ class BatchStore {
       }
 
       for (let i = 0; i < batch.fragments.length; i += 2000) {
-        const rows = batch.fragments.slice(i, i + 2000).map((f) => ({ range_id: rangeId, file_key: f.fileKey, article_number: f.articleNumber, segment: JSON.stringify(f) }))
+        const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
+          range_id: rangeId,
+          file_key: f.fileKey,
+          article_number: Math.min(...f.segments.map((s) => s[3])),
+          segments: f.segments.length,
+          fragment: JSON.stringify(f),
+        }))
         await trx.insertQuery().table('fragments').multiInsert(rows)
       }
 

app/services/prefetcher.ts

@@ -2,39 +2,48 @@ import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
-import Spool from '#services/spool'
+import Spool, { type ChunkSpan } from '#services/spool'
 
 interface Target {
   id: number
   group: string
-  start: number
-  end: number
-  next: number
-  leased: boolean
+  /** Chunks not spooled yet, in order. */
+  missing: ChunkSpan[]
+  /** All of the range's chunks from its resume point. */
+  total: number
 }
 
+/** How often the targets are read again from the database. */
+const REFRESH_MS = 2000
+
 /**
- * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch):
- * the missing chunks of leased ranges first, then the next planned ranges,
- * about SPOOL_AHEAD_PER_LEASE per active lease (at least SPOOL_MIN_AHEAD),
- * within SPOOL_MAX_GB. One range per connection at a time, in order.
+ * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
+ * Every connection takes the next missing chunk: the leased ranges' first
+ * (a worker waits for them; all connections work on the same range, in
+ * order), then the next planned ranges, about SPOOL_AHEAD_PER_LEASE per
+ * active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
+ * connection fills ~26 k overview lines/s, so a range on one connection
+ * kept its worker waiting (LOG.md 2026-10-09).
  */
 export class Prefetcher {
-  private busy = new Set<number>()
+  private targets: Target[] = []
+  private claimed = new Set<string>()
+  private running = 0
+  private refreshedAt = 0
   private stopped = false
 
-  /** One round: starts work for free connections; returns how many ranges it started. */
-  async tick(): Promise<number> {
-    const free = NntpPool.size - this.busy.size
-    if (free <= 0) return 0
-    const targets = await this.targets(free)
-    for (const target of targets) {
-      this.busy.add(target.id)
-      this.spoolRange(target)
-        .catch((error) => logger.warn(`Spooling range ${target.id}: ${error.message}`))
-        .finally(() => this.busy.delete(target.id))
+  /** One round: refreshes the targets now and then, and puts idle connections to work. */
+  async tick() {
+    if (Date.now() - this.refreshedAt >= REFRESH_MS) {
+      this.refreshedAt = Date.now()
+      this.targets = await this.readTargets()
+    }
+    while (!this.stopped && this.running < NntpPool.size && this.peek()) {
+      this.running++
+      this.work()
+        .catch((error) => logger.warn(`Prefetching: ${error.message}`))
+        .finally(() => this.running--)
     }
-    return targets.length
   }
 
   stop() {
@@ -42,49 +51,64 @@ export class Prefetcher {
   }
 
   get active() {
-    return this.busy.size
+    return this.running
   }
 
-  private async targets(limit: number): Promise<Target[]> {
-    const busy = [...this.busy]
-    const select = (q: any) =>
-      q
-        .from('ranges as r')
-        .join('groups as g', 'g.id', 'r.group_id')
-        .whereNull('r.spooled_at')
-        .whereNotIn('r.id', busy.length > 0 ? busy : [0])
-        .select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.status')
-    // Leased ranges whose chunks aren't all there: a worker is waiting for them.
-    const leased = await select(db.query()).where('r.status', 'leased').orderBy('r.leased_at').limit(limit)
-    const out: Target[] = leased.map(toTarget)
-    if (out.length >= limit) return out
-
-    const [{ leases }] = await db.from('ranges').where('status', 'leased').count('* as leases')
-    const [{ ahead }] = await db.from('ranges').where('status', 'planned').where('spooled_chunks', '>', 0).count('* as ahead')
-    const wanted = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * Number(leases)) - Number(ahead)
-    if (wanted <= 0 || (await Spool.size()) >= coordinatorConfig.spool.maxBytes) return out
-    const planned = await select(db.query()).where('r.status', 'planned').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(Math.min(limit - out.length, wanted))
-    return [...out, ...planned.map(toTarget)]
-  }
-
-  private async spoolRange(target: Target) {
-    let count = 0
-    for (const span of Spool.chunksOf({ start: target.start, end: target.end }, target.next)) {
-      if (this.stopped) return
-      if (!(await Spool.has(target.id, span))) {
+  /** One connection's loop: chunks until there is nothing left to fetch. */
+  private async work() {
+    while (!this.stopped) {
+      const next = this.take()
+      if (!next) return
+      const { target, span } = next
+      try {
         const data = await NntpPool.use(async (client) => {
           await client.group(target.group)
           return client.xoverRaw(span.from, span.to)
         })
         await Spool.write(target.id, span, data)
+        const { rows } = await db.rawQuery(
+          `UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
+          [target.id]
+        )
+        if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+      } finally {
+        this.claimed.delete(`${target.id}:${span.from}`)
       }
-      count++
-      await db.from('ranges').where('id', target.id).update({ spooled_chunks: count })
     }
-    await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+  }
+
+  private peek(): boolean {
+    return this.targets.some((t) => t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
+  }
+
+  /** The first unclaimed missing chunk, in target order. */
+  private take(): { target: Target; span: ChunkSpan } | null {
+    for (const target of this.targets) {
+      const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
+      if (index === -1) continue
+      const [span] = target.missing.splice(index, 1)
+      this.claimed.add(`${target.id}:${span.from}`)
+      return { target, span }
+    }
+    return null
+  }
+
+  private async readTargets(): Promise<Target[]> {
+    const rows = (q: any) =>
+      q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').whereNull('r.spooled_at').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id')
+    // Leased ranges whose chunks aren't all there: a worker is waiting for them.
+    const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
+    const [{ leases }] = await db.from('ranges').where('status', 'leased').count('* as leases')
+    const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * Number(leases))
+    const room = (await Spool.size()) < coordinatorConfig.spool.maxBytes
+    const planned = room ? await rows(db.query()).where('r.status', 'planned').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead) : []
+    const targets: Target[] = []
+    for (const row of [...leased, ...planned]) {
+      const spans = Spool.chunksOf({ start: Number(row.start_id), end: Number(row.end_id) }, Number(row.next_id))
+      const missing: ChunkSpan[] = []
+      for (const span of spans) if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
+      targets.push({ id: Number(row.id), group: row.group, missing, total: spans.length })
+    }
+    return targets
   }
 }
-
-function toTarget(row: any): Target {
-  return { id: Number(row.id), group: row.group, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id), leased: row.status === 'leased' }
-}

app/services/rule_sets.ts

@@ -12,12 +12,15 @@ class RuleSets {
   async current(): Promise<Rules> {
     let row = await db.from('rule_sets').orderBy('version', 'desc').first()
     if (!row) {
-      const { version: _v, ...rules } = DEFAULT_RULES
+      const { version: _version, ...rules } = DEFAULT_RULES
       await db.table('rule_sets').insert({ version: 1, rules: JSON.stringify({ ...rules, droppedSignatures: [] }), note: 'default' }).onConflict('version').ignore()
       row = await db.from('rule_sets').orderBy('version', 'desc').first()
     }
     const groups = await db.from('groups').where('active', true).select('name', 'priority')
+    const { version: _v, ...defaults } = DEFAULT_RULES
     return {
+      // A field newer than the stored set takes today's default.
+      ...defaults,
       ...(row.rules as StoredRules),
       version: Number(row.version),
       priorities: Object.fromEntries(groups.map((g) => [g.name, Number(g.priority)])),