Voyager 3 schema: the slim catalog (migration 6000, the reset) and blocks (6100)

releases rebuilt without its text copies: poster_id, group_set_id,
bytea keys, source smallint, edge, settled_at as the finish. posters by
id, group_sets. blocks as the lease unit, ranges.block_id. fragments,
range_blocks, release_final, the settler and the fragment merger gone;
release_chunks, rule_counts and rule_samples keyed by block. Cold tables
on CHUNK_TABLESPACE. Migration 6000 truncates the catalog and refuses
without VOYAGER3_RESET=1.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

commands/coordinator_run.ts

@@ -3,9 +3,6 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
 import { setTimeout as sleep } from 'node:timers/promises'
 import { Prefetcher } from '#services/prefetcher'
 import Planner from '#services/planner'
-import FragmentMerger from '#services/fragment_merger'
-import Settler from '#services/settler'
-import RangeBlocks from '#services/range_blocks'
 import Leases from '#services/leases'
 import NntpPool from '#services/nntp_pool'
 import { NntpShare } from '#services/nntp_shares'
@@ -13,38 +10,38 @@ import coordinatorConfig from '#config/coordinator'
 
 const TICK_MS = 2000
 const PLAN_EVERY_MS = 60 * 60 * 1000
-const MERGE_EVERY_MS = 60 * 1000
-const MERGE_BUDGET_MS = 30 * 1000
 const STOP_WAIT_MS = 15 * 1000
-/**
- * Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
- * rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
- * (LOG.md 2026-10-16 (67)).
- */
-const SETTLE_EVERY_MS = 5 * 60 * 1000
-const SETTLE_BATCHES = 4
-/** range_blocks (release_final) after ranges finish. */
-const BLOCKS_EVERY_MS = 5 * 60 * 1000
+/** Leases whose worker went quiet after reading everything are finished every few minutes. */
+const ABANDONED_EVERY_MS = 5 * 60 * 1000
 
 /**
- * The coordinator's background work (REBUILD.md, part 1): the prefetcher,
- * merging fragments of finished ranges every minute, and every hour
- * planning ranges for new articles and settling quiet releases. The API (leases, stream,
- * batches) runs in the web server.
+ * The coordinator's background work (REBUILD.md, part 1): the prefetcher, the tail's ranges every few minutes,
+ * new groups' history every hour, and abandoned leases. The API (leases, stream, batches) runs in the web server.
+ * Voyager 3: the settler and the fragment merger are gone (the window finishes a release once; the block merger
+ * joins edge releases).
  */
 export default class CoordinatorRun extends BaseCommand {
   static commandName = 'coordinator:run'
   static description = 'Prefetches ranges into the spool and plans new ones'
   static options: CommandOptions = { startApp: true, staysAlive: true }
 
-  @flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
+  @flags.string({
+    description:
+      'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles',
+  })
   declare shard?: string
 
   async run() {
     const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
-    if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
+    if (!(count >= 1 && index >= 1 && index <= count))
+      throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
     const prefetcher = new Prefetcher({ index: index - 1, count })
-    const share = new NntpShare(count > 1 ? `coordinator ${index}/${count}` : 'coordinator', coordinatorConfig.nntpShares.coordinatorPriority, coordinatorConfig.nntpMinConnections, coordinatorConfig.nntpConnections)
+    const share = new NntpShare(
+      count > 1 ? `coordinator ${index}/${count}` : 'coordinator',
+      coordinatorConfig.nntpShares.coordinatorPriority,
+      coordinatorConfig.nntpMinConnections,
+      coordinatorConfig.nntpConnections
+    )
     const main = index === 1
     let stopping = false
     let stopped: () => void
@@ -56,7 +53,11 @@ export default class CoordinatorRun extends BaseCommand {
       prefetcher.stop()
       await done
     })
-    this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
+    this.logger.info(
+      count > 1
+        ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).`
+        : 'Coordinator started.'
+    )
     // The pool's size: its share of the account's connections, the prober the rest (app/services/nntp_shares.ts).
     // Before the first connection: until then it's NNTP_CONNECTIONS.
     const first = await NntpPool.share(share)
@@ -69,11 +70,16 @@ export default class CoordinatorRun extends BaseCommand {
         } catch (error: any) {
           this.logger.error(`NNTP share failed: ${error.message}`)
         }
-        for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !stopping; waited += TICK_MS) await sleep(TICK_MS)
+        for (
+          let waited = 0;
+          waited < coordinatorConfig.nntpShares.everyMs && !stopping;
+          waited += TICK_MS
+        )
+          await sleep(TICK_MS)
       }
     })()
-    // Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
-    // of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
+    // The prefetcher's ticks in a loop of their own: planning doesn't hold them (a merge round of up to 30 s once
+    // left the connections idle once their targets were fetched).
     const prefetching = (async () => {
       while (!stopping) {
         try {
@@ -84,24 +90,24 @@ export default class CoordinatorRun extends BaseCommand {
         await sleep(TICK_MS)
       }
     })()
-    // range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
-    const blocks = (async () => {
+    const abandoned = (async () => {
       while (main && !stopping) {
         try {
-          const abandoned = await Leases.finishAbandoned()
-          if (abandoned > 0) this.logger.info(`Finished ${abandoned} ranges read to their end whose worker went quiet.`)
-          await RangeBlocks.rebuild()
+          const finished = await Leases.finishAbandoned()
+          if (finished > 0)
+            this.logger.info(
+              `Finished ${finished} ranges read to their end whose worker went quiet.`
+            )
         } catch (error: any) {
-          this.logger.error(`Range blocks failed: ${error.message}`)
+          this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
         }
-        for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
+        for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
+          await sleep(TICK_MS)
       }
     })()
     // History for new groups hourly; the tail every few minutes.
     let plannedAt = 0
     let tailedAt = 0
-    let mergedAt = 0
-    let settledAt = 0
     while (!stopping) {
       try {
         if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
@@ -109,45 +115,22 @@ export default class CoordinatorRun extends BaseCommand {
           const { groups, ranges } = await Planner.planAll()
           if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
         }
-        if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
-          settledAt = Date.now()
-          const settled = await Settler.run(SETTLE_BATCHES)
-          if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
-        }
         if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
           tailedAt = Date.now()
           const { groups, ranges } = await Planner.planTail()
           if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
         }
-        if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
-          mergedAt = Date.now()
-          // Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
-          // (Going on while only waiting files were left kept Postgres busy all the time.)
-          const totals = { completed: 0, incomplete: 0, waiting: 0 }
-          let decidedThisPass = 0
-          while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
-            const merged = await FragmentMerger.run()
-            totals.completed += merged.completed
-            totals.incomplete += merged.incomplete
-            totals.waiting = merged.waiting
-            decidedThisPass += merged.completed + merged.incomplete
-            if (merged.wrapped) {
-              if (decidedThisPass === 0) break
-              decidedThisPass = 0
-            }
-          }
-          if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
-        }
       } catch (error: any) {
         this.logger.error(`Coordinator round failed: ${error.message}`)
       }
       await sleep(TICK_MS)
     }
-    await Promise.all([prefetching, blocks, sharing])
+    await Promise.all([prefetching, abandoned, sharing])
     // The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
     const waitUntil = Date.now() + STOP_WAIT_MS
     while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
-    if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
+    if (prefetcher.active > 0)
+      this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
     await NntpPool.close()
     await share.leave().catch(() => {})
     this.logger.info('Coordinator stopped.')