voyager 3: the batch store writes releases once, by id references and bytea keys

Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

commands/edge_bench.ts

@@ -108,7 +108,7 @@ export default class EdgeBench extends BaseCommand {
     let text = 0
     let peakRss = 0
     let peakDetail = ''
-    const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
+    const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, samples: 0 }
     const counts = new Map<string, number>()
     const keys = new Set<string>()
     let batches = 0
@@ -164,7 +164,6 @@ export default class EdgeBench extends BaseCommand {
         out.payload += r.payload.length
       }
       out.candidates += batch.candidates.length
-      out.fragments += batch.fragments.length
       out.samples += batch.samples.length
       for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
       const m = process.memoryUsage()
@@ -182,7 +181,7 @@ export default class EdgeBench extends BaseCommand {
     this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
     this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
     this.logger.info(`  ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
-    this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
+    this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.samples} samples`)
     for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(`  dropped ${k}: ${v.toLocaleString('en')} articles`)
     if (this.store) this.logger.info(`  stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
   }

commands/releases_regroup.ts

@@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
 import db from '@adonisjs/lucid/services/db'
 import { decodeChunk } from '#lib/chunks/index'
 import { releaseIdentity } from '#lib/release/index'
+import { sourceCode } from '#lib/release/source'
 
 /**
  * What lib/release's identity makes of releases already in the catalog: a
@@ -27,7 +28,7 @@ export default class ReleasesRegroup extends BaseCommand {
   async run() {
     for (const name of this.group ?? []) {
       const { rows } = await db.rawQuery(
-        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
+        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
         [name]
       )
       const ids = rows.map((r: { release_id: string }) => r.release_id)
@@ -54,7 +55,7 @@ export default class ReleasesRegroup extends BaseCommand {
     }
     if (this.show?.length) return
     for (const source of ['subject', 'post', 'stem']) {
-      const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
+      const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
       let split = 0
       let pieces = 0
       const examples: string[] = []