voyager 3: the batch store writes releases once, by id references and bytea keys

Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

lib/edge/index.ts

@@ -2,7 +2,6 @@ export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsO
 export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
 export { toWire, fromWire, type WireBatch } from './wire.js'
 export type { Lease } from './lease.js'
-export { mergeFragments } from './merge.js'
 export type { EdgeSegment, FileMeta } from './scan.js'
 export { throttle } from './throttle.js'
 export { filterOverviews, undelta, type ChunkDrops } from './corpus_filter.js'

lib/edge/merge.ts

@@ -1,35 +0,0 @@
-import type { AggregatedFile } from '../files/index.js'
-import type { Fragment } from './range_indexer.js'
-import type { EdgeSegment, FileMeta } from './scan.js'
-
-/**
- * The fragments of one file from neighbouring ranges, put together
- * (REBUILD.md, fragments): the complete file when every part from 1 to its
- * total is there (the first article seen of a part wins), else null.
- */
-export function mergeFragments(fragments: Fragment[]): AggregatedFile<EdgeSegment> | null {
-  if (fragments.length === 0) return null
-  const first = fragments[0]
-  const meta: FileMeta = {
-    fileKey: first.fileKey,
-    name: first.name,
-    poster: first.poster,
-    subject: first.subject,
-    totalParts: first.totalParts,
-    groups: first.groups,
-    release: first.release ? { ...first.release, source: first.release.source as any } : null,
-  }
-  const parts = new Map<number, EdgeSegment>()
-  for (const fragment of fragments) {
-    const postedAt = fragment.postedAt ?? Number.NaN
-    for (const [number, messageId, bytes, articleNumber] of fragment.segments) {
-      const known = parts.get(number)
-      if (!known || articleNumber < known.articleNumber) {
-        parts.set(number, { fileKey: meta.fileKey, number, totalParts: meta.totalParts, messageId, bytes, articleNumber, postedAt, file: meta })
-      }
-    }
-  }
-  if (parts.size < meta.totalParts) return null
-  for (let n = 1; n <= meta.totalParts; n++) if (!parts.has(n)) return null
-  return { fileKey: meta.fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }
-}

lib/edge/range_indexer.ts

@@ -26,6 +26,13 @@ export interface ReleaseChunk {
   /** A file with a readable name; a par2 file. */
   readable: boolean
   hasPar2: boolean
+  /**
+   * Voyager 3: the window closed on the release (default), or it is a tail release sent provisionally, finished by a
+   * later chunk of the same release key (the totals then are the release's; the payload only the files not sent yet).
+   */
+  finished?: boolean
+  /** Still open at its block's end: the block merger joins it with the neighbour block's part. */
+  edge?: boolean
   payload: Buffer
 }
 

lib/release/keys.ts

@@ -0,0 +1,34 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * The catalog's keys as voyager 3 stores them (migration 6000): 20-byte SHA-1s in bytea columns.
+ * - identity_key: the identity's hash (lib/release/identity.ts gives it as 40 hex characters).
+ * - release_key: one upload of an identity, unique: the identity, the owner group and the first article the window
+ *   saw of it. The same release sent twice (a tail release provisionally, then finished) hits the same row; a repost
+ *   starts at another article and is a release of its own (decision d). Deterministic, so no numbering under a lock.
+ */
+const HEX40 = /^[0-9a-f]{40}$/i
+
+function sha1(text: string): Buffer {
+  return createHash('sha1').update(text).digest()
+}
+
+/** The identity key's bytes: 40 hex characters decoded, any other text (a spot's, a test's) hashed. */
+export function identityKey(key: string): Buffer {
+  return HEX40.test(key) ? Buffer.from(key, 'hex') : sha1(key)
+}
+
+/** The release key's bytes for an identity in a group, first seen at an article (null for releases without one: a spot's). */
+export function releaseKey(
+  identity: string,
+  groupId: number | null,
+  firstArticle: number | null
+): Buffer {
+  return sha1(`release\n${identity}\n${groupId ?? ''}\n${firstArticle ?? ''}`)
+}
+
+/** A stored key as hex, for logs, the API and release_merges.from_key. */
+export function keyHex(key: Buffer | Uint8Array | string | null | undefined): string | null {
+  if (key === null || key === undefined) return null
+  return typeof key === 'string' ? key : Buffer.from(key).toString('hex')
+}

lib/release/source.ts

@@ -15,6 +15,8 @@ export const SOURCES = {
   nzb: 5,
   /** One poster's files that differ only by a number sequence, or share a prefix (voyager 3). */
   album: 6,
+  /** An obfuscated post's stem, read from the yEnc body (identity.ts: weak). */
+  'weak-stem': 7,
 } as const
 
 export type Source = keyof typeof SOURCES