voyager 3: the batch store writes releases once, by id references and bytea keys
Test and deploy / test (push) Successful in 51s
Test and deploy / edge (push) Successful in 1m48s
Test and deploy / deploy (push) Skipped

Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 21:04:03 +01:00
1 parent bc4e17e86f
commit 2580f20a43
32 files changed
+962 -497

No files matched your search

+15 -13
View File
@@ -93,11 +93,12 @@ const HOLDERS = `SELECT c.release_id FROM (
interface Member {
id: number
release_key: string
group_id: number
files: number
posted_at: Date | null
readable: boolean
poster_id?: number | null
/** The poster's text (joined from posters) where a query needs it. */
poster?: string | null
file_total?: number | null
}
@@ -295,7 +296,8 @@ export class Par2Merger {
*/
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
const { rows: all } = await trx.rawQuery(
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster FROM par2_sets s JOIN releases r ON r.id = s.release_id
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
[setId]
)
@@ -310,12 +312,12 @@ export class Par2Merger {
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
const at = new Date(anchor.posted_at).getTime()
const { rows: candidates } = await trx.rawQuery(
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster FROM releases r
WHERE left(r.poster, 200) = left(?, 200) AND r.poster = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
AND r.files <= ?
ORDER BY left(r.poster, 200), r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
// More files than the set describes, with its sidecars, can't all be named in it.
[anchor.poster, anchor.poster, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
)
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
@@ -344,7 +346,7 @@ export class Par2Merger {
async mergeStem(stem: string): Promise<number> {
return db.transaction(async (trx) => {
const { rows } = await trx.rawQuery(
`SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
FROM release_inspections i JOIN releases r ON r.id = i.release_id
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
[stem]
@@ -356,7 +358,7 @@ export class Par2Merger {
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
const near = (r: Member) =>
r.group_id === anchor.group_id &&
(r.poster ?? '') === (anchor.poster ?? '') &&
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
const found = new Map<string, Member>()
@@ -374,7 +376,7 @@ export class Par2Merger {
async mergeSet(setId: string): Promise<number> {
return db.transaction(async (trx) => {
const { rows: holders } = await trx.rawQuery(
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
WHERE s.set_id = ? ORDER BY r.id`,
[setId]
)
@@ -383,7 +385,7 @@ export class Par2Merger {
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
const { rows: matches } = await trx.rawQuery(
`SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
WHERE f.set_id = ?`,
[setId]
@@ -426,7 +428,7 @@ export class Par2Merger {
await trx.rawQuery(
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
[target.id, key, via, others]
)
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
@@ -462,13 +464,13 @@ export class Par2Merger {
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
first_seen_at = least(t.first_seen_at, s.first_seen_at),
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
name = coalesce(nullif(?, ''), t.name)
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
max(file_total) AS file_total, max(inner_name) AS inner_name
FROM releases WHERE id = ANY(?)) s