Voyager 3 schema: the slim catalog (migration 6000, the reset) and blocks (6100)
releases rebuilt without its text copies: poster_id, group_set_id, bytea keys, source smallint, edge, settled_at as the finish. posters by id, group_sets. blocks as the lease unit, ranges.block_id. fragments, range_blocks, release_final, the settler and the fragment merger gone; release_chunks, rule_counts and rule_samples keyed by block. Cold tables on CHUNK_TABLESPACE. Migration 6000 truncates the catalog and refuses without VOYAGER3_RESET=1. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
1a448f37f9
commit
346493b6c3
17 files changed
+527
-545
No files matched your search
@@ -86,18 +86,6 @@ class BatchStore {
|
||||
})
|
||||
}
|
||||
|
||||
for (let i = 0; i < batch.fragments.length; i += 2000) {
|
||||
const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
|
||||
range_id: rangeId,
|
||||
file_key: f.fileKey,
|
||||
// Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
|
||||
article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
|
||||
segments: f.segments.length,
|
||||
fragment: JSON.stringify(f),
|
||||
}))
|
||||
await trx.insertQuery().table('fragments').multiInsert(rows)
|
||||
}
|
||||
|
||||
const moved = await trx
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { RuleSet } from '#lib/rules/index'
|
||||
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
|
||||
import type { AggregatedFile } from '#lib/files/index'
|
||||
import type { EdgeSegment } from '#lib/edge/index'
|
||||
|
||||
export interface MergeResult {
|
||||
files: number
|
||||
completed: number
|
||||
incomplete: number
|
||||
waiting: number
|
||||
/** This round reached the end of the file keys (the next one starts over). */
|
||||
wrapped: boolean
|
||||
}
|
||||
|
||||
/** File keys per round. */
|
||||
const BATCH = 20_000
|
||||
|
||||
/**
|
||||
* Files cut by a range's edge (REBUILD.md, fragments), put together once
|
||||
* the ranges around them are done: a complete file is judged by the rules
|
||||
* like any other and written as a release chunk (with probe candidates); a
|
||||
* file still missing parts while every neighbouring range is done is
|
||||
* incomplete, and dropped (counted on its ranges). Others wait.
|
||||
*/
|
||||
class FragmentMerger {
|
||||
/** Where the next round goes on: waiting files must not take every round's places. */
|
||||
private cursor = ''
|
||||
|
||||
async run(): Promise<MergeResult> {
|
||||
const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
|
||||
const { rows: keys } = await db.rawQuery(
|
||||
`SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
|
||||
WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
|
||||
[this.cursor, BATCH]
|
||||
)
|
||||
// At the end of the keys: the next round starts over.
|
||||
result.wrapped = keys.length < BATCH
|
||||
this.cursor = result.wrapped ? '' : keys.at(-1).file_key
|
||||
if (keys.length === 0) return result
|
||||
const rules = new RuleSet(await RuleSets.current())
|
||||
const rows = await db
|
||||
.from('fragments as f')
|
||||
.join('ranges as r', 'r.id', 'f.range_id')
|
||||
.whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
|
||||
.select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
|
||||
const byKey = new Map<string, typeof rows>()
|
||||
for (const row of rows) {
|
||||
const list = byKey.get(row.file_key)
|
||||
if (list) list.push(row)
|
||||
else byKey.set(row.file_key, [row])
|
||||
}
|
||||
|
||||
const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
|
||||
const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
|
||||
const neighbours = await this.neighbourStatus([...byKey.values()])
|
||||
for (const list of byKey.values()) {
|
||||
result.files++
|
||||
const file = mergeFragments(list.map((r) => r.fragment as Fragment))
|
||||
const ids = list.map((r) => Number(r.id))
|
||||
const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
|
||||
if (file) {
|
||||
completed.push({ file, rangeId, ids })
|
||||
continue
|
||||
}
|
||||
if (list.every((r) => r.status === 'done') && neighbours(list)) {
|
||||
incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
|
||||
} else {
|
||||
result.waiting++
|
||||
}
|
||||
}
|
||||
|
||||
// Completed files, grouped into their releases, judged and written like a batch's.
|
||||
const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
|
||||
for (const { file, rangeId } of completed) {
|
||||
const key = releaseKeyOf(file.segments[0])
|
||||
const entry = byRelease.get(key)
|
||||
if (entry) entry.files.push(file)
|
||||
else byRelease.set(key, { files: [file], rangeId })
|
||||
}
|
||||
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
||||
for (const [key, { files, rangeId }] of byRelease) {
|
||||
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
|
||||
if (verdict.rule !== null) continue
|
||||
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
||||
}
|
||||
await ChunkPartitions.ensure()
|
||||
await db.transaction(async (trx) => {
|
||||
// In key order (batches lock their group; the merger only ever waits on one batch at a time).
|
||||
releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
|
||||
for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
|
||||
const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
|
||||
for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
|
||||
for (const { rangeId, segments } of incomplete) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
|
||||
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
|
||||
[rangeId, segments]
|
||||
)
|
||||
}
|
||||
})
|
||||
result.completed = completed.length
|
||||
result.incomplete = incomplete.length
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* For each file's fragments: whether the ranges right before and after
|
||||
* their ranges (where a missing part could be) are done, read in one
|
||||
* query for the whole round (per file it was three, ~2,000 files a
|
||||
* minute). A neighbour not planned yet counts as not done (newer articles
|
||||
* come later), except before the group's oldest article.
|
||||
*/
|
||||
private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
|
||||
const span = (list: (typeof lists)[number]) => ({
|
||||
group: Number(list[0].group_id),
|
||||
first: Math.min(...list.map((r) => Number(r.start_id))),
|
||||
last: Math.max(...list.map((r) => Number(r.end_id))),
|
||||
})
|
||||
const spans = lists.map(span)
|
||||
const groups = [...new Set(spans.map((s) => s.group))]
|
||||
// The range holding the article before and the one after, each by the (group_id, start_id) index:
|
||||
// "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
|
||||
FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
|
||||
ORDER BY r.start_id DESC LIMIT 1) b ON true
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
|
||||
ORDER BY r.start_id DESC LIMIT 1) a ON true`,
|
||||
[spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
|
||||
)
|
||||
const byEnd = new Map<string, string>()
|
||||
const byStart = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
|
||||
if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
|
||||
}
|
||||
const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
|
||||
return (list: (typeof lists)[number]) => {
|
||||
const { group, first, last } = span(list)
|
||||
const before = byEnd.get(`${group}:${first - 1}`)
|
||||
const low = lows.get(group)
|
||||
const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
|
||||
return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new FragmentMerger()
|
||||
@@ -75,7 +75,7 @@ export class Par2Pruner {
|
||||
WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
|
||||
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) IS NOT TRUE
|
||||
r.settled_at IS NULL
|
||||
OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
|
||||
AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
|
||||
* release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
|
||||
* range that isn't done or doesn't start where the one before ended.
|
||||
*/
|
||||
class RangeBlocks {
|
||||
async rebuild(): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`DELETE FROM range_blocks`)
|
||||
const result = await trx.rawQuery(
|
||||
`WITH r AS (
|
||||
SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
|
||||
min(start_id) OVER (PARTITION BY group_id) AS group_first,
|
||||
lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
|
||||
FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
|
||||
runs AS (
|
||||
SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
|
||||
OVER (PARTITION BY group_id ORDER BY start_id) AS run
|
||||
FROM r)
|
||||
INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
|
||||
SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
|
||||
FROM runs WHERE done GROUP BY group_id, run`
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new RangeBlocks()
|
||||
@@ -102,10 +102,10 @@ export const RULES: Rule[] = [
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||
// Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
|
||||
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||
AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
|
||||
AND r.settled_at IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||
|
||||
@@ -51,7 +51,7 @@ export interface ReleaseSummary {
|
||||
size: number
|
||||
readable: boolean
|
||||
complete: boolean | null
|
||||
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
|
||||
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
|
||||
final: boolean
|
||||
category: number
|
||||
settled: boolean
|
||||
@@ -98,8 +98,8 @@ class SearchService {
|
||||
}
|
||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||
if (params.complete) where.push('r.complete')
|
||||
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
|
||||
// Incomplete only when final: a provisional tail release may yet get its missing files.
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
|
||||
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
@@ -166,7 +166,7 @@ class SearchService {
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
||||
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
||||
r.settled_at IS NOT NULL AS final
|
||||
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...source.bindings, ...bindings, perPage, offset]
|
||||
),
|
||||
@@ -205,10 +205,9 @@ class SearchService {
|
||||
async release(id: number) {
|
||||
const r = await db.from('releases').where('id', id).first()
|
||||
if (!r) return null
|
||||
const [files, { rows: inspections }, { rows: finality }, passwords] = await Promise.all([
|
||||
const [files, { rows: inspections }, passwords] = await Promise.all([
|
||||
NzbService.files(id),
|
||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
||||
NzbService.passwords(id),
|
||||
])
|
||||
return {
|
||||
@@ -224,7 +223,7 @@ class SearchService {
|
||||
segments: Number(r.segments),
|
||||
readable: r.readable,
|
||||
complete: r.complete,
|
||||
final: finality[0].final,
|
||||
final: r.settled_at !== null,
|
||||
hasPar2: r.has_par2,
|
||||
category: r.category,
|
||||
firstSeenAt: r.first_seen_at,
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
|
||||
const QUIET_HOURS = 6
|
||||
/**
|
||||
* History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
|
||||
* neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
|
||||
* Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
|
||||
* each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
|
||||
*/
|
||||
const QUIET_HISTORY_MINUTES = 30
|
||||
const BATCH = 50_000
|
||||
|
||||
/**
|
||||
* Settles releases (REBUILD.md, part 8): once nothing has been added for
|
||||
* QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
|
||||
* [i/N] counter announced was found, unknown without a counter (phase 2
|
||||
* decides those by their par2 sets: app/services/par2_completeness.ts).
|
||||
* Phase 2 picks its work from settled releases.
|
||||
*/
|
||||
class Settler {
|
||||
/** Up to `batches` batches of BATCH (all there are without). */
|
||||
async run(batches = Infinity): Promise<number> {
|
||||
let settled = 0
|
||||
for (let batch = 0; batch < batches; batch++) {
|
||||
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH settled AS (
|
||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END
|
||||
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
||||
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
||||
RETURNING id, readable, files, bytes, poster, sidecar_only),
|
||||
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
|
||||
-- A small one of sidecars only (the batch store's sidecar_only) may be an upload's .nfo or .nzb that came apart
|
||||
-- from it: the merger's sidecar fold looks at them.
|
||||
queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE sidecar_only AND files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
|
||||
ON CONFLICT DO NOTHING)
|
||||
SELECT count(*)::int AS n FROM settled`,
|
||||
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
|
||||
)
|
||||
const n = rows[0].n
|
||||
settled += n
|
||||
if (n < BATCH) return settled
|
||||
}
|
||||
return settled
|
||||
}
|
||||
}
|
||||
|
||||
export default new Settler()
|
||||
Reference in new issue
Block a user