daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: re-uploads are releases of their own
Test and deploy / test (push) Successful in 26s
Details
Test and deploy / deploy (push) Skipped
Details

Browse Source 
A poster re-uploading under the same name made one release of all
uploads: in a sample, 24% of releases with 2+ files had a file name more
than once (4.3% of files extra copies), 70% of those copies posted a day
or more apart; their NZBs listed each copy.

- File assembly keeps a re-upload's copy apart: a part already in every
  open copy, posted more than 3 h from them, starts another copy; a
  missing part fills the nearest copy (a re-send of what was missing),
  a re-sent part near a copy that has it is ignored as before.
- A key's files in a batch split into uploads where posting pauses for
  more than 3 h, counted from the latest article posted so far (a file
  that takes hours to post doesn't split it); each upload is judged and
  chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry
  their upload).
- The catalog matches a chunk to the release of its key whose articles
  the range's group owns and whose posting window is within 3 h, else
  starts one (key, key:2, …): releases get identity_key, group_id,
  last_posted_at. Keys are locked in order (advisory locks), so
  concurrent batches neither deadlock nor both start one; the merger
  writes in key order too. The same upload multi-posted to other groups
  is a release per group.
- The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies
  come later (releases:upload-gaps). releases:duplicates measures the
  duplicate names.

Both implementations; check.ts compares the windows and uploads too
(identical on all captures; Rust tests mirror the TypeScript cases).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 13:31:35 +02:00
1 parent d0f3497ec9
commit 0613984de9
15 files changed
+715 -140
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/batch_store.ts
+90 -54
View File
@@ -1,6 +1,6 @@
import db from '@adonisjs/lucid/services/db'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import ChunkPartitions from '#services/chunk_partitions'
export class StaleBatch extends Error {}
@@ -95,7 +95,10 @@ class BatchStore {
/**
* Release aggregates in the catalog, one write-once chunk per release, and
* probe candidates (a batch's, or files the fragment merger completed).
* Releases are sorted by key, so concurrent writers lock them in the same order.
* A chunk joins the release of its key whose articles the range's group
* owns and whose posting window is within UPLOAD_GAP_MS of its own, else
* starts one: re-uploads are releases of their own (decision d). Keys are
* locked in order, so concurrent writers don't deadlock or both start one.
*/
async writeReleases(
trx: TransactionClientContract,
@@ -104,64 +107,97 @@ class BatchStore {
releases: ReleaseChunk[],
candidates: ProbeCandidate[]
) {
const out = { newReleases: 0, candidates: 0, ids: new Map<string, number>() }
const ids = out.ids
// Releases, sorted by key so concurrent batches lock them in the same order.
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key))
const out = { newReleases: 0, candidates: 0 }
const ids = new Map<string, number>()
const slot = (key: string, upload: number) => `${key}\n${upload}`
const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
for (let i = 0; i < sorted.length; i += 1000) {
const chunk = sorted.slice(i, i + 1000)
const { rows } = await trx.rawQuery(
`INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2,
CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
ON CONFLICT (release_key) DO UPDATE SET
files = releases.files + excluded.files,
segments = releases.segments + excluded.segments,
bytes = releases.bytes + excluded.bytes,
chunks = releases.chunks + 1,
posted_at = least(releases.posted_at, excluded.posted_at),
file_total = coalesce(releases.file_total, excluded.file_total),
first_article = least(releases.first_article, excluded.first_article),
last_article = greatest(releases.last_article, excluded.last_article),
readable = releases.readable OR excluded.readable,
has_par2 = releases.has_par2 OR excluded.has_par2,
-- As it stands (the counter's files all found); settling makes it final.
complete = CASE WHEN coalesce(releases.file_total, excluded.file_total) IS NULL THEN NULL
ELSE releases.files + excluded.files >= coalesce(releases.file_total, excluded.file_total) END,
last_seen_at = now()
RETURNING id, release_key, (xmax = 0) AS inserted`,
[
JSON.stringify(
chunk.map((r) => ({
key: r.key,
name: r.name,
source: r.source,
poster: r.poster,
groups: r.groups,
posted_at: r.postedAt,
file_total: r.fileTotal,
files: r.files,
segments: r.segments,
bytes: r.bytes,
first_article: r.firstArticle,
last_article: r.lastArticle,
readable: r.readable,
has_par2: r.hasPar2,
}))
),
]
const part = sorted.slice(i, i + 1000)
const keys = [...new Set(part.map((r) => r.key))]
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended(k, 0)) FROM unnest(?::text[]) AS k ORDER BY k`, [keys])
const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
const { rows: matches } = await trx.rawQuery(
`SELECT DISTINCT ON (v.idx) v.idx, r.id
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
WHERE v.first IS NULL OR r.posted_at IS NULL
OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
[JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
)
for (const row of rows) {
ids.set(row.release_key, Number(row.id))
if (row.inserted) out.newReleases++
const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
const row = (r: ReleaseChunk) => ({
name: r.name,
source: r.source,
poster: r.poster,
groups: r.groups,
first: r.postedAt,
last: r.postedLast ?? r.postedAt,
file_total: r.fileTotal,
files: r.files,
segments: r.segments,
bytes: r.bytes,
first_article: r.firstArticle,
last_article: r.lastArticle,
readable: r.readable,
has_par2: r.hasPar2,
})
const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
if (updates.length > 0) {
await trx.rawQuery(
`WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
FROM v GROUP BY id)
UPDATE releases r SET
files = r.files + a.files,
segments = r.segments + a.segments,
bytes = r.bytes + a.bytes,
chunks = r.chunks + a.chunks,
posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
file_total = coalesce(r.file_total, a.file_total),
first_article = least(r.first_article, a.first_article),
last_article = greatest(r.last_article, a.last_article),
readable = r.readable OR a.readable,
has_par2 = r.has_par2 OR a.has_par2,
-- As it stands (the counter's files all found); settling makes it final.
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
last_seen_at = now()
FROM a WHERE r.id = a.id`,
[JSON.stringify(updates)]
)
part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
}
const fresh = part.filter((_, idx) => !joined.has(idx))
if (fresh.length > 0) {
// The first upload of a key keeps the key; later ones are key:2, key:3, …
const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
const inserts = fresh.map((r) => {
const n = (taken.get(r.key) ?? 0) + 1
taken.set(r.key, n)
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
})
const { rows } = await trx.rawQuery(
`INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
RETURNING id, release_key`,
[groupId, JSON.stringify(inserts)]
)
const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
for (const r of inserts) ids.set(slot(r.key, r.upload), byReleaseKey.get(r.release_key)!)
out.newReleases += rows.length
}
}
// One write-once chunk per release.
for (let i = 0; i < sorted.length; i += 500) {
const rows = sorted.slice(i, i + 500).map((r) => ({
release_id: ids.get(r.key)!,
release_id: ids.get(slot(r.key, r.upload))!,
range_id: rangeId,
batch_from: batchFrom,
files: r.files,
@@ -171,12 +207,12 @@ class BatchStore {
if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
}
const kept = candidates.filter((c) => ids.has(c.releaseKey))
const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
for (let i = 0; i < kept.length; i += 2000) {
const rows = kept
.slice(i, i + 2000)
.map((c) => ({
release_id: ids.get(c.releaseKey)!,
release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
kind: c.kind,
message_id: c.messageId,
file: c.file,
app/services/fragment_merger.ts
+4 -2
View File
@@ -3,7 +3,7 @@ import BatchStore from '#services/batch_store'
import ChunkPartitions from '#services/chunk_partitions'
import RuleSets from '#services/rule_sets'
import { RuleSet } from '#lib/rules/index'
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import type { AggregatedFile } from '#lib/files/index'
import type { EdgeSegment } from '#lib/edge/index'
@@ -85,10 +85,12 @@ class FragmentMerger {
for (const [key, { files, rangeId }] of byRelease) {
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })))
if (verdict.rule !== null) continue
releases.push({ rangeId, chunk: chunkOf(key, files), candidates: candidatesOf(key, files) })
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
}
await ChunkPartitions.ensure()
await db.transaction(async (trx) => {
// In key order, as batches lock keys (no deadlocks).
releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()