Rebuild: re-uploads are releases of their own
Test and deploy / test (push) Successful in 26s
Test and deploy / deploy (push) Skipped

A poster re-uploading under the same name made one release of all
uploads: in a sample, 24% of releases with 2+ files had a file name more
than once (4.3% of files extra copies), 70% of those copies posted a day
or more apart; their NZBs listed each copy.

- File assembly keeps a re-upload's copy apart: a part already in every
  open copy, posted more than 3 h from them, starts another copy; a
  missing part fills the nearest copy (a re-send of what was missing),
  a re-sent part near a copy that has it is ignored as before.
- A key's files in a batch split into uploads where posting pauses for
  more than 3 h, counted from the latest article posted so far (a file
  that takes hours to post doesn't split it); each upload is judged and
  chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry
  their upload).
- The catalog matches a chunk to the release of its key whose articles
  the range's group owns and whose posting window is within 3 h, else
  starts one (key, key:2, …): releases get identity_key, group_id,
  last_posted_at. Keys are locked in order (advisory locks), so
  concurrent batches neither deadlock nor both start one; the merger
  writes in key order too. The same upload multi-posted to other groups
  is a release per group.
- The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies
  come later (releases:upload-gaps). releases:duplicates measures the
  duplicate names.

Both implementations; check.ts compares the windows and uploads too
(identical on all captures; Rust tests mirror the TypeScript cases).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 13:31:35 +02:00
1 parent d0f3497ec9
commit 0613984de9
15 files changed
+715 -140

No files matched your search

+7 -7
View File
@@ -50,9 +50,9 @@ const release = (r: Item, b: number) => {
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
batches.forEach((batch, b) => {
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
for (const r of batch.releases) ts.r.push(release({ key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
for (const s of batch.samples) ts.s.push({ b, key: s.key, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
for (const c of batch.candidates) ts.c.push({ b, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
})
@@ -65,15 +65,15 @@ for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
const { t, ...item } = JSON.parse(line)
if (t === 'b') b++
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
else if (t === 's') rs.s.push({ b, key: item.key, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
else rs[t].push({ b, ...item })
}
const keyOf: Record<string, (i: Item) => string> = {
b: (i) => String(i.b),
r: (i) => `${i.b} ${i.key}`,
s: (i) => `${i.b} ${i.key}`,
c: (i) => `${i.b} ${i.releaseKey} ${i.kind} ${i.messageId}`,
r: (i) => `${i.b} ${i.key} ${i.upload}`,
s: (i) => `${i.b} ${i.key} ${i.upload}`,
c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
n: (i) => `${i.b} ${i.rule} ${i.key}`,
f: (i) => `${i.b} ${i.fileKey}`,
}