Rebuild: re-uploads are releases of their own

A poster re-uploading under the same name made one release of all
uploads: in a sample, 24% of releases with 2+ files had a file name more
than once (4.3% of files extra copies), 70% of those copies posted a day
or more apart; their NZBs listed each copy.

- File assembly keeps a re-upload's copy apart: a part already in every
  open copy, posted more than 3 h from them, starts another copy; a
  missing part fills the nearest copy (a re-send of what was missing),
  a re-sent part near a copy that has it is ignored as before.
- A key's files in a batch split into uploads where posting pauses for
  more than 3 h, counted from the latest article posted so far (a file
  that takes hours to post doesn't split it); each upload is judged and
  chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry
  their upload).
- The catalog matches a chunk to the release of its key whose articles
  the range's group owns and whose posting window is within 3 h, else
  starts one (key, key:2, …): releases get identity_key, group_id,
  last_posted_at. Keys are locked in order (advisory locks), so
  concurrent batches neither deadlock nor both start one; the merger
  writes in key order too. The same upload multi-posted to other groups
  is a release per group.
- The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies
  come later (releases:upload-gaps). releases:duplicates measures the
  duplicate names.

Both implementations; check.ts compares the windows and uploads too
(identical on all captures; Rust tests mirror the TypeScript cases).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/batch_store.spec.ts

@@ -32,6 +32,32 @@ function indexer() {
   return new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } }, edgeArticles: 1000 })
 }
 
+/** One file's articles, posted `minutes` apart from `start` (an ISO time), in `group`. */
+function upload(at: number, name: string, parts: number, counter: string, start: string, group = GROUP, minutes = 1): Overview[] {
+  return Array.from({ length: parts }, (_, i) => {
+    id++
+    const date = new Date(Date.parse(start) + i * minutes * 60_000).toUTCString().replace('GMT', '+0000')
+    return [at + i, { 'subject': `${counter} - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': date, 'message-id': `<up${id}@test>`, ':bytes': '700000', 'xref': `news.example ${group}:${at + i}` }] as Overview
+  })
+}
+
+async function rangeIn(groupName: string, start: number, nodeId?: number) {
+  const g = (await db.from('groups').where('name', groupName).first()) ?? (await db.table('groups').insert({ name: groupName, priority: groupName === GROUP ? 1 : 2 }).returning('*'))[0]
+  const node = nodeId ?? Number((await db.table('nodes').insert({ name: `n${groupName}${start}` }).returning('id'))[0].id)
+  const [range] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: start + 99_999, next_id: start, status: 'leased', node_id: node, rule_version: 1 }).returning('id')
+  return { rangeId: Number(range.id), nodeId: node }
+}
+
+async function storeIn(groupName: string, start: number, overviews: Overview[]) {
+  const { rangeId, nodeId } = await rangeIn(groupName, start)
+  const ix = new RangeIndexer({ group: groupName, start, end: start + 99_999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1, 'alt.binaries.other': 2 } }, edgeArticles: 1 })
+  ix.add(overviews, start + 100_000)
+  ix.finish()
+  await BatchStore.store(rangeId, nodeId, ix.take())
+}
+
+const releasesOf = () => db.from('releases').orderBy('id').select('release_key', 'identity_key', 'files', 'posted_at', 'last_posted_at')
+
 test.group('BatchStore', (group) => {
   group.each.setup(() => testUtils.db().withGlobalTransaction())
 
@@ -73,4 +99,32 @@ test.group('BatchStore', (group) => {
     ix.add(post(100_000, 'A.Release.2020.mkv', 1, 'A [1/1]'), 200_000)
     await assert.rejects(() => BatchStore.store(rangeId, nodeId + 1, ix.take()), /not leased to this node/)
   })
+  test('a re-upload is a release of its own: in one range, or a later range; a continuation joins', async ({ assert }) => {
+    // One range: the same file posted Monday and again Thursday.
+    await storeIn(GROUP, 1, [
+      ...upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-05T10:00:00Z'),
+      ...upload(20, 'Some.Show.S01E01.1080p.par2', 1, 'Some.Show.S01E01.1080p [2/2]', '2026-10-05T10:05:00Z'),
+      ...upload(500, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-08T10:00:00Z'),
+    ])
+    let rows = await releasesOf()
+    assert.lengthOf(rows, 2)
+    assert.equal(rows[1].release_key, `${rows[0].release_key}:2`)
+    assert.deepEqual(rows.map((r) => [r.identity_key, r.files]), [[rows[0].release_key, 2], [rows[0].release_key, 1]])
+
+    // The next range: Thursday's par2 an hour later joins Thursday's release; a Saturday copy is a third.
+    await storeIn(GROUP, 100_001, upload(100_010, 'Some.Show.S01E01.1080p.par2', 1, 'Some.Show.S01E01.1080p [2/2]', '2026-10-08T11:00:00Z'))
+    await storeIn(GROUP, 200_001, upload(200_010, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-10T10:00:00Z'))
+    rows = await releasesOf()
+    assert.deepEqual(rows.map((r) => [r.release_key.split(':')[1] ?? '1', r.files]), [['1', 2], ['2', 2], ['3', 1]])
+    const chunks = await db.from('release_chunks').count('* as n').first()
+    assert.equal(Number(chunks!.n), 4)
+  })
+
+  test('the same upload in another group (multi-posted) is a release of its own', async ({ assert }) => {
+    await storeIn(GROUP, 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:00:00Z'))
+    await storeIn('alt.binaries.other', 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:01:00Z', 'alt.binaries.other'))
+    const rows = await releasesOf()
+    assert.lengthOf(rows, 2)
+    assert.equal(rows[0].identity_key, rows[1].identity_key)
+  })
 })

tests/functional/settler.spec.ts

@@ -9,7 +9,7 @@ test.group('Settler', (group) => {
   test('settles quiet releases: complete by the [i/N] counter, unknown without one', async ({ assert }) => {
     const old = new Date(Date.now() - 7 * 3600 * 1000)
     const row = (key: string, files: number, fileTotal: number | null, lastSeen: Date) => ({
-      release_key: key.padEnd(40, '0'), name: key, source: 'subject', groups: '[]', files, file_total: fileTotal, last_seen_at: lastSeen,
+      release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'subject', groups: '[]', files, file_total: fileTotal, last_seen_at: lastSeen,
     })
     await db.table('releases').multiInsert([row('a', 10, 10, old), row('b', 8, 10, old), row('c', 3, null, old), row('d', 10, 10, new Date())])
     assert.equal(await Settler.run(), 3)

tests/unit/edge/range_indexer.spec.ts

@@ -81,4 +81,20 @@ test.group('edge / taking over a range', () => {
     indexer.add(post(500_001, 'A.Release.2020.mkv', 1), 600_000)
     assert.deepEqual([indexer.take().from, indexer.take().from], [500_001, 600_000])
   })
+  test('uploads split where posting pauses for more than 3 hours, counted from the latest article', ({ assert }) => {
+    const art = (n: number, subject: string, minutes: number): Overview => [n, { 'subject': subject, 'from': 'p <p@x>', 'date': new Date(Date.UTC(2026, 9, 5, 10) + minutes * 60_000).toUTCString(), 'message-id': `<u${n}@test>`, ':bytes': '1000', 'xref': `news.example ${GROUP}:${n}` }]
+    const ix = new RangeIndexer({ group: GROUP, start: 1, end: 10_000, rules })
+    ix.add([
+      // A big file posted over 4 hours, then its par2 right after: one upload.
+      ...Array.from({ length: 5 }, (_, i) => art(100 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, i * 60)),
+      art(200, 'Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)', 245),
+      // The same file again a day later: another.
+      ...Array.from({ length: 5 }, (_, i) => art(300 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, 1440 + i)),
+    ], 10_001)
+    ix.finish()
+    const batch = ix.take()
+    assert.deepEqual(batch.releases.map((r) => [r.upload, r.files]), [[0, 2], [1, 1]])
+    assert.equal(batch.releases[0].postedLast! - batch.releases[0].postedAt!, 245 * 60_000)
+    assert.equal(batch.releases[0].key, batch.releases[1].key)
+  })
 })

tests/unit/files/aggregator.spec.ts

@@ -36,4 +36,20 @@ test.group('aggregate / FileAggregator', () => {
     assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
     assert.equal(aggregator.openFiles, 0)
   })
+  test('a re-upload is a copy of its own; a missing part sent later fills the first', ({ assert }) => {
+    const hour = 3_600_000
+    const at = (number: number, h: number, id: string) => ({ ...seg('a', number, 3, `${id}@test`), postedAt: h * hour })
+    const aggregator = new FileAggregator()
+    // Parts 1 and 2 at 10:00; part 1 again right after (a re-send: ignored).
+    assert.lengthOf(aggregator.add([at(1, 10, 'a1'), at(2, 10, 'a2'), at(1, 10.1, 'resend')]), 0)
+    // A day later: parts 1 and 2 again (a re-upload, another copy), and part 3 (fills the nearest copy lacking it: the new one).
+    assert.lengthOf(aggregator.add([at(1, 34, 'b1'), at(2, 34, 'b2')]), 0)
+    assert.equal(aggregator.openFiles, 2)
+    const [second] = aggregator.add([at(3, 34.2, 'b3')])
+    assert.deepEqual(second.segments.map((s) => s.messageId), ['b1@test', 'b2@test', 'b3@test'])
+    // The first copy's missing part, sent two days later: it completes the first.
+    const [first] = aggregator.add([at(3, 58, 'a3')])
+    assert.deepEqual(first.segments.map((s) => s.messageId), ['a1@test', 'a2@test', 'a3@test'])
+    assert.equal(aggregator.openFiles, 0)
+  })
 })