Edge aggregation: seed:range indexes history one row per file

A seeding node fetches a range over one NNTP connection, parses headers
with the live rules, groups them into files in memory (lib/aggregate) and
stores per complete file its release, the file and a completed binary
(SeedService), instead of one parts row per article. What doesn't complete
within the window goes through the regular ingest path as fragments.
`--dry-run` only fetches and aggregates, to measure a node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/seed.spec.ts

@@ -0,0 +1,47 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import SeedService from '#services/SeedService'
+import QueueService from '#services/QueueService'
+import { FileAggregator } from '#lib/aggregate/index'
+import { segmentsOf } from '#services/FileStore'
+
+function header(subject: string, n: number) {
+  return {
+    'subject': subject,
+    'from': 'seeder <seed@example.invalid>',
+    'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
+    'message-id': `<seed-${n}-${subject.length}@test>`,
+    ':bytes': '700000',
+  }
+}
+
+test.group('SeedService', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('stores aggregated files with their release and a completed binary', async ({ assert }) => {
+    const aggregator = new FileAggregator<IngestSegment>()
+    const subjects = [1, 2].map((i) => `Seed.Test.S01E01.1080p - [1/2] - "Seed.Test.S01E01.1080p.part${i}.mkv" yEnc (%d/3)`)
+    const segments = subjects.flatMap((s) => [1, 2, 3].map((n) => segmentFromHeader(header(s.replace('%d', String(n)), n), 'alt.binaries.test', n)!))
+    const files = aggregator.add(segments)
+    assert.lengthOf(files, 2)
+
+    const result = await SeedService.store(files)
+    assert.include(result, { files: 2, existing: 0, releases: 1 })
+
+    const stored = await db.from('files').where('filename', 'Seed.Test.S01E01.1080p.part1.mkv').first()
+    assert.equal(stored.parts, 3)
+    assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
+    const release = await db.from('releases').where('id', stored.release_id).first()
+    assert.equal(release.name, segments[0].release!.name) // named as live indexing would
+    const binary = await db.from('binaries').where('file_key', stored.file_key).first()
+    assert.equal(Number(binary.file_id), Number(stored.id))
+    assert.isNotNull(binary.completed_at)
+
+    // The same files again (a crosspost, a second run): nothing new.
+    const again = await SeedService.store(files)
+    assert.include(again, { files: 0, existing: 2 })
+  })
+})

tests/unit/aggregate.spec.ts

@@ -0,0 +1,39 @@
+import { test } from '@japa/runner'
+import { FileAggregator, type AggregatableSegment } from '#lib/aggregate/index'
+
+const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
+  fileKey,
+  number,
+  totalParts,
+  messageId,
+  bytes: 100,
+})
+
+test.group('aggregate / FileAggregator', () => {
+  test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
+    const aggregator = new FileAggregator()
+    assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
+    assert.equal(aggregator.openFiles, 1)
+    const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
+    assert.equal(done.fileKey, 'a')
+    assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
+    assert.equal(aggregator.openFiles, 0)
+  })
+
+  test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
+    const aggregator = new FileAggregator()
+    const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
+    assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
+    assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
+  })
+
+  test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
+    const aggregator = new FileAggregator()
+    aggregator.add([seg('old', 1, 3)])
+    aggregator.add([seg('new', 1, 3)])
+    aggregator.add([seg('new', 2, 3)])
+    assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
+    assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
+    assert.equal(aggregator.openFiles, 0)
+  })
+})