usenet-indexer/tests/functional/batch_store.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import BatchStore, { StaleBatch } from '#services/batch_store'
import { RangeIndexer } from '#lib/edge/index'
import { decodeChunk } from '#lib/chunks/index'
import { DEFAULT_RULES } from '#lib/rules/index'
import type { Overview } from '#lib/nntp/index'

const GROUP = 'alt.binaries.test'
let id = 0

function post(at: number, name: string, parts: number, counter: string, tool: 'ngPost' | 'plain' = 'plain'): Overview[] {
  return Array.from({ length: parts }, (_, i) => {
    id++
    const mid = tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
    return [at + i, { 'subject': `${counter} - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<${mid}>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at + i}` }] as Overview
  })
}

async function leasedRange() {
  const [group] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
  const [node] = await db.table('nodes').insert({ name: 'test-node' }).returning('id')
  const [range] = await db
    .table('ranges')
    .insert({ group_id: group.id, start_id: 1, end_id: 1_000_000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 })
    .returning('id')
  return { rangeId: Number(range.id), nodeId: Number(node.id) }
}

function indexer() {
  return new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } }, edgeArticles: 1000 })
}

/** One file's articles, posted `minutes` apart from `start` (an ISO time), in `group`. */
function upload(at: number, name: string, parts: number, counter: string, start: string, group = GROUP, minutes = 1): Overview[] {
  return Array.from({ length: parts }, (_, i) => {
    id++
    const date = new Date(Date.parse(start) + i * minutes * 60_000).toUTCString().replace('GMT', '+0000')
    return [at + i, { 'subject': `${counter} - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': date, 'message-id': `<up${id}@test>`, ':bytes': '700000', 'xref': `news.example ${group}:${at + i}` }] as Overview
  })
}

async function rangeIn(groupName: string, start: number, nodeId?: number) {
  const g = (await db.from('groups').where('name', groupName).first()) ?? (await db.table('groups').insert({ name: groupName, priority: groupName === GROUP ? 1 : 2 }).returning('*'))[0]
  const node = nodeId ?? Number((await db.table('nodes').insert({ name: `n${groupName}${start}` }).returning('id'))[0].id)
  const [range] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: start + 99_999, next_id: start, status: 'leased', node_id: node, rule_version: 1 }).returning('id')
  return { rangeId: Number(range.id), nodeId: node }
}

async function storeIn(groupName: string, start: number, overviews: Overview[]) {
  const { rangeId, nodeId } = await rangeIn(groupName, start)
  const ix = new RangeIndexer({ group: groupName, start, end: start + 99_999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1, 'alt.binaries.other': 2 } }, edgeArticles: 1 })
  ix.add(overviews, start + 100_000)
  ix.finish()
  await BatchStore.store(rangeId, nodeId, ix.take())
}

const releasesOf = () => db.from('releases').orderBy('id').select('release_key', 'identity_key', 'files', 'posted_at', 'last_posted_at')

test.group('BatchStore', (group) => {
  group.each.setup(() => testUtils.db().withGlobalTransaction())

  test('stores releases, their chunks, probe candidates, rule counts and fragments, and moves the resume point', async ({ assert }) => {
    const { rangeId, nodeId } = await leasedRange()
    const ix = indexer()
    ix.add([
      ...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 2, 'Some.Show.S01E01 [1/2]'),
      ...post(100_010, 'Some.Show.S01E01.1080p.WEB.par2', 1, 'Some.Show.S01E01 [2/2]'),
      ...post(100_020, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 3, '[1/1]', 'ngPost'),
      ...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(1),
    ], 500_000)
    const first = ix.take()
    const stored = await BatchStore.store(rangeId, nodeId, first)
    assert.include(stored, { releases: 1, newReleases: 1, candidates: 2, fragments: 0 })

    const release = await db.from('releases').first()
    assert.include(release, { files: 2, segments: '3', chunks: 1, readable: true, has_par2: true, file_total: 2 })
    const chunk = await db.from('release_chunks').where('release_id', release.id).first()
    assert.deepEqual(decodeChunk(chunk.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar'])
    assert.deepEqual((await db.from('probe_candidates').orderBy('kind')).map((c) => c.kind), ['archive', 'par2'])
    assert.deepEqual((await db.from('rule_counts')).map((c) => [c.rule, c.key, Number(c.articles)]), [['garbage', 'ngPost alt.binaries.test', 3]])
    assert.equal(Number((await db.from('ranges').where('id', rangeId).first()).next_id), 500_000)

    // The same batch again (a retry after a lost reply) is refused, and changes nothing.
    await assert.rejects(() => BatchStore.store(rangeId, nodeId, first), StaleBatch)
    assert.equal((await db.from('releases').first()).files, 2)

    // The end of the range: the cut-off file near the start edge is kept as a fragment.
    ix.finish()
    const last = ix.take()
    assert.include(await BatchStore.store(rangeId, nodeId, last), { fragments: 1 })
    assert.equal(Number((await db.from('ranges').where('id', rangeId).first()).next_id), 1_000_001)
  })

  test('a range leased to another node is refused', async ({ assert }) => {
    const { rangeId, nodeId } = await leasedRange()
    const ix = indexer()
    ix.add(post(100_000, 'A.Release.2020.mkv', 1, 'A [1/1]'), 200_000)
    await assert.rejects(() => BatchStore.store(rangeId, nodeId + 1, ix.take()), /not leased to this node/)
  })
  test('a re-upload is a release of its own: in one range, or a later range; a continuation joins', async ({ assert }) => {
    // One range: the same file posted Monday and again Thursday.
    await storeIn(GROUP, 1, [
      ...upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-05T10:00:00Z'),
      ...upload(20, 'Some.Show.S01E01.1080p.par2', 1, 'Some.Show.S01E01.1080p [2/2]', '2026-10-05T10:05:00Z'),
      ...upload(500, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-08T10:00:00Z'),
    ])
    let rows = await releasesOf()
    assert.lengthOf(rows, 2)
    assert.equal(rows[1].release_key, `${rows[0].release_key}:2`)
    assert.deepEqual(rows.map((r) => [r.identity_key, r.files]), [[rows[0].release_key, 2], [rows[0].release_key, 1]])

    // The next range: Thursday's par2 an hour later joins Thursday's release; a Saturday copy is a third.
    await storeIn(GROUP, 100_001, upload(100_010, 'Some.Show.S01E01.1080p.par2', 1, 'Some.Show.S01E01.1080p [2/2]', '2026-10-08T11:00:00Z'))
    await storeIn(GROUP, 200_001, upload(200_010, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/2]', '2026-10-10T10:00:00Z'))
    rows = await releasesOf()
    assert.deepEqual(rows.map((r) => [r.release_key.split(':')[1] ?? '1', r.files]), [['1', 2], ['2', 2], ['3', 1]])
    const chunks = await db.from('release_chunks').count('* as n').first()
    assert.equal(Number(chunks!.n), 4)
  })

  test('the same upload in another group (multi-posted) is a release of its own', async ({ assert }) => {
    await storeIn(GROUP, 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:00:00Z'))
    await storeIn('alt.binaries.other', 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:01:00Z', 'alt.binaries.other'))
    const rows = await releasesOf()
    assert.lengthOf(rows, 2)
    assert.equal(rows[0].identity_key, rows[1].identity_key)
  })

  test('an upload spread over groups (other files in another group) is one release', async ({ assert }) => {
    await storeIn(GROUP, 1, upload(10, 'Some.Album-WEB-2026-FTD.part1.rar', 2, 'Some.Album-WEB-2026-FTD [2/3]', '2026-10-05T10:00:00Z'))
    await storeIn('alt.binaries.other', 1, upload(10, 'Some.Album-WEB-2026-FTD.part2.rar', 2, 'Some.Album-WEB-2026-FTD [3/3]', '2026-10-05T10:00:30Z', 'alt.binaries.other'))
    const rows = await releasesOf()
    assert.lengthOf(rows, 1)
    assert.equal(rows[0].files, 2)
  })

  test('a release key taken meanwhile by another group\'s writer is skipped for the next number', async ({ assert }) => {
    await storeIn(GROUP, 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:00:00Z'))
    const [first] = await releasesOf()
    // As if another group's batch numbered the key at the same moment: key:2 exists, not counted for this key.
    const [g] = await db.table('groups').insert({ name: 'alt.binaries.third', priority: 3 }).returning('id')
    await db.table('releases').insert({ release_key: `${first.identity_key}:2`, identity_key: 'someone-else', group_id: g.id, name: 'x', source: 'subject', poster: 'p', groups: JSON.stringify([]), files: 0, segments: 0, bytes: 0, chunks: 0 })
    await storeIn('alt.binaries.other', 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:01:00Z', 'alt.binaries.other'))
    const keys = (await releasesOf()).filter((r) => r.identity_key === first.identity_key).map((r) => r.release_key)
    assert.deepEqual(keys, [first.identity_key, `${first.identity_key}:3`])
  })
})