usenet-indexer/tests/functional/pending_namer.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { deflateSync } from 'node:zlib'
import { PendingNamer } from '#services/pending_namer'
import { PENDING } from '#services/pending_ladder'
import { RangeIndexer } from '#lib/edge/index'
import { DEFAULT_RULES } from '#lib/rules/index'
import { decodeChunk } from '#lib/chunks/index'
import { identityKey, releaseKey } from '#lib/release/keys'

const GROUP = 'alt.binaries.namer'

/** One overview line as XOVER gives it (the corpus stores the reply's text). */
function line(n: number, subject: string, minutes: number) {
  const date = new Date(Date.UTC(2016, 0, 4, 10, minutes)).toUTCString()
  return `${n}\t${subject}\tp <p@x>\t${date}\t<u${n}@test>\t\t1000\t10\tXref: news.example ${GROUP}:${n}`
}

test.group('Pending namer', (group) => {
  let dir = ''
  group.each.setup(async () => {
    const rollback = await testUtils.db().withGlobalTransaction()
    dir = await mkdtemp(join(tmpdir(), 'namer-'))
    return rollback
  })
  group.each.teardown(() => rm(dir, { recursive: true, force: true }))

  test('a named pending release is read again from the corpus and written to the catalog with its chunk and candidates', async ({
    assert,
  }) => {
    const groupId = Number(
      (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id'))[0].id
    )
    // The range stored in the corpus: one chunk with a random-named upload (4 parts and its par2) among others.
    const lines = [
      line(5, 'Some.Show [1/1] - "Some.Show.mkv" yEnc (1/1)', 0),
      ...[1, 2, 3, 4].map((i) =>
        line(9 + i, `[1/2] - "q3B9OrpKiCxUnzgtUGnp.part01.rar" yEnc (${i}/4)`, i)
      ),
      line(14, '[2/2] - "q3B9OrpKiCxUnzgtUGnp.par2" yEnc (1/1)', 5),
      line(20, 'Other.Thing [1/1] - "Other.Thing.mkv" yEnc (1/1)', 6),
    ]
    await mkdir(join(dir, GROUP, '1-200000'), { recursive: true })
    await writeFile(
      join(dir, GROUP, '1-200000', '0000-1-100000.gz'),
      deflateSync(Buffer.from(lines.join('\r\n') + '\r\n', 'latin1'))
    )
    await db.table('ranges').insert({
      group_id: groupId,
      start_id: 1,
      end_id: 200_000,
      next_id: 200_001,
      status: 'done',
      stored_at: new Date(),
      capture: true,
    })

    // The key the edge gave it: the same identity rules, run here over the same lines.
    const ix = new RangeIndexer({
      group: GROUP,
      start: 1,
      end: 200_000,
      rules: { ...DEFAULT_RULES, droppedSignatures: [], priorities: { [GROUP]: 1 } },
    })
    ix.addText(lines.join('\r\n'), 200_001)
    ix.finish()
    const edge = ix.take().releases.find((r) => !r.readable)!
    assert.equal(edge.firstArticle, 10)
    const key = releaseKey(edge.key, groupId, 10)
    await db.table('pending_releases').insert({
      release_key: key,
      identity_key: identityKey(edge.key),
      group_id: groupId,
      name: 'q3B9OrpKiCxUnzgtUGnp',
      poster: 'p <p@x>',
      first_article: 10,
      last_article: 14,
      files: 2,
      segments: 5,
      bytes: 5000,
      file_total: 2,
      tool: 'other',
      candidates: JSON.stringify([
        { kind: 'archive', messageId: 'u10@test', file: 'q3B9OrpKiCxUnzgtUGnp.part01.rar' },
      ]),
      status: PENDING.naming,
      named: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
    })

    const namer = new PendingNamer(dir)
    const [row] = await namer.take(5)
    assert.equal(row.named, 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv')
    await namer.move(row)

    const release = await db.from('releases').where('release_key', key).firstOrFail()
    assert.include(release, {
      name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
      inner_name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
      source: 7,
      readable: true,
      files: 2,
      segments: '5',
      file_total: 2,
      complete: true,
      first_article: '10',
      last_article: '14',
    })
    assert.isNotNull(release.settled_at)
    const chunk = await db.from('release_chunks').where('release_id', release.id).firstOrFail()
    assert.deepEqual(
      decodeChunk(chunk.payload)
        .map((f) => [f.name, f.segments.length])
        .sort(),
      [
        ['q3B9OrpKiCxUnzgtUGnp.par2', 1],
        ['q3B9OrpKiCxUnzgtUGnp.part01.rar', 4],
      ]
    )
    assert.deepEqual(
      (await db.from('probe_candidates').where('release_id', release.id).orderBy('kind')).map(
        (c) => [c.kind, c.ready, c.unreadable]
      ),
      [
        ['archive', true, false],
        ['par2', true, false],
      ]
    )
    assert.deepEqual(
      await db.from('release_texts').where('release_id', release.id).select('kind', 'text'),
      [{ kind: 1, text: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv' }]
    )
    assert.equal(
      (await db.from('pending_releases').where('release_key', key).firstOrFail()).status,
      PENDING.named
    )
    // Nothing else of the chunk got in.
    assert.equal(Number((await db.from('releases').count('* as n').first())!.n), 1)
  })
})