usenet-indexer/tests/functional/status.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import coordinatorConfig from '#config/coordinator'
import StatusService from '#services/status'

test.group('Status', (group) => {
  let dir = ''
  let snapshotDir = ''
  group.each.setup(async () => {
    const rollback = await testUtils.db().withGlobalTransaction()
    dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
    snapshotDir = coordinatorConfig.snapshotDir
    coordinatorConfig.snapshotDir = dir
    return rollback
  })
  group.each.teardown(async () => {
    coordinatorConfig.snapshotDir = snapshotDir
    await rm(dir, { recursive: true, force: true })
  })

  test('the ranges by band, the held groups with their snapshots, the backlogs', async ({
    assert,
  }) => {
    const [g] = await db
      .table('groups')
      .insert({ name: 'alt.binaries.status', priority: 1 })
      .returning('id')
    const [n] = await db
      .table('nodes')
      .insert({ name: 'status-node', last_seen_at: new Date() })
      .returning('id')
    const ranges = [
      { start_id: 1, end_id: 100, next_id: 1, priority: -3 },
      { start_id: 101, end_id: 200, next_id: 101, priority: -1 },
      {
        start_id: 201,
        end_id: 300,
        next_id: 301,
        priority: 0,
        capture: true,
        status: 'done',
        done_at: new Date(),
      },
      {
        start_id: 301,
        end_id: 400,
        next_id: 351,
        priority: 0,
        capture: true,
        status: 'leased',
        node_id: n.id,
        spooled_chunks: 2,
      },
      {
        start_id: 401,
        end_id: 500,
        next_id: 401,
        priority: 2,
        capture: true,
        stored_at: new Date(),
      },
      { start_id: 501, end_id: 600, next_id: 501, priority: 500 },
      { start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' },
    ]
    const ids = await db
      .table('ranges')
      .multiInsert(ranges.map((r) => ({ group_id: g.id, ...r })))
      .returning('id')
    await db.table('group_leases').multiInsert([
      {
        group_id: g.id,
        lane: 'history',
        node_id: n.id,
        leased_at: new Date(),
        heartbeat_at: new Date(),
        range_id: ids[3].id,
        snapshot_next: 301,
        snapshot_at: new Date(Date.now() - 600_000),
        ranges_done: 1,
        headers: 100,
      },
      {
        group_id: g.id,
        lane: 'tail',
        node_id: null,
        heartbeat_at: new Date(Date.now() - 3600_000),
        snapshot_next: 1,
        snapshot_at: new Date(Date.now() - 3600_000),
      },
    ])
    await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000))
    await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000))
    // A group planned from its low watermark 1000: its ancient range counts in no band and is never leased.
    const [old] = await db
      .table('groups')
      .insert({ name: 'alt.binaries.ancient', priority: 2 })
      .returning('id')
    await db.table('ranges').multiInsert([
      { group_id: old.id, start_id: 1, end_id: 999, next_id: 1, priority: 0, status: 'ancient' },
      { group_id: old.id, start_id: 1000, end_id: 1099, next_id: 1000, priority: 0 },
    ])

    const s = await StatusService.collect()
    assert.deepEqual(
      s.bands.map((b) => [
        b.band,
        b.planned,
        b.plannedArticles,
        b.leased,
        b.done,
        b.doneArticles,
        b.failed,
      ]),
      [
        ['tail', 1, 100, 0, 0, 0, 0],
        ['raised', 1, 100, 0, 0, 0, 0],
        ['window 0', 0, 0, 1, 1, 100, 0],
        ['window 2', 1, 100, 0, 0, 0, 0],
        ['history 0-999', 2, 200, 0, 0, 0, 0],
        ['history 4000+', 0, 0, 0, 0, 0, 1],
      ]
    )
    assert.deepEqual(s.ancient, { groups: 1, articles: 999 })
    assert.deepEqual(s.lastHour, {
      indexedArticles: 100,
      doneRanges: 1,
      storedArticles: 100,
      storedRanges: 1,
    })
    assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 })
    assert.deepEqual(s.spool, { ranges: 1, chunks: 2 })
    assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 })
    assert.lengthOf(s.held, 1)
    assert.include(s.held[0], {
      node: 'status-node',
      group: 'alt.binaries.status',
      lane: 'history',
      snapshotNext: 301,
      snapshotMb: 2,
      rangesDone: 1,
      headers: 100,
    })
    assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 })
    assert.isAtLeast(s.held[0].snapshotSeconds!, 599)
    assert.equal(s.snapshots.count, 2)
    assert.deepEqual(
      s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]),
      [
        ['alt.binaries.status', 'history', 2],
        ['alt.binaries.status', 'tail', 0.001],
      ]
    )
    assert.equal(s.releases.provisional, 0)
    assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 })
    assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0)
    assert.isAbove(s.catalog.gb, 0)

    const text = StatusService.render(s)
    assert.include(text, 'window 0')
    assert.include(text, 'alt.binaries.status  history  status-node  301-400  351')
    assert.include(text, 'Snapshots: 2 files')
    assert.include(text, 'Ancient: 999 articles below the low watermark of 1 groups')
  })
})