import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import coordinatorConfig from '#config/coordinator' import StatusService from '#services/status' test.group('Status', (group) => { let dir = '' let snapshotDir = '' group.each.setup(async () => { const rollback = await testUtils.db().withGlobalTransaction() dir = await mkdtemp(join(tmpdir(), 'snapshots-')) snapshotDir = coordinatorConfig.snapshotDir coordinatorConfig.snapshotDir = dir return rollback }) group.each.teardown(async () => { coordinatorConfig.snapshotDir = snapshotDir await rm(dir, { recursive: true, force: true }) }) test('the ranges by band, the held groups with their snapshots, the backlogs', async ({ assert, }) => { const [g] = await db .table('groups') .insert({ name: 'alt.binaries.status', priority: 1 }) .returning('id') const [n] = await db .table('nodes') .insert({ name: 'status-node', last_seen_at: new Date() }) .returning('id') const ranges = [ { start_id: 1, end_id: 100, next_id: 1, priority: -3 }, { start_id: 101, end_id: 200, next_id: 101, priority: -1 }, { start_id: 201, end_id: 300, next_id: 301, priority: 0, capture: true, status: 'done', done_at: new Date(), }, { start_id: 301, end_id: 400, next_id: 351, priority: 0, capture: true, status: 'leased', node_id: n.id, spooled_chunks: 2, }, { start_id: 401, end_id: 500, next_id: 401, priority: 2, capture: true, stored_at: new Date(), }, { start_id: 501, end_id: 600, next_id: 501, priority: 500 }, { start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' }, ] const ids = await db .table('ranges') .multiInsert(ranges.map((r) => ({ group_id: g.id, ...r }))) .returning('id') await db.table('group_leases').multiInsert([ { group_id: g.id, lane: 'history', node_id: n.id, leased_at: new Date(), heartbeat_at: new Date(), range_id: ids[3].id, snapshot_next: 301, snapshot_at: new Date(Date.now() - 600_000), ranges_done: 1, headers: 100, }, { group_id: g.id, lane: 'tail', node_id: null, heartbeat_at: new Date(Date.now() - 3600_000), snapshot_next: 1, snapshot_at: new Date(Date.now() - 3600_000), }, ]) await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000)) await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000)) // A group planned from its low watermark 1000: its ancient range counts in no band and is never leased. const [old] = await db .table('groups') .insert({ name: 'alt.binaries.ancient', priority: 2 }) .returning('id') await db.table('ranges').multiInsert([ { group_id: old.id, start_id: 1, end_id: 999, next_id: 1, priority: 0, status: 'ancient' }, { group_id: old.id, start_id: 1000, end_id: 1099, next_id: 1000, priority: 0 }, ]) const s = await StatusService.collect() assert.deepEqual( s.bands.map((b) => [ b.band, b.planned, b.plannedArticles, b.leased, b.done, b.doneArticles, b.failed, ]), [ ['tail', 1, 100, 0, 0, 0, 0], ['raised', 1, 100, 0, 0, 0, 0], ['window 0', 0, 0, 1, 1, 100, 0], ['window 2', 1, 100, 0, 0, 0, 0], ['history 0-999', 2, 200, 0, 0, 0, 0], ['history 4000+', 0, 0, 0, 0, 0, 1], ] ) assert.deepEqual(s.ancient, { groups: 1, articles: 999 }) assert.deepEqual(s.lastHour, { indexedArticles: 100, doneRanges: 1, storedArticles: 100, storedRanges: 1, }) assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 }) assert.deepEqual(s.spool, { ranges: 1, chunks: 2 }) assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 }) assert.lengthOf(s.held, 1) assert.include(s.held[0], { node: 'status-node', group: 'alt.binaries.status', lane: 'history', snapshotNext: 301, snapshotMb: 2, rangesDone: 1, headers: 100, }) assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 }) assert.isAtLeast(s.held[0].snapshotSeconds!, 599) assert.equal(s.snapshots.count, 2) assert.deepEqual( s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]), [ ['alt.binaries.status', 'history', 2], ['alt.binaries.status', 'tail', 0.001], ] ) assert.equal(s.releases.provisional, 0) assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 }) assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0) assert.isAbove(s.catalog.gb, 0) const text = StatusService.render(s) assert.include(text, 'window 0') assert.include(text, 'alt.binaries.status history status-node 301-400 351') assert.include(text, 'Snapshots: 2 files') assert.include(text, 'Ancient: 999 articles below the low watermark of 1 groups') }) })