The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog

pending_ladder.ts claims the oldest waiting pending releases' next target
(par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each
claim; a usable name (the largest named file of a par2 set or a listing, or a
lone file's yEnc name, neither packaging nor random) hands the release to the
namer, no name the next target or the end of the climb; climbs count on
pending_signatures per (tool, group, poster) and (tool, group), and a signature
with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are
discarded without a probe, the rules' sample excepted. pending_namer.ts reads
a named release's articles again from the corpus or by XOVER and writes it to
the catalog under the pending key (source weak-stem, the name as inner name)
with its chunk and candidates. Migration 6400; config PENDING_*.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/pending_namer.spec.ts

@@ -0,0 +1,143 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { deflateSync } from 'node:zlib'
+import { PendingNamer } from '#services/pending_namer'
+import { PENDING } from '#services/pending_ladder'
+import { RangeIndexer } from '#lib/edge/index'
+import { DEFAULT_RULES } from '#lib/rules/index'
+import { decodeChunk } from '#lib/chunks/index'
+import { identityKey, releaseKey } from '#lib/release/keys'
+
+const GROUP = 'alt.binaries.namer'
+
+/** One overview line as XOVER gives it (the corpus stores the reply's text). */
+function line(n: number, subject: string, minutes: number) {
+  const date = new Date(Date.UTC(2016, 0, 4, 10, minutes)).toUTCString()
+  return `${n}\t${subject}\tp <p@x>\t${date}\t<u${n}@test>\t\t1000\t10\tXref: news.example ${GROUP}:${n}`
+}
+
+test.group('Pending namer', (group) => {
+  let dir = ''
+  group.each.setup(async () => {
+    const rollback = await testUtils.db().withGlobalTransaction()
+    dir = await mkdtemp(join(tmpdir(), 'namer-'))
+    return rollback
+  })
+  group.each.teardown(() => rm(dir, { recursive: true, force: true }))
+
+  test('a named pending release is read again from the corpus and written to the catalog with its chunk and candidates', async ({
+    assert,
+  }) => {
+    const groupId = Number(
+      (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id'))[0].id
+    )
+    // The range stored in the corpus: one chunk with a random-named upload (4 parts and its par2) among others.
+    const lines = [
+      line(5, 'Some.Show [1/1] - "Some.Show.mkv" yEnc (1/1)', 0),
+      ...[1, 2, 3, 4].map((i) =>
+        line(9 + i, `[1/2] - "q3B9OrpKiCxUnzgtUGnp.part01.rar" yEnc (${i}/4)`, i)
+      ),
+      line(14, '[2/2] - "q3B9OrpKiCxUnzgtUGnp.par2" yEnc (1/1)', 5),
+      line(20, 'Other.Thing [1/1] - "Other.Thing.mkv" yEnc (1/1)', 6),
+    ]
+    await mkdir(join(dir, GROUP, '1-200000'), { recursive: true })
+    await writeFile(
+      join(dir, GROUP, '1-200000', '0000-1-100000.gz'),
+      deflateSync(Buffer.from(lines.join('\r\n') + '\r\n', 'latin1'))
+    )
+    await db.table('ranges').insert({
+      group_id: groupId,
+      start_id: 1,
+      end_id: 200_000,
+      next_id: 200_001,
+      status: 'done',
+      stored_at: new Date(),
+      capture: true,
+    })
+
+    // The key the edge gave it: the same identity rules, run here over the same lines.
+    const ix = new RangeIndexer({
+      group: GROUP,
+      start: 1,
+      end: 200_000,
+      rules: { ...DEFAULT_RULES, droppedSignatures: [], priorities: { [GROUP]: 1 } },
+    })
+    ix.addText(lines.join('\r\n'), 200_001)
+    ix.finish()
+    const edge = ix.take().releases.find((r) => !r.readable)!
+    assert.equal(edge.firstArticle, 10)
+    const key = releaseKey(edge.key, groupId, 10)
+    await db.table('pending_releases').insert({
+      release_key: key,
+      identity_key: identityKey(edge.key),
+      group_id: groupId,
+      name: 'q3B9OrpKiCxUnzgtUGnp',
+      poster: 'p <p@x>',
+      first_article: 10,
+      last_article: 14,
+      files: 2,
+      segments: 5,
+      bytes: 5000,
+      file_total: 2,
+      tool: 'other',
+      candidates: JSON.stringify([
+        { kind: 'archive', messageId: 'u10@test', file: 'q3B9OrpKiCxUnzgtUGnp.part01.rar' },
+      ]),
+      status: PENDING.naming,
+      named: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
+    })
+
+    const namer = new PendingNamer(dir)
+    const [row] = await namer.take(5)
+    assert.equal(row.named, 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv')
+    await namer.move(row)
+
+    const release = await db.from('releases').where('release_key', key).firstOrFail()
+    assert.include(release, {
+      name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
+      inner_name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
+      source: 7,
+      readable: true,
+      files: 2,
+      segments: '5',
+      file_total: 2,
+      complete: true,
+      first_article: '10',
+      last_article: '14',
+    })
+    assert.isNotNull(release.settled_at)
+    const chunk = await db.from('release_chunks').where('release_id', release.id).firstOrFail()
+    assert.deepEqual(
+      decodeChunk(chunk.payload)
+        .map((f) => [f.name, f.segments.length])
+        .sort(),
+      [
+        ['q3B9OrpKiCxUnzgtUGnp.par2', 1],
+        ['q3B9OrpKiCxUnzgtUGnp.part01.rar', 4],
+      ]
+    )
+    assert.deepEqual(
+      (await db.from('probe_candidates').where('release_id', release.id).orderBy('kind')).map(
+        (c) => [c.kind, c.ready, c.unreadable]
+      ),
+      [
+        ['archive', true, false],
+        ['par2', true, false],
+      ]
+    )
+    assert.deepEqual(
+      await db.from('release_texts').where('release_id', release.id).select('kind', 'text'),
+      [{ kind: 1, text: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv' }]
+    )
+    assert.equal(
+      (await db.from('pending_releases').where('release_key', key).firstOrFail()).status,
+      PENDING.named
+    )
+    // Nothing else of the chunk got in.
+    assert.equal(Number((await db.from('releases').count('* as n').first())!.n), 1)
+  })
+})