Pending ladder by class, twins by content, dropped classes at the edge

The prober names 6% of the unnamed releases it finishes and the pending
queue (44M) grows eleven times faster than it probes. A sample of 44,530
finished pending rows shows the headers predict a name: size bucket, name
shape, completeness, tool and group separate 0% classes from 95% ones,
while the learner keyed on tool and group alone (LOG.md (130)).

- lib/release/classes.ts, names::pending_class, SQL pending_class():
  a release's class (size × shape × partial, 1..70), the three in step.
- Migration 6800: pending_releases.klass and target_bytes (the first
  target's article; candidates carry bytes now), pending_classes with
  each class's climbs, names, waiting and verdict, the claim's index.
- The ladder claims the classes with the best record first (a prior for
  the unexplored, at most a quarter of a claim from one), the cheapest
  target first within one, then the oldest of the rest; records classes
  like signatures and drops them at the same thresholds; discards their
  waiting releases. Claimed rows sorted in code: an UPDATE returns none.
- The rule set's droppedClasses: the edge keeps unnamed releases of a
  dropped class out of the pending store (counted as garbage), the
  explored sample excepted.
- hash_twins.ts: a probed file whose first 16 KiB and size match a named
  catalog release's names a pending release the probe didn't, and sets
  the inner name of a catalog release without one.
- pending:classes classes the rows from before and seeds the record from
  the finished ones; rules:classes publishes the dropped classes;
  releases:readable also marks the rows' candidates unreadable.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/pending_classes.spec.ts

@@ -0,0 +1,28 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import { pendingClass } from '#lib/release/index'
+
+test.group('Pending classes / SQL pending_class()', () => {
+  test('the SQL function agrees with pendingClass', async ({ assert }) => {
+    const cases: [string, number, boolean][] = [
+      ['c04212a389cf69edc54767355ab5285f', 5_000_000, false],
+      ['c04212a389cf69edc54767355ab5285f', 5_000_000, true],
+      ['Zx5AjNaK-mh7CFg6Hf8uu', 500_000_000, false],
+      ['1650777167', 100, false],
+      ['mrrtmzrzmi4dgmrzmyytqmlbztczbx', 2_000_000_000, false],
+      ['PsNdnYiugfZ.rar', 20_000_000, false],
+      ['n5d1ueIf34X', 20_000_000, true],
+      ['sgkra vU7u2yuw9LDLCjDu04pOez', 999, false],
+      ['Description - Lt5nHjlo3RiuBb3C', 40_000_000, false],
+      ['ABCDEFGHIJKLMNOP', 0, false],
+    ]
+    for (const [name, bytes, partial] of cases) {
+      const { rows } = await db.rawQuery(`SELECT pending_class(?, ?::bigint, ?) AS k`, [
+        name,
+        bytes,
+        partial,
+      ])
+      assert.equal(Number(rows[0].k), pendingClass(name, bytes, partial), name)
+    }
+  })
+})

tests/functional/pending_ladder.spec.ts

@@ -9,7 +9,7 @@ let n = 0
 
 async function pending(
   name: string,
-  candidates: { kind: string; messageId: string; file: string }[],
+  candidates: { kind: string; messageId: string; file: string; bytes?: number }[],
   extra: Record<string, unknown> = {}
 ) {
   n++
@@ -206,3 +206,198 @@ test.group('Pending ladder', (group) => {
     assert.lengthOf(await db.from('pending_signatures'), 0)
   })
 })
+
+test.group('Pending ladder / classes', (group) => {
+  group.each.setup(async () => {
+    PendingLadder.forget()
+    const rollback = await testUtils.db().withGlobalTransaction()
+    groupId = Number(
+      (
+        await db
+          .table('groups')
+          .insert({ name: 'alt.binaries.classes', priority: 1 })
+          .returning('id')
+      )[0].id
+    )
+    n = 0
+    return rollback
+  })
+
+  test('a claim takes the class with the best record first, the cheapest target within it, the rest oldest first', async ({
+    assert,
+  }) => {
+    // Class 15 (1-10 MB hex, whole) named 9 of 10; class 16 (partial) named 0 of 50; class 0: rows from before.
+    await db.table('pending_classes').multiInsert([
+      { tool: 'other', group_id: groupId, klass: 15, probed: 10, named: 9, waiting: 2 },
+      { tool: 'other', group_id: groupId, klass: 16, probed: 50, named: 0, waiting: 1 },
+    ])
+    const old = await pending('00000000000000000000000000000031', [
+      { kind: 'lone', messageId: '<o@x>', file: 'o' },
+    ])
+    const poor = await pending(
+      '00000000000000000000000000000032',
+      [{ kind: 'lone', messageId: '<p@x>', file: 'p' }],
+      { klass: 16 }
+    )
+    const bigTarget = await pending(
+      '00000000000000000000000000000033',
+      [{ kind: 'archive', messageId: '<b@x>', file: 'b.rar', bytes: 700_000 }],
+      { klass: 15, target_bytes: 700_000 }
+    )
+    const smallTarget = await pending(
+      '00000000000000000000000000000034',
+      [{ kind: 'archive', messageId: '<s@x>', file: 's.rar', bytes: 50_000 }],
+      { klass: 15, target_bytes: 50_000 }
+    )
+    const probes = await PendingLadder.claim(10)
+    assert.deepEqual(
+      probes.map((p) => [p.releaseKey, p.klass]),
+      [
+        [smallTarget, 15],
+        [bigTarget, 15],
+        [poor, 16],
+        [old, 0],
+      ]
+    )
+    const waiting = await db
+      .from('pending_classes')
+      .where({ group_id: groupId })
+      .orderBy('klass')
+      .select('klass', 'waiting')
+    assert.deepEqual(
+      waiting.map((c) => [c.klass, c.waiting]),
+      [
+        [15, 0],
+        [16, 0],
+      ]
+    )
+    // Outcomes count on the class; a probe given back waits again.
+    await PendingLadder.record([
+      {
+        probe: probes[0],
+        status: 'done',
+        inspection: inspection('s.rar', ['Some.Movie.2016.1080p.mkv']),
+      },
+      { probe: probes[1], status: 'waiting' },
+      { probe: probes[2], status: 'missing' },
+    ])
+    const classes = await db.from('pending_classes').where({ group_id: groupId }).orderBy('klass')
+    assert.deepEqual(
+      classes.map((c) => [c.klass, c.probed, c.named, c.waiting, c.dropped]),
+      [
+        [15, 11, 10, 1, false],
+        [16, 51, 0, 0, false],
+      ]
+    )
+  })
+
+  test('a class that names nothing is dropped and its waiting releases discarded, the other classes untouched', async ({
+    assert,
+  }) => {
+    await db
+      .table('pending_classes')
+      .insert({ tool: 'other', group_id: groupId, klass: 16, probed: 99, named: 0, waiting: 1 })
+    const last = await pending(
+      '00000000000000000000000000000041',
+      [{ kind: 'lone', messageId: '<z@x>', file: 'z' }],
+      { klass: 16 }
+    )
+    const [probe] = await PendingLadder.claim(1)
+    assert.equal(probe.releaseKey, last)
+    await PendingLadder.record([{ probe, status: 'missing' }])
+    assert.include(
+      await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail(),
+      { probed: 100, named: 0, dropped: true }
+    )
+    const waiting = await pending(
+      '00000000000000000000000000000042',
+      [{ kind: 'lone', messageId: '<w@x>', file: 'w' }],
+      { klass: 16 }
+    )
+    const other = await pending(
+      '00000000000000000000000000000043',
+      [{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
+      { klass: 15 }
+    )
+    await db.rawQuery(
+      `UPDATE pending_classes SET waiting = waiting + 1 WHERE group_id = ? AND klass = 16`,
+      [groupId]
+    )
+    await db.table('rule_sets').insert({
+      version: 98,
+      rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
+      note: 'test',
+    })
+    assert.equal(await PendingLadder.discard(), 1)
+    const statusOf = async (key: string) =>
+      (
+        await db
+          .from('pending_releases')
+          .where('release_key', Buffer.from(key, 'hex'))
+          .firstOrFail()
+      ).status
+    assert.equal(await statusOf(waiting), PENDING.discarded)
+    assert.equal(await statusOf(other), PENDING.waiting)
+    assert.equal(
+      (await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail())
+        .waiting,
+      0
+    )
+  })
+
+  test('a twin by content names a release whose own probe did not', async ({ assert }) => {
+    // A catalog release with a readable name holds the same first 16 KiB and size.
+    const [poster] = await db.table('posters').insert({ poster: 'Twin <twin@x>' }).returning('id')
+    const [set] = await db
+      .table('group_sets')
+      .insert({ groups: JSON.stringify(['alt.binaries.classes']) })
+      .returning('id')
+    const [twin] = await db
+      .table('releases')
+      .insert({
+        release_key: Buffer.from('aa'.repeat(20), 'hex'),
+        identity_key: Buffer.from('bb'.repeat(20), 'hex'),
+        group_id: groupId,
+        group_set_id: set.id,
+        poster_id: poster.id,
+        name: 'Some.Movie.2016.1080p.BluRay-GRP',
+        source: 3,
+        posted_at: new Date('2026-10-01T00:00:00Z'),
+        first_article: 1,
+        last_article: 2,
+        files: 2,
+        segments: 2,
+        bytes: 1000,
+        readable: true,
+      })
+      .returning('id')
+    await db.table('file_hashes').insert({
+      candidate_id: 777_001,
+      release_id: twin.id,
+      hash16k: 'deadbeef'.repeat(4),
+      size: 1000,
+    })
+    const key = await pending('00000000000000000000000000000051', [
+      { kind: 'lone', messageId: '<t@x>', file: '00000000000000000000000000000051' },
+    ])
+    const [probe] = await PendingLadder.claim(1)
+    assert.equal(probe.releaseKey, key)
+    await PendingLadder.record([
+      {
+        probe,
+        status: 'done',
+        inspection: {
+          ...inspection('00000000000000000000000000000051'),
+          hash16k: 'deadbeef'.repeat(4),
+        },
+      },
+    ])
+    assert.include(
+      await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
+      {
+        status: PENDING.naming,
+        named: 'Some.Movie.2016.1080p.BluRay-GRP',
+      }
+    )
+  })
+})