Pending ladder by class, twins by content, dropped classes at the edge

The prober names 6% of the unnamed releases it finishes and the pending
queue (44M) grows eleven times faster than it probes. A sample of 44,530
finished pending rows shows the headers predict a name: size bucket, name
shape, completeness, tool and group separate 0% classes from 95% ones,
while the learner keyed on tool and group alone (LOG.md (130)).

- lib/release/classes.ts, names::pending_class, SQL pending_class():
  a release's class (size × shape × partial, 1..70), the three in step.
- Migration 6800: pending_releases.klass and target_bytes (the first
  target's article; candidates carry bytes now), pending_classes with
  each class's climbs, names, waiting and verdict, the claim's index.
- The ladder claims the classes with the best record first (a prior for
  the unexplored, at most a quarter of a claim from one), the cheapest
  target first within one, then the oldest of the rest; records classes
  like signatures and drops them at the same thresholds; discards their
  waiting releases. Claimed rows sorted in code: an UPDATE returns none.
- The rule set's droppedClasses: the edge keeps unnamed releases of a
  dropped class out of the pending store (counted as garbage), the
  explored sample excepted.
- hash_twins.ts: a probed file whose first 16 KiB and size match a named
  catalog release's names a pending release the probe didn't, and sets
  the inner name of a catalog release without one.
- pending:classes classes the rows from before and seeds the record from
  the finished ones; rules:classes publishes the dropped classes;
  releases:readable also marks the rows' candidates unreadable.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

commands/pending_classes.ts

@@ -0,0 +1,97 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import coordinatorConfig from '#config/coordinator'
+import { describeClass } from '#lib/release/index'
+
+/**
+ * The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and
+ * `target_bytes` (the first target's bytes, when its candidates carry them) set in windows of created_at; the
+ * finished rows among them (named, exhausted, naming) seed each class's record, which gets its verdict (dropped) as
+ * the ladder gives it; then pending_classes' `waiting` counted afresh from the rows. --report shows the record.
+ */
+export default class PendingClasses extends BaseCommand {
+  static commandName = 'pending:classes'
+  static description =
+    'Classes the pending releases from before the classes and recounts pending_classes; --report shows the record'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Hours of created_at per window' })
+  declare hours: number
+
+  @flags.boolean({ description: 'Show the classes with the best and worst records, write nothing' })
+  declare report: boolean
+
+  async run() {
+    if (this.report) return this.showReport()
+    const hours = this.hours ?? 6
+    const { rows: span } = await db.rawQuery(
+      `SELECT min(created_at) AS lo, max(created_at) AS hi FROM pending_releases WHERE klass = 0`
+    )
+    if (!span[0].lo) {
+      this.logger.info('No rows without a class.')
+    } else {
+      let from = new Date(span[0].lo)
+      const hi = new Date(span[0].hi)
+      let classed = 0
+      const started = Date.now()
+      while (from <= hi) {
+        const to = new Date(from.getTime() + hours * 3600_000)
+        // Rows of the window classed; the finished ones among them (named, exhausted, naming) seed the record.
+        const { rows } = await db.rawQuery(
+          `WITH classed AS (
+             UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
+               target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
+             WHERE klass = 0 AND created_at >= ? AND created_at < ?
+             RETURNING tool, group_id, klass, status),
+           seeded AS (
+             INSERT INTO pending_classes (tool, group_id, klass, probed, named)
+             SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed WHERE status IN (1, 3, 4) GROUP BY 1, 2, 3
+             ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named
+             RETURNING 1)
+           SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM seeded) AS seeded`,
+          [from.toISOString(), to.toISOString()]
+        )
+        classed += Number(rows[0].classed)
+        from = to
+        this.logger.info(
+          `Up to ${to.toISOString().slice(0, 16)}: ${classed} rows classed (${Math.round((Date.now() - started) / 1000)} s).`
+        )
+      }
+      // The record's verdicts on the seeded classes.
+      const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
+      await db.rawQuery(
+        `UPDATE pending_classes SET dropped = true WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`,
+        [pendingMinProbes, pendingMaxNamed]
+      )
+    }
+    // The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here).
+    await db.rawQuery(`
+      INSERT INTO pending_classes (tool, group_id, klass, waiting)
+      SELECT tool, group_id, klass, count(*) FROM pending_releases WHERE status = 0 AND finished GROUP BY 1, 2, 3
+      ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = excluded.waiting`)
+    await db.rawQuery(
+      `UPDATE pending_classes c SET waiting = 0 WHERE waiting > 0 AND NOT EXISTS (
+         SELECT 1 FROM pending_releases p WHERE p.status = 0 AND p.finished AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass)`
+    )
+    const { rows: totals } = await db.rawQuery(
+      `SELECT count(*) AS classes, coalesce(sum(waiting), 0) AS waiting, count(*) FILTER (WHERE dropped) AS dropped FROM pending_classes`
+    )
+    this.logger.info(
+      `Done: ${totals[0].classes} classes, ${totals[0].waiting} waiting, ${totals[0].dropped} dropped.`
+    )
+  }
+
+  private async showReport() {
+    const { rows } = await db.rawQuery(
+      `SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named, c.waiting, c.dropped FROM pending_classes c JOIN groups g ON g.id = c.group_id
+       WHERE c.probed >= 20 ORDER BY c.named::float / c.probed DESC, c.probed DESC`
+    )
+    const line = (r: any) =>
+      `  ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} probed ${String(r.probed).padStart(7)} named ${String(Math.round((100 * r.named) / r.probed)).padStart(3)}% waiting ${String(r.waiting).padStart(8)}${r.dropped ? ' dropped' : ''}`
+    this.logger.info(`${rows.length} classes with 20+ climbs; the best:`)
+    for (const r of rows.slice(0, 25)) this.logger.info(line(r))
+    this.logger.info('The worst:')
+    for (const r of rows.slice(-25)) this.logger.info(line(r))
+  }
+}

commands/releases_readable.ts

@@ -9,7 +9,8 @@ const CURSOR = 'readable'
 /**
  * The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
  * readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
- * false, so the readable filter hides them (an inner name from the prober still shows them). By the release name,
+ * false, so the readable filter hides them (an inner name from the prober still shows them), and their probe
+ * candidates marked unreadable, the prober's first tiers. By the release name,
  * which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
  * Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
  */
@@ -59,10 +60,16 @@ export default class ReleasesReadable extends BaseCommand {
       }
       changed += ids.length
       if (this.apply) {
-        if (ids.length > 0)
+        if (ids.length > 0) {
           await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
             ids,
           ])
+          // Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)).
+          await db.rawQuery(
+            `UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`,
+            [ids]
+          )
+        }
         await db.rawQuery(
           `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
           [CURSOR, String(to)]

commands/rules_classes.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import RuleSets from '#services/rule_sets'
+import { classKey, describeClass } from '#lib/release/index'
+
+/**
+ * Publishes a rule set whose dropped classes are the pending ladder's (pending_classes.dropped; LOG.md (130)): the
+ * edge keeps unnamed releases of those classes out of the pending store from then on, their explored sample
+ * excepted. Run again as the record grows; unchanged classes publish nothing.
+ */
+export default class RulesClasses extends BaseCommand {
+  static commandName = 'rules:classes'
+  static description = "Publishes a rule set with the pending ladder's dropped classes"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Show what would be published, write nothing' })
+  declare dryRun: boolean
+
+  async run() {
+    const { rows } = await db.rawQuery(
+      `SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named FROM pending_classes c JOIN groups g ON g.id = c.group_id WHERE c.dropped ORDER BY 1, 2, 3`
+    )
+    const next = new Set<string>(rows.map((r: any) => classKey(r.tool, r.grp, Number(r.klass))))
+    const { version, priorities: _p, ...current } = await RuleSets.current()
+    const before = new Set(current.droppedClasses ?? [])
+    const added = [...next].filter((c) => !before.has(c))
+    const removed = [...before].filter((c) => !next.has(c))
+    if (added.length === 0 && removed.length === 0) {
+      this.logger.info(
+        `Unchanged: rule set ${version} has the ${before.size} dropped classes already.`
+      )
+      return
+    }
+    for (const r of rows.slice(0, 40)) {
+      this.logger.info(
+        `  ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} ${r.named} of ${r.probed} named`
+      )
+    }
+    if (this.dryRun) {
+      this.logger.info(
+        `Would publish ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
+      )
+      return
+    }
+    const published = await RuleSets.publish(
+      { ...current, droppedClasses: [...next].sort() },
+      `${next.size} dropped classes from the pending ladder`
+    )
+    this.logger.info(
+      `Rule set ${published}: ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
+    )
+  }
+}