Rebuild: completeness as indexed, file totals from any counter, pausable tails, progress by articles

- releases.complete is computed with every batch (files >= the
  counter's total) instead of only when a release settles after 6 quiet
  hours: the 'complete' filter matched nothing in a fresh catalog.
- A release's file total comes from any counter form (counterTotal:
  '(01/10)', 'File 27 of 132', …), not only [i/M]; the post rule's key
  keeps the [i/M] total, so keys are unchanged. Both implementations.
- groups.tail: a group's live tail can be paused (its history and owner
  priority stay), e.g. to let other groups' history go first.
- Stats: history by articles, and the current pass (history ranges
  before those deferred 2000 priorities behind) as its own figure; the
  header showed 0.1% of ranges counting 145k deferred ones.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -111,8 +111,9 @@ class BatchStore {
     for (let i = 0; i < sorted.length; i += 1000) {
       const chunk = sorted.slice(i, i + 1000)
       const { rows } = await trx.rawQuery(
-        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2)
-           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2
+        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
+           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2,
+                  CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
            FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
            ON CONFLICT (release_key) DO UPDATE SET
              files = releases.files + excluded.files,
@@ -125,6 +126,9 @@ class BatchStore {
              last_article = greatest(releases.last_article, excluded.last_article),
              readable = releases.readable OR excluded.readable,
              has_par2 = releases.has_par2 OR excluded.has_par2,
+             -- As it stands (the counter's files all found); settling makes it final.
+             complete = CASE WHEN coalesce(releases.file_total, excluded.file_total) IS NULL THEN NULL
+                             ELSE releases.files + excluded.files >= coalesce(releases.file_total, excluded.file_total) END,
              last_seen_at = now()
            RETURNING id, release_key, (xmax = 0) AS inserted`,
         [

app/services/planner.ts

@@ -76,7 +76,7 @@ class Planner {
    * group took minutes for 1000 groups).
    */
   async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
-    const groups = await db.from('groups').where('active', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
     if (groups.length === 0) return { groups: 0, ranges: 0 }
     const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
     const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))

app/services/search_service.ts

@@ -222,12 +222,18 @@ class SearchService {
     const sampled = Number(s[0].sampled)
     const releases = sample ? Math.max(estimate, 0) : sampled
     const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    // History by articles; the pass: the history ranges before those deferred behind them (priority
+    // 2000 apart; e.g. groups added later that go first).
     const { rows: ranges } = await db.rawQuery(
-      `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total,
-              count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history,
-              max(done_at) AS last_done_at,
-              max(done_at) FILTER (WHERE priority < 0) AS tail_done_at
-       FROM ranges`
+      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
+            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
+       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
+              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
+              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
+              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
+              (SELECT max(done_at) FROM ranges) AS last_done_at,
+              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
+       FROM h`
     )
     const g = ranges[0]
     return {
@@ -236,7 +242,10 @@ class SearchService {
       readableReleases: scale(s[0].readable),
       files: scale(s[0].files),
       size: scale(s[0].bytes),
-      ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) },
+      ranges: { done: Number(g.done), total: Number(g.total) },
+      /** History and the current pass, in articles. */
+      history: { done: Number(g.history_done), total: Number(g.history) },
+      pass: { done: Number(g.pass_done), total: Number(g.pass) },
       lastIndexedAt: g.last_done_at,
       tailIndexedAt: g.tail_done_at,
     }