Rebuild: completeness as indexed, file totals from any counter, pausable tails, progress by articles

- releases.complete is computed with every batch (files >= the
  counter's total) instead of only when a release settles after 6 quiet
  hours: the 'complete' filter matched nothing in a fresh catalog.
- A release's file total comes from any counter form (counterTotal:
  '(01/10)', 'File 27 of 132', …), not only [i/M]; the post rule's key
  keeps the [i/M] total, so keys are unchanged. Both implementations.
- groups.tail: a group's live tail can be paused (its history and owner
  priority stay), e.g. to let other groups' history go first.
- Stats: history by articles, and the current pass (history ranges
  before those deferred 2000 priorities behind) as its own figure; the
  header showed 0.1% of ranges counting 145k deferred ones.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -111,8 +111,9 @@ class BatchStore {
     for (let i = 0; i < sorted.length; i += 1000) {
       const chunk = sorted.slice(i, i + 1000)
       const { rows } = await trx.rawQuery(
-        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2)
-           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2
+        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
+           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2,
+                  CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
            FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
            ON CONFLICT (release_key) DO UPDATE SET
              files = releases.files + excluded.files,
@@ -125,6 +126,9 @@ class BatchStore {
              last_article = greatest(releases.last_article, excluded.last_article),
              readable = releases.readable OR excluded.readable,
              has_par2 = releases.has_par2 OR excluded.has_par2,
+             -- As it stands (the counter's files all found); settling makes it final.
+             complete = CASE WHEN coalesce(releases.file_total, excluded.file_total) IS NULL THEN NULL
+                             ELSE releases.files + excluded.files >= coalesce(releases.file_total, excluded.file_total) END,
              last_seen_at = now()
            RETURNING id, release_key, (xmax = 0) AS inserted`,
         [

app/services/planner.ts

@@ -76,7 +76,7 @@ class Planner {
    * group took minutes for 1000 groups).
    */
   async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
-    const groups = await db.from('groups').where('active', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
     if (groups.length === 0) return { groups: 0, ranges: 0 }
     const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
     const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))

app/services/search_service.ts

@@ -222,12 +222,18 @@ class SearchService {
     const sampled = Number(s[0].sampled)
     const releases = sample ? Math.max(estimate, 0) : sampled
     const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    // History by articles; the pass: the history ranges before those deferred behind them (priority
+    // 2000 apart; e.g. groups added later that go first).
     const { rows: ranges } = await db.rawQuery(
-      `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total,
-              count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history,
-              max(done_at) AS last_done_at,
-              max(done_at) FILTER (WHERE priority < 0) AS tail_done_at
-       FROM ranges`
+      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
+            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
+       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
+              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
+              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
+              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
+              (SELECT max(done_at) FROM ranges) AS last_done_at,
+              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
+       FROM h`
     )
     const g = ranges[0]
     return {
@@ -236,7 +242,10 @@ class SearchService {
       readableReleases: scale(s[0].readable),
       files: scale(s[0].files),
       size: scale(s[0].bytes),
-      ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) },
+      ranges: { done: Number(g.done), total: Number(g.total) },
+      /** History and the current pass, in articles. */
+      history: { done: Number(g.history_done), total: Number(g.history) },
+      pass: { done: Number(g.pass_done), total: Number(g.pass) },
       lastIndexedAt: g.last_done_at,
       tailIndexedAt: g.tail_done_at,
     }

database/migrations/1770000001300_groups_tail.ts

@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A group's live tail can be paused (its history and its place in the owner
+ * rule stay): e.g. to let other groups' history go first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE groups ADD COLUMN IF NOT EXISTS tail boolean NOT NULL DEFAULT true`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE groups DROP COLUMN IF EXISTS tail`)
+  }
+}

lib/release/identity.ts

@@ -185,7 +185,9 @@ export function isPosterTag(remainder: string, stem: string): boolean {
  */
 export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
   const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
-  const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
+  // The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's.
+  const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined
+  const fileTotal = counterTotal(input.subject)
   const stem = fileStem(input.name)
 
   if (input.weak) {
@@ -223,8 +225,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
   }
   // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
   // [i/M] counter it groups the files of one post whose names are random.
-  if (!tag && fileTotal && isPostToken(remainder)) {
-    return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
+  if (!tag && postTotal && isPostToken(remainder)) {
+    return { key: hash('post', input.poster, remainder, postTotal), name: remainder, fileTotal, source: 'post' }
   }
   // An image without words around it: its set by its name (keyed as a stem, so "x.jpg" joins "x.part01.rar").
   if (image) return { key: hash('stem', input.poster, prefix || stem), name: prefix || stem, fileTotal, source: 'stem' }

public/app.js

@@ -164,13 +164,17 @@ async function loadStats() {
     const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0
     // Estimated from Postgres statistics and a sample.
     const about = stats.approximate ? '≈ ' : ''
-    const history = stats.ranges.history ? (100 * stats.ranges.historyDone) / stats.ranges.history : 0
+    const percent = ({ done, total }) => {
+      const p = total ? (100 * done) / total : 0
+      return `${p < 10 ? p.toFixed(1) : Math.round(p)}%`
+    }
     const items = [
       ['Releases', about + number.format(stats.releases)],
       ['Readable', `${about}${readable}%`],
       ['Files', about + number.format(stats.files)],
       ['Size', about + formatSize(stats.size)],
-      ['History', `${history < 10 ? history.toFixed(1) : Math.round(history)}%`],
+      ['History', percent(stats.history)],
+      ...(stats.pass.total < stats.history.total ? [['Pass', percent(stats.pass)]] : []),
       ['Live', formatAge(stats.tailIndexedAt) || '–'],
     ]
     $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd }))))

tests/functional/releases_api.spec.ts

@@ -57,9 +57,13 @@ test.group('Releases API', (group) => {
     const detail = (await client.get(`/api/v1/releases/${tv.releases[0].id}`)).body() as { files: { name: string; parts: number; size: number }[] }
     assert.deepEqual(detail.files, [{ ...detail.files[0], name: 'Some.Show.S01E01.1080p.WEB.mkv', parts: 2, size: 1400001 }])
     ;(await client.get('/api/v1/releases/999999')).assertStatus(404)
-    const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number } }
+    // Complete as soon as the counter's files are all found (settling makes it final).
+    const complete = (await client.get('/api/v1/releases?complete=1')).body() as { releases: unknown[] }
+    assert.lengthOf(complete.releases, 2)
+    const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number }; history: { total: number } }
     assert.equal(stats.releases, 2)
     assert.equal(stats.ranges.total, 1)
+    assert.equal(stats.history.total, 1000)
 
     // Newznab: caps, a TV search with season and episode, the NZB; the key when one is set.
     const caps = await client.get('/api?t=caps')

tools/edge-bench-rs/src/names.rs

@@ -510,7 +510,9 @@ pub fn is_poster_tag(remainder: &str, stem: &str) -> bool {
 
 /// releaseIdentity for a named (not weak) file: never null.
 pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
-    let file_total = file_counter_total(subject);
+    // The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal).
+    let post_total = file_counter_total(subject);
+    let file_total = counter_total(subject).and_then(|t| t.parse::<i64>().ok());
     let stem = file_stem(name);
     let remainder = subject_remainder(subject);
     // A document or comic is a release of its own.
@@ -527,7 +529,7 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
         let bound = counter_total(subject).unwrap_or_else(|| prefix.clone());
         return Release { key: hash(&["subject", poster, &remainder, &bound]), name: remainder, source: "subject", file_total };
     }
-    if let Some(total) = file_total.filter(|&n| n != 0 && !tag) {
+    if let Some(total) = post_total.filter(|&n| n != 0 && !tag) {
         if is_post_token(&remainder) {
             return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
         }