Decide completeness by par2 set for releases without a counter

59% of settled releases had complete = NULL: without an [i/N] counter
the settler left it unknown, and the "phase 2 reads par2 sets for those"
step was never built. A release is now complete when every file its par2
sets describe is there by name or by its first-16-KiB hash, or, for an
upload renamed after its par2 was made, when it has as many non-par2
files and bytes. Decided when a par2 index is probed, after a par2 merge,
and before the pruner drops a set's file list; the batch store, settler
and merger keep a decided value instead of resetting it to NULL.
releases:par2-complete decides the releases probed before this.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -237,8 +237,9 @@ class BatchStore {
              last_article = greatest(r.last_article, a.last_article),
              readable = r.readable OR a.readable,
              has_par2 = r.has_par2 OR a.has_par2,
-             -- As it stands (the counter's files all found); settling makes it final.
-             complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
+             -- As it stands (the counter's files all found); settling makes it final. Without a counter, what its par2
+             -- set decided stays (app/services/par2_completeness.ts).
+             complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN r.complete ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
              last_seen_at = now()
            FROM a WHERE r.id = a.id`,
           [JSON.stringify(updates)]

app/services/nzb_service.ts

@@ -21,8 +21,8 @@ class NzbService {
     return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
   }
 
-  async files(releaseId: number): Promise<ChunkFile[]> {
-    const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
+  async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
+    const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
     const files = new Map<string, ChunkFile>()
     for (const chunk of chunks) {
       for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)

app/services/par2_completeness.ts

@@ -0,0 +1,53 @@
+import NzbService from '#services/nzb_service'
+
+/** A file a par2 set describes (par2_files). */
+export interface SetFile {
+  name: string
+  length: number
+  hash16k: string
+}
+
+const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
+
+/**
+ * Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
+ * whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
+ * or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
+ * described name among its files) is complete when it has as many files besides par2 as the set describes, and as
+ * many bytes (its articles' bytes, a little over the files' with yEnc).
+ */
+export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
+  const names = new Set(files.map((f) => base(f.name)))
+  const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
+  const byName = set.filter((f) => names.has(base(f.name))).length
+  if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
+  if (byName > 0) return false
+  const data = files.filter((f) => !/\.par2$/i.test(f.name))
+  return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
+}
+
+/**
+ * Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
+ * list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
+ * a transaction that moved chunks (a merge) reads them as moved.
+ */
+export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
+  if (releaseIds.length === 0) return 0
+  const { rows: releases } = await client.rawQuery(
+    `SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
+    [releaseIds.map(String)]
+  )
+  let decided = 0
+  for (const { id } of releases) {
+    const { rows: set } = await client.rawQuery(
+      `SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
+      [id]
+    )
+    if (set.length === 0) continue
+    const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
+    const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
+    await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
+    decided++
+  }
+  return decided
+}

app/services/par2_merger.ts

@@ -5,6 +5,7 @@ import { looksRandom } from '#lib/release/random'
 import { SidecarFolder } from '#services/sidecar_folder'
 import { Par2Pruner } from '#services/par2_pruner'
 import { pruneSevenZipTails } from '#services/seven_zip_tails'
+import { decidePar2Completeness } from '#services/par2_completeness'
 
 /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
 const MAX_SPREAD_MS = 48 * 3600 * 1000
@@ -371,12 +372,13 @@ export class Par2Merger {
          WHERE t.id = ?`,
         [name ?? '', others, target.id]
       )
-      // Completeness as the settler decides it, once settled.
+      // Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
       await trx.rawQuery(
-        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
+        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
         [target.id]
       )
       await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
+      await decidePar2Completeness(trx, [target.id])
       return others.length
     }
   }

app/services/par2_pruner.ts

@@ -1,4 +1,5 @@
 import db from '@adonisjs/lucid/services/db'
+import { decidePar2Completeness } from '#services/par2_completeness'
 
 /** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
 const SETTLED_MS = 2 * 86400_000
@@ -86,6 +87,11 @@ export class Par2Pruner {
       for (const reason of ['complete', 'judged', 'final'] as const) {
         const ids = done(reason)
         round[reason] += ids.length
+        // The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
+        if (reason !== 'judged' && ids.length > 0) {
+          const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
+          await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
+        }
         round.rows += await this.drop(ids)
       }
       // Past the last set: the next batch starts the next pass.

app/services/prober.ts

@@ -8,6 +8,7 @@ import { YencError } from '#lib/yenc/index'
 import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
 import { NntpShare } from '#services/nntp_shares'
 import { DiskSpace } from '#services/disk_space'
+import { decidePar2Completeness } from '#services/par2_completeness'
 
 interface Candidate {
   id: number
@@ -306,6 +307,8 @@ export class Prober {
         [jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
       )
       await this.storeForMerge(trx, done)
+      // A par2 index read: the release's completeness by its set, when it has no counter.
+      await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
       // Each touched release from all its inspections (earlier rounds may have stored others).
       const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
       const { rows } = await trx.rawQuery(

app/services/settler.ts

@@ -15,7 +15,8 @@ const BATCH = 50_000
  * Settles releases (REBUILD.md, part 8): once nothing has been added for
  * QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
  * [i/N] counter announced was found, unknown without a counter (phase 2
- * reads par2 sets for those). Phase 2 picks its work from settled releases.
+ * decides those by their par2 sets: app/services/par2_completeness.ts).
+ * Phase 2 picks its work from settled releases.
  */
 class Settler {
   /** Up to `batches` batches of BATCH (all there are without). */
@@ -25,7 +26,7 @@ class Settler {
       // Their probe candidates become claimable (Prober.claim), the unreadable ones first.
       const { rows } = await db.rawQuery(
         `WITH settled AS (
-           UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+           UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END
            WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
              AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
            RETURNING id, readable, files, bytes, poster, sidecar_only),