The par2 merge: releases whose files one par2 set describes become one

The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.

Also: corpus_sweep.py counts the marked ranges without psql's command tag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

database/schema.ts

@@ -32,6 +32,21 @@ export class AuthAccessTokenSchema extends BaseModel {
   declare updatedAt: DateTime | null
 }
 
+export class FileHashSchema extends BaseModel {
+  static $columns = ['candidateId', 'createdAt', 'hash16K', 'releaseId', 'size'] as const
+  $columns = FileHashSchema.$columns
+  @column({ isPrimary: true })
+  declare candidateId: bigint | number
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare hash16K: string
+  @column()
+  declare releaseId: bigint | number
+  @column()
+  declare size: bigint | number
+}
+
 export class FragmentSchema extends BaseModel {
   static $columns = ['articleNumber', 'fileKey', 'fragment', 'id', 'rangeId', 'segments'] as const
   $columns = FragmentSchema.$columns
@@ -50,7 +65,7 @@ export class FragmentSchema extends BaseModel {
 }
 
 export class GroupSchema extends BaseModel {
-  static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority'] as const
+  static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority', 'tail'] as const
   $columns = GroupSchema.$columns
   @column()
   declare active: boolean
@@ -68,6 +83,8 @@ export class GroupSchema extends BaseModel {
   declare plannedAt: DateTime | null
   @column()
   declare priority: number
+  @column()
+  declare tail: boolean
 }
 
 export class NodeAccessTokenSchema extends BaseModel {
@@ -108,6 +125,34 @@ export class NodeSchema extends BaseModel {
   declare name: string
 }
 
+export class Par2FileSchema extends BaseModel {
+  static $columns = ['fileId', 'hash16K', 'length', 'name', 'setId'] as const
+  $columns = Par2FileSchema.$columns
+  @column()
+  declare fileId: string
+  @column()
+  declare hash16K: string
+  @column()
+  declare length: bigint | number
+  @column()
+  declare name: string
+  @column({ isPrimary: true })
+  declare setId: string
+}
+
+export class Par2SetSchema extends BaseModel {
+  static $columns = ['createdAt', 'files', 'releaseId', 'setId'] as const
+  $columns = Par2SetSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare files: number
+  @column()
+  declare releaseId: bigint | number
+  @column({ isPrimary: true })
+  declare setId: string
+}
+
 export class ProbeCandidateSchema extends BaseModel {
   static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const
   $columns = ProbeCandidateSchema.$columns
@@ -128,10 +173,12 @@ export class ProbeCandidateSchema extends BaseModel {
 }
 
 export class RangeSchema extends BaseModel {
-  static $columns = ['attempts', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
+  static $columns = ['attempts', 'capture', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'plainChunks', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
   $columns = RangeSchema.$columns
   @column()
   declare attempts: number
+  @column()
+  declare capture: boolean
   @column.dateTime()
   declare doneAt: DateTime | null
   @column()
@@ -153,6 +200,8 @@ export class RangeSchema extends BaseModel {
   @column()
   declare nodeId: number | null
   @column()
+  declare plainChunks: any
+  @column()
   declare priority: number
   @column()
   declare ruleVersion: number | null
@@ -187,12 +236,48 @@ export class ReleaseChunkSchema extends BaseModel {
   declare segments: number
 }
 
+export class ReleaseInspectionSchema extends BaseModel {
+  static $columns = ['candidateId', 'createdAt', 'kind', 'releaseId', 'result'] as const
+  $columns = ReleaseInspectionSchema.$columns
+  @column()
+  declare candidateId: bigint | number
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare kind: string
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+  @column()
+  declare result: any
+}
+
+export class ReleaseMergeSchema extends BaseModel {
+  static $columns = ['createdAt', 'files', 'fromId', 'fromKey', 'id', 'intoId', 'setId'] as const
+  $columns = ReleaseMergeSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare files: number
+  @column()
+  declare fromId: bigint | number
+  @column()
+  declare fromKey: string
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column()
+  declare intoId: bigint | number
+  @column()
+  declare setId: string
+}
+
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['bytes', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groups', 'hasPar2', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
+  static $columns = ['bytes', 'category', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groupId', 'groups', 'hasPar2', 'id', 'identityKey', 'innerName', 'inspectedAt', 'lastArticle', 'lastPostedAt', 'lastSeenAt', 'name', 'passworded', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare bytes: bigint | number
   @column()
+  declare category: number | null
+  @column()
   declare chunks: number
   @column()
   declare complete: boolean | null
@@ -205,17 +290,29 @@ export class ReleaseSchema extends BaseModel {
   @column.dateTime()
   declare firstSeenAt: DateTime
   @column()
+  declare groupId: number | null
+  @column()
   declare groups: any
   @column()
   declare hasPar2: boolean
   @column({ isPrimary: true })
   declare id: bigint | number
   @column()
+  declare identityKey: string
+  @column()
+  declare innerName: string | null
+  @column.dateTime()
+  declare inspectedAt: DateTime | null
+  @column()
   declare lastArticle: bigint | number | null
   @column.dateTime()
+  declare lastPostedAt: DateTime | null
+  @column.dateTime()
   declare lastSeenAt: DateTime
   @column()
   declare name: string
+  @column()
+  declare passworded: boolean | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()