The par2 merge: releases whose files one par2 set describes become one
The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.
Also: corpus_sweep.py counts the marked ranges without psql's command tag.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
9ea2669b2a
commit
3d0dc6ca1f
13 files changed
+502
-9
No files matched your search
@@ -0,0 +1,60 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The par2 merge (REBUILD.md, phase 2; app/services/par2_merger.ts): what the
|
||||
* prober read that ties files to a par2 set, and the merges made from it.
|
||||
* - file_hashes: per probed article that starts a file, the MD5 of its first 16 KiB and the file's size.
|
||||
* - par2_files: per par2 recovery set, every file it describes (FileDesc: the same 16 KiB MD5, length, name).
|
||||
* - par2_sets: which releases hold a par2 file of a set.
|
||||
* - release_merges: releases merged into another, by set.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE file_hashes (
|
||||
candidate_id bigint PRIMARY KEY,
|
||||
release_id bigint NOT NULL,
|
||||
hash16k char(32) NOT NULL,
|
||||
size bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX file_hashes_hash_idx ON file_hashes (hash16k)`)
|
||||
this.schema.raw(`CREATE INDEX file_hashes_release_idx ON file_hashes (release_id)`)
|
||||
this.schema.raw(`CREATE INDEX file_hashes_created_idx ON file_hashes (created_at)`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE par2_files (
|
||||
set_id char(32) NOT NULL,
|
||||
file_id char(32) NOT NULL,
|
||||
hash16k char(32) NOT NULL,
|
||||
length bigint NOT NULL,
|
||||
name text NOT NULL,
|
||||
PRIMARY KEY (set_id, file_id)
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX par2_files_hash_idx ON par2_files (hash16k)`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE par2_sets (
|
||||
set_id char(32) NOT NULL,
|
||||
release_id bigint NOT NULL,
|
||||
files integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (set_id, release_id)
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX par2_sets_release_idx ON par2_sets (release_id)`)
|
||||
this.schema.raw(`CREATE INDEX par2_sets_created_idx ON par2_sets (created_at)`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE release_merges (
|
||||
id bigserial PRIMARY KEY,
|
||||
from_id bigint NOT NULL,
|
||||
from_key varchar(48) NOT NULL,
|
||||
into_id bigint NOT NULL,
|
||||
set_id char(32) NOT NULL,
|
||||
files integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX release_merges_into_idx ON release_merges (into_id)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS file_hashes, par2_files, par2_sets, release_merges`)
|
||||
}
|
||||
}
|
||||
+100
-3
@@ -32,6 +32,21 @@ export class AuthAccessTokenSchema extends BaseModel {
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class FileHashSchema extends BaseModel {
|
||||
static $columns = ['candidateId', 'createdAt', 'hash16K', 'releaseId', 'size'] as const
|
||||
$columns = FileHashSchema.$columns
|
||||
@column({ isPrimary: true })
|
||||
declare candidateId: bigint | number
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare hash16K: string
|
||||
@column()
|
||||
declare releaseId: bigint | number
|
||||
@column()
|
||||
declare size: bigint | number
|
||||
}
|
||||
|
||||
export class FragmentSchema extends BaseModel {
|
||||
static $columns = ['articleNumber', 'fileKey', 'fragment', 'id', 'rangeId', 'segments'] as const
|
||||
$columns = FragmentSchema.$columns
|
||||
@@ -50,7 +65,7 @@ export class FragmentSchema extends BaseModel {
|
||||
}
|
||||
|
||||
export class GroupSchema extends BaseModel {
|
||||
static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority'] as const
|
||||
static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority', 'tail'] as const
|
||||
$columns = GroupSchema.$columns
|
||||
@column()
|
||||
declare active: boolean
|
||||
@@ -68,6 +83,8 @@ export class GroupSchema extends BaseModel {
|
||||
declare plannedAt: DateTime | null
|
||||
@column()
|
||||
declare priority: number
|
||||
@column()
|
||||
declare tail: boolean
|
||||
}
|
||||
|
||||
export class NodeAccessTokenSchema extends BaseModel {
|
||||
@@ -108,6 +125,34 @@ export class NodeSchema extends BaseModel {
|
||||
declare name: string
|
||||
}
|
||||
|
||||
export class Par2FileSchema extends BaseModel {
|
||||
static $columns = ['fileId', 'hash16K', 'length', 'name', 'setId'] as const
|
||||
$columns = Par2FileSchema.$columns
|
||||
@column()
|
||||
declare fileId: string
|
||||
@column()
|
||||
declare hash16K: string
|
||||
@column()
|
||||
declare length: bigint | number
|
||||
@column()
|
||||
declare name: string
|
||||
@column({ isPrimary: true })
|
||||
declare setId: string
|
||||
}
|
||||
|
||||
export class Par2SetSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'files', 'releaseId', 'setId'] as const
|
||||
$columns = Par2SetSchema.$columns
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare files: number
|
||||
@column()
|
||||
declare releaseId: bigint | number
|
||||
@column({ isPrimary: true })
|
||||
declare setId: string
|
||||
}
|
||||
|
||||
export class ProbeCandidateSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const
|
||||
$columns = ProbeCandidateSchema.$columns
|
||||
@@ -128,10 +173,12 @@ export class ProbeCandidateSchema extends BaseModel {
|
||||
}
|
||||
|
||||
export class RangeSchema extends BaseModel {
|
||||
static $columns = ['attempts', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
|
||||
static $columns = ['attempts', 'capture', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'plainChunks', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
|
||||
$columns = RangeSchema.$columns
|
||||
@column()
|
||||
declare attempts: number
|
||||
@column()
|
||||
declare capture: boolean
|
||||
@column.dateTime()
|
||||
declare doneAt: DateTime | null
|
||||
@column()
|
||||
@@ -153,6 +200,8 @@ export class RangeSchema extends BaseModel {
|
||||
@column()
|
||||
declare nodeId: number | null
|
||||
@column()
|
||||
declare plainChunks: any
|
||||
@column()
|
||||
declare priority: number
|
||||
@column()
|
||||
declare ruleVersion: number | null
|
||||
@@ -187,12 +236,48 @@ export class ReleaseChunkSchema extends BaseModel {
|
||||
declare segments: number
|
||||
}
|
||||
|
||||
export class ReleaseInspectionSchema extends BaseModel {
|
||||
static $columns = ['candidateId', 'createdAt', 'kind', 'releaseId', 'result'] as const
|
||||
$columns = ReleaseInspectionSchema.$columns
|
||||
@column()
|
||||
declare candidateId: bigint | number
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare kind: string
|
||||
@column({ isPrimary: true })
|
||||
declare releaseId: bigint | number
|
||||
@column()
|
||||
declare result: any
|
||||
}
|
||||
|
||||
export class ReleaseMergeSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'files', 'fromId', 'fromKey', 'id', 'intoId', 'setId'] as const
|
||||
$columns = ReleaseMergeSchema.$columns
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare files: number
|
||||
@column()
|
||||
declare fromId: bigint | number
|
||||
@column()
|
||||
declare fromKey: string
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column()
|
||||
declare intoId: bigint | number
|
||||
@column()
|
||||
declare setId: string
|
||||
}
|
||||
|
||||
export class ReleaseSchema extends BaseModel {
|
||||
static $columns = ['bytes', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groups', 'hasPar2', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
|
||||
static $columns = ['bytes', 'category', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groupId', 'groups', 'hasPar2', 'id', 'identityKey', 'innerName', 'inspectedAt', 'lastArticle', 'lastPostedAt', 'lastSeenAt', 'name', 'passworded', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
|
||||
$columns = ReleaseSchema.$columns
|
||||
@column()
|
||||
declare bytes: bigint | number
|
||||
@column()
|
||||
declare category: number | null
|
||||
@column()
|
||||
declare chunks: number
|
||||
@column()
|
||||
declare complete: boolean | null
|
||||
@@ -205,17 +290,29 @@ export class ReleaseSchema extends BaseModel {
|
||||
@column.dateTime()
|
||||
declare firstSeenAt: DateTime
|
||||
@column()
|
||||
declare groupId: number | null
|
||||
@column()
|
||||
declare groups: any
|
||||
@column()
|
||||
declare hasPar2: boolean
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column()
|
||||
declare identityKey: string
|
||||
@column()
|
||||
declare innerName: string | null
|
||||
@column.dateTime()
|
||||
declare inspectedAt: DateTime | null
|
||||
@column()
|
||||
declare lastArticle: bigint | number | null
|
||||
@column.dateTime()
|
||||
declare lastPostedAt: DateTime | null
|
||||
@column.dateTime()
|
||||
declare lastSeenAt: DateTime
|
||||
@column()
|
||||
declare name: string
|
||||
@column()
|
||||
declare passworded: boolean | null
|
||||
@column.dateTime()
|
||||
declare postedAt: DateTime | null
|
||||
@column()
|
||||
|
||||
Reference in new issue
Block a user