Daan MeijerandClaude Opus 5.5
3d0dc6ca1f
The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.
Also: corpus_sweep.py counts the marked ranges without psql's command tag.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.4 KiB
TypeScript
61 lines
2.4 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* The par2 merge (REBUILD.md, phase 2; app/services/par2_merger.ts): what the
|
|
* prober read that ties files to a par2 set, and the merges made from it.
|
|
* - file_hashes: per probed article that starts a file, the MD5 of its first 16 KiB and the file's size.
|
|
* - par2_files: per par2 recovery set, every file it describes (FileDesc: the same 16 KiB MD5, length, name).
|
|
* - par2_sets: which releases hold a par2 file of a set.
|
|
* - release_merges: releases merged into another, by set.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
async up() {
|
|
this.schema.raw(`
|
|
CREATE TABLE file_hashes (
|
|
candidate_id bigint PRIMARY KEY,
|
|
release_id bigint NOT NULL,
|
|
hash16k char(32) NOT NULL,
|
|
size bigint NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now()
|
|
)`)
|
|
this.schema.raw(`CREATE INDEX file_hashes_hash_idx ON file_hashes (hash16k)`)
|
|
this.schema.raw(`CREATE INDEX file_hashes_release_idx ON file_hashes (release_id)`)
|
|
this.schema.raw(`CREATE INDEX file_hashes_created_idx ON file_hashes (created_at)`)
|
|
this.schema.raw(`
|
|
CREATE TABLE par2_files (
|
|
set_id char(32) NOT NULL,
|
|
file_id char(32) NOT NULL,
|
|
hash16k char(32) NOT NULL,
|
|
length bigint NOT NULL,
|
|
name text NOT NULL,
|
|
PRIMARY KEY (set_id, file_id)
|
|
)`)
|
|
this.schema.raw(`CREATE INDEX par2_files_hash_idx ON par2_files (hash16k)`)
|
|
this.schema.raw(`
|
|
CREATE TABLE par2_sets (
|
|
set_id char(32) NOT NULL,
|
|
release_id bigint NOT NULL,
|
|
files integer NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (set_id, release_id)
|
|
)`)
|
|
this.schema.raw(`CREATE INDEX par2_sets_release_idx ON par2_sets (release_id)`)
|
|
this.schema.raw(`CREATE INDEX par2_sets_created_idx ON par2_sets (created_at)`)
|
|
this.schema.raw(`
|
|
CREATE TABLE release_merges (
|
|
id bigserial PRIMARY KEY,
|
|
from_id bigint NOT NULL,
|
|
from_key varchar(48) NOT NULL,
|
|
into_id bigint NOT NULL,
|
|
set_id char(32) NOT NULL,
|
|
files integer NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now()
|
|
)`)
|
|
this.schema.raw(`CREATE INDEX release_merges_into_idx ON release_merges (into_id)`)
|
|
}
|
|
|
|
async down() {
|
|
this.schema.raw(`DROP TABLE IF EXISTS file_hashes, par2_files, par2_sets, release_merges`)
|
|
}
|
|
}
|