Daan MeijerandClaude Opus 5.5
6d1f2cc2a1
131 uploads of 1 GiB test files ("kantodoNNN.bin", a.b.test) share their
first 16 KiB and size, so all matched one set's single file and became one
release of 1,284 files. A par2 file that matches more than one release in
the anchor's group and time window is left out; a plain repost of the same
file stays apart too.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
171 lines
8.7 KiB
TypeScript
171 lines
8.7 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import { par2ReleaseName } from '#lib/release/index'
|
|
import { looksRandom } from '#lib/release/random'
|
|
|
|
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
|
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
|
/** Between rounds. */
|
|
const ROUND_MS = 60_000
|
|
/** A round looks back this far past the previous one's start (rows committed while it ran). */
|
|
const OVERLAP_MS = 60_000
|
|
|
|
export interface MergeRound {
|
|
sets: number
|
|
merges: number
|
|
releases: number
|
|
}
|
|
|
|
interface Member {
|
|
id: number
|
|
release_key: string
|
|
group_id: number
|
|
files: number
|
|
posted_at: Date | null
|
|
readable: boolean
|
|
}
|
|
|
|
/**
|
|
* The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
|
|
* describes are one upload. Obfuscated uploads post each file under a name of
|
|
* its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
|
|
* one-file releases), so the header pass can't group them; the prober reads
|
|
* each such file's first article (a `lone` candidate) and stores the MD5 of
|
|
* its first 16 KiB, and a par2 body's file list holds the same hash per file,
|
|
* whatever the file was renamed to. Releases in one group, posted within
|
|
* MAX_SPREAD_MS of a release that holds the set's par2, become the one with
|
|
* the most files: its chunks, probe candidates, inspections and hashes, the
|
|
* sums of its counts, and when none of them has a readable name the name the
|
|
* par2's file names share. Each merge is kept in release_merges.
|
|
*/
|
|
export class Par2Merger {
|
|
private stopped = false
|
|
private since = new Date(0)
|
|
|
|
stop() {
|
|
this.stopped = true
|
|
}
|
|
|
|
async run() {
|
|
while (!this.stopped) {
|
|
const started = Date.now()
|
|
try {
|
|
const round = await this.round()
|
|
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
|
} catch (error: any) {
|
|
logger.error(`Par2 merge round failed: ${error.message}`)
|
|
}
|
|
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
|
}
|
|
}
|
|
|
|
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
|
|
async round(): Promise<MergeRound> {
|
|
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
|
|
const since = new Date(this.since.getTime() - OVERLAP_MS)
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT set_id FROM par2_sets WHERE created_at >= ?
|
|
UNION
|
|
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
|
|
[since, since]
|
|
)
|
|
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0 }
|
|
for (const { set_id: setId } of rows) {
|
|
if (this.stopped) return round
|
|
const merged = await this.mergeSet(setId.trim())
|
|
if (merged > 0) {
|
|
round.merges++
|
|
round.releases += merged + 1
|
|
}
|
|
}
|
|
this.since = now[0].at
|
|
return round
|
|
}
|
|
|
|
/** Merges the releases of one set; how many went into another. */
|
|
async mergeSet(setId: string): Promise<number> {
|
|
return db.transaction(async (trx) => {
|
|
const { rows: holders } = await trx.rawQuery(
|
|
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
|
WHERE s.set_id = ? ORDER BY r.id`,
|
|
[setId]
|
|
)
|
|
const anchor: Member | undefined = holders[0]
|
|
if (!anchor) return 0
|
|
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
|
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
|
const { rows: matches } = await trx.rawQuery(
|
|
`SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
|
|
FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
|
|
WHERE f.set_id = ?`,
|
|
[setId]
|
|
)
|
|
// A file of the set that matches more than one release nearby is ambiguous: files that share their first
|
|
// 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
|
|
// one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
|
|
const byFile = new Map<string, Map<string, Member>>()
|
|
for (const m of matches.filter(near)) {
|
|
const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
|
|
releases.set(String(m.id), m)
|
|
byFile.set(m.file_id, releases)
|
|
}
|
|
const found = new Map<string, Member>()
|
|
for (const r of holders.filter(near)) found.set(String(r.id), r)
|
|
for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
|
|
if (found.size < 2) return 0
|
|
// Locked in id order (the batch store and other rounds take the same rows).
|
|
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
|
const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
|
|
if (members.length < 2) return 0
|
|
const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
|
|
const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
|
|
const name = members.every((m) => !m.readable) ? await this.setName(trx, setId) : undefined
|
|
|
|
await trx.rawQuery(
|
|
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files)
|
|
SELECT id, release_key, ?, ?, files FROM releases WHERE id = ANY(?)`,
|
|
[target.id, setId, others]
|
|
)
|
|
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
|
|
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
|
}
|
|
// A set held by the target already isn't held twice.
|
|
await trx.rawQuery(`DELETE FROM par2_sets WHERE release_id = ANY(?) AND set_id IN (SELECT set_id FROM par2_sets WHERE release_id = ?)`, [others, target.id])
|
|
await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
|
await trx.rawQuery(
|
|
`UPDATE releases t SET
|
|
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
|
|
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
|
|
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
|
|
first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
|
|
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
|
|
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
|
|
name = coalesce(nullif(?, ''), t.name)
|
|
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
|
|
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
|
|
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
|
|
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
|
|
max(file_total) AS file_total, max(inner_name) AS inner_name
|
|
FROM releases WHERE id = ANY(?)) s
|
|
WHERE t.id = ?`,
|
|
[name ?? '', others, target.id]
|
|
)
|
|
// Completeness as the settler decides it, once settled.
|
|
await trx.rawQuery(
|
|
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
|
[target.id]
|
|
)
|
|
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
|
return others.length
|
|
})
|
|
}
|
|
|
|
/** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
|
|
private async setName(trx: any, setId: string): Promise<string | undefined> {
|
|
const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
|
const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
|
|
// A par2 made after the renaming names the random names again.
|
|
return name && !looksRandom(name) ? name.slice(0, 255) : undefined
|
|
}
|
|
}
|