96 lines
4.8 KiB
TypeScript
96 lines
4.8 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
|
|
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
|
const SETTLED_MS = 2 * 86400_000
|
|
/** Sets per statement. */
|
|
const BATCH = 20_000
|
|
/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
|
|
const CURSOR = 'par2-prune'
|
|
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
|
|
const JUNK = ['dead', 'fake', 'repeated']
|
|
|
|
export interface PruneRound {
|
|
complete: number
|
|
judged: number
|
|
rows: number
|
|
}
|
|
|
|
/**
|
|
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
|
|
* 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
|
|
* done when
|
|
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
|
|
* set (all of it merged into one);
|
|
* - judged: every release holding it has a junk verdict (dead, fake, repeated).
|
|
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
|
|
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
|
*/
|
|
export class Par2Pruner {
|
|
/** Sets per batch (BATCH; smaller in the tests). */
|
|
constructor(private batch = BATCH) {}
|
|
|
|
/**
|
|
* Batches until done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from it): run
|
|
* from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md 2026-10-17 (72)).
|
|
*/
|
|
async round(budgetMs = Infinity): Promise<PruneRound> {
|
|
const deadline = Date.now() + budgetMs
|
|
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
|
|
// (created_at to the microsecond, set_id): one prober flush gives thousands of sets the same created_at, and a
|
|
// timestamp read back to the millisecond took the same batch again every round (LOG.md 2026-10-17 (73)).
|
|
const { rows: cursors } = await db.rawQuery(`SELECT at::text AS at, key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
|
let since: string = cursors[0]?.at ?? '1970-01-01 00:00:00+00'
|
|
let sinceKey: string = cursors[0]?.key ?? ''
|
|
const until = new Date(Date.now() - SETTLED_MS)
|
|
for (;;) {
|
|
const { rows: sets } = await db.rawQuery(
|
|
`SELECT set_id, max(created_at)::text AS at FROM par2_sets GROUP BY set_id
|
|
HAVING (max(created_at), set_id::text) > (?::timestamptz, ?::text) AND max(created_at) <= ?
|
|
ORDER BY max(created_at), set_id LIMIT ?`,
|
|
[since, sinceKey, until, this.batch]
|
|
)
|
|
if (sets.length === 0) break
|
|
const ids = sets.map((s: { set_id: string }) => s.set_id)
|
|
const { rows: done } = await db.rawQuery(
|
|
`SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
|
|
WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
|
|
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
|
WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
|
|
[ids]
|
|
)
|
|
round.complete += done.length
|
|
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
|
|
since = sets.at(-1).at
|
|
sinceKey = sets.at(-1).set_id
|
|
await db.rawQuery(
|
|
`INSERT INTO worker_cursors (name, at, key) VALUES (?, ?::timestamptz, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
|
[CURSOR, since, sinceKey]
|
|
)
|
|
if (sets.length < this.batch || Date.now() > deadline) break
|
|
}
|
|
if (Date.now() > deadline) return round
|
|
// Judged: no cursor (verdicts come and go); only sets with rows left.
|
|
const { rows: judged } = await db.rawQuery(
|
|
`SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
|
|
WHERE v.rule = ANY(?)
|
|
AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
|
|
WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
|
|
[JUNK, JUNK]
|
|
)
|
|
round.judged = judged.length
|
|
const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
|
|
for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
|
|
return round
|
|
}
|
|
|
|
private async drop(setIds: string[]): Promise<number> {
|
|
if (setIds.length === 0) return 0
|
|
const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
|
|
return result.rowCount ?? 0
|
|
}
|
|
}
|