Daan MeijerandClaude Fable 5.1
2580f20a43
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
499 lines
27 KiB
TypeScript
499 lines
27 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
import { par2ReleaseName } from '#lib/release/index'
|
||
import { looksRandom } from '#lib/release/random'
|
||
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
|
||
import { randomPoster } from '#lib/release/identity'
|
||
import NzbService from '#services/nzb_service'
|
||
import { Par2Pruner } from '#services/par2_pruner'
|
||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||
|
||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||
/**
|
||
* A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every
|
||
* "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)).
|
||
*/
|
||
const MAX_HOLDERS = 4
|
||
/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
|
||
const MAX_FILES_PER_SET_FILE = 3
|
||
const SLACK_FILES = 10
|
||
/**
|
||
* A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
|
||
* it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
|
||
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
|
||
*/
|
||
const COMMON_FILE = 20
|
||
/**
|
||
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
|
||
* NAME_CANDIDATES of them (a poster's uploads of a day).
|
||
*/
|
||
const NAME_SPREAD_MS = 12 * 3600 * 1000
|
||
const NAME_CANDIDATES = 200
|
||
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
|
||
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
|
||
/** Between rounds. */
|
||
const ROUND_MS = 60_000
|
||
/** A window looks back this far past the previous one's end (rows committed after it was read). */
|
||
const OVERLAP_MS = 60_000
|
||
/**
|
||
* A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
|
||
* two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
|
||
*/
|
||
const WINDOW_MS = 60 * 60_000
|
||
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
|
||
const FOLD_EVERY_MS = 5 * 60_000
|
||
/** Between par2_files prunes (complete and judged sets' file lists). */
|
||
const PRUNE_EVERY_MS = 60 * 60_000
|
||
/** A prune's share of the merger's time: the rest goes on in the next one. */
|
||
const PRUNE_BUDGET_MS = 2 * 60_000
|
||
/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
|
||
const CURSOR = 'merger'
|
||
|
||
export interface MergeRound {
|
||
sets: number
|
||
merges: number
|
||
releases: number
|
||
/** The name pass: merges, releases in them. */
|
||
nameMerges: number
|
||
nameReleases: number
|
||
/** The inner-name pass: stems checked, merges, releases in them. */
|
||
stems: number
|
||
stemMerges: number
|
||
stemReleases: number
|
||
/** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
|
||
folded?: { looked: number; sidecars: number; folded: number }
|
||
}
|
||
|
||
/**
|
||
* A lone inspection's inner stem: its yEnc name up to the first dot, when 16–64 letters and digits
|
||
* ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"). The same expression as release_inspections_inner_stem_idx.
|
||
*/
|
||
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
|
||
|
||
/**
|
||
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
|
||
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
|
||
*/
|
||
export function fileKey(name: string): string {
|
||
let n = name.replace(/^.*[\\/]/, '')
|
||
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
|
||
const read = Buffer.from(n, 'latin1').toString('utf8')
|
||
if (!read.includes('\ufffd')) n = read
|
||
}
|
||
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
||
}
|
||
|
||
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
|
||
const HOLDERS = `SELECT c.release_id FROM (
|
||
SELECT l.release_id, count(*) OVER () AS holders
|
||
FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
|
||
) c WHERE c.holders <= ${COMMON_FILE}`
|
||
|
||
interface Member {
|
||
id: number
|
||
group_id: number
|
||
files: number
|
||
posted_at: Date | null
|
||
readable: boolean
|
||
poster_id?: number | null
|
||
/** The poster's text (joined from posters) where a query needs it. */
|
||
poster?: string | null
|
||
file_total?: number | null
|
||
}
|
||
|
||
/**
|
||
* The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
|
||
* describes are one upload. Obfuscated uploads post each file under a name of
|
||
* its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
|
||
* one-file releases), so the header pass can't group them; the prober reads
|
||
* each such file's first article (a `lone` candidate) and stores the MD5 of
|
||
* its first 16 KiB, and a par2 body's file list holds the same hash per file,
|
||
* whatever the file was renamed to. Releases in one group, posted within
|
||
* MAX_SPREAD_MS of a release that holds the set's par2, become the one with
|
||
* the most files: its chunks, probe candidates, inspections and hashes, the
|
||
* sums of its counts, and when none of them has a readable name the name the
|
||
* par2's file names share. Each merge is kept in release_merges.
|
||
*
|
||
* A second pass merges by inner name: the yEnc name in a lone file's first
|
||
* article keeps the upload's random stem when the subject names each file
|
||
* apart ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
|
||
* hep, hephaestus, gyges: every stem under one counter total; LOG.md
|
||
* 2026-10-12 (32)). Releases whose lone inspections share such a stem, in one
|
||
* group, by one poster, under the same counter total and within MAX_SPREAD_MS,
|
||
* are one upload. Only random stems: a readable one is grouped from the
|
||
* headers already, and a repost would share it.
|
||
*/
|
||
export class Par2Merger {
|
||
private stopped = false
|
||
private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
|
||
/** Null until read from worker_cursors (1970 the first time: everything). */
|
||
private since: Date | null = null
|
||
|
||
stop() {
|
||
this.stopped = true
|
||
}
|
||
|
||
async run() {
|
||
let prunedAt = 0
|
||
while (!this.stopped) {
|
||
const started = Date.now()
|
||
try {
|
||
// First, at start too: a prune needn't wait for a long round.
|
||
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
|
||
prunedAt = Date.now()
|
||
const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
|
||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
|
||
}
|
||
const round = await this.round()
|
||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
|
||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||
} catch (error: any) {
|
||
logger.error(`Par2 merge round failed: ${error.message}`)
|
||
}
|
||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||
}
|
||
}
|
||
|
||
/**
|
||
* The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
|
||
* lone inspection), a window at a time.
|
||
*/
|
||
async round(): Promise<MergeRound> {
|
||
// A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
|
||
const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
|
||
const end = new Date(now[0].at)
|
||
if (!this.since) {
|
||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||
this.since = saved ? new Date(saved.at) : new Date(0)
|
||
}
|
||
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
|
||
// par2 pass (hours when that starts over).
|
||
const folded = await this.folder.round()
|
||
let foldedAt = Date.now()
|
||
const foldAgain = async () => {
|
||
if (Date.now() - foldedAt < FOLD_EVERY_MS) return
|
||
const more = await this.folder.round()
|
||
folded.looked += more.looked
|
||
folded.sidecars += more.sidecars
|
||
folded.folded += more.folded
|
||
foldedAt = Date.now()
|
||
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||
}
|
||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||
while (this.since < end) {
|
||
if (this.stopped) return round
|
||
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
|
||
const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
|
||
if (from) {
|
||
const { touched, sets } = await this.setsToMerge(from, until)
|
||
round.sets += touched
|
||
for (const setId of sets) {
|
||
if (this.stopped) return round
|
||
await foldAgain()
|
||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||
return 0
|
||
})
|
||
if (merged > 0) {
|
||
round.merges++
|
||
round.releases += merged + 1
|
||
}
|
||
}
|
||
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
|
||
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
|
||
// 2026-10-23 (97)).
|
||
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
|
||
for (const { set_id: setId } of found) {
|
||
if (this.stopped) return round
|
||
const merged = await this.mergeByName(setId.trim()).catch((error) => {
|
||
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
|
||
return 0
|
||
})
|
||
if (merged > 0) {
|
||
round.nameMerges++
|
||
round.nameReleases += merged + 1
|
||
}
|
||
}
|
||
const { rows: stems } = await db.rawQuery(
|
||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
|
||
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
|
||
[from, until]
|
||
)
|
||
for (const { stem } of stems) {
|
||
if (this.stopped) return round
|
||
if (!looksRandom(stem)) continue
|
||
round.stems++
|
||
const merged = await this.mergeStem(stem).catch((error) => {
|
||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||
return 0
|
||
})
|
||
if (merged > 0) {
|
||
round.stemMerges++
|
||
round.stemReleases += merged + 1
|
||
}
|
||
}
|
||
}
|
||
// Only after a whole window: a restart halfway goes on from the window before.
|
||
this.since = until
|
||
await db.rawQuery(
|
||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||
[CURSOR, this.since]
|
||
)
|
||
}
|
||
return round
|
||
}
|
||
|
||
/** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
|
||
private async firstNew(from: Date, end: Date): Promise<Date | null> {
|
||
const { rows } = await db.rawQuery(
|
||
`SELECT least(
|
||
(SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
|
||
(SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
|
||
(SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
|
||
[from, from, from]
|
||
)
|
||
const at = rows[0].at ? new Date(rows[0].at) : null
|
||
return at && at <= end ? at : null
|
||
}
|
||
|
||
/**
|
||
* The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
|
||
* change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
|
||
* hour's 147k): mergeSet needn't look at them.
|
||
*/
|
||
private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
|
||
const { rows } = await db.rawQuery(
|
||
`WITH touched AS MATERIALIZED (
|
||
SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
|
||
UNION
|
||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
|
||
WHERE h.created_at >= ? AND h.created_at <= ?
|
||
)
|
||
SELECT (SELECT count(*) FROM touched) AS touched, array(
|
||
SELECT t.set_id FROM touched t
|
||
WHERE EXISTS (
|
||
SELECT 1 FROM (
|
||
SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
|
||
UNION ALL
|
||
SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
|
||
) u HAVING count(DISTINCT u.release_id) >= 2)
|
||
) AS sets`,
|
||
[from, until, from, until]
|
||
)
|
||
return { touched: Number(rows[0].touched), sets: rows[0].sets }
|
||
}
|
||
|
||
/**
|
||
* The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
|
||
* and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
|
||
* sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
|
||
* again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
|
||
*/
|
||
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
|
||
const { rows: all } = await trx.rawQuery(
|
||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
|
||
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
|
||
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
|
||
[setId]
|
||
)
|
||
const anchor: Member | undefined = all[0]
|
||
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
|
||
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
|
||
const holders = all.filter(
|
||
(h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
|
||
)
|
||
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||
if (described.length === 0) return null
|
||
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
|
||
const at = new Date(anchor.posted_at).getTime()
|
||
const { rows: candidates } = await trx.rawQuery(
|
||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
|
||
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
|
||
AND r.files <= ?
|
||
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
|
||
// More files than the set describes, with its sidecars, can't all be named in it.
|
||
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
|
||
)
|
||
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
|
||
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
|
||
for (const c of candidates) {
|
||
const own = await NzbService.files(Number(c.id), trx)
|
||
const content = own.filter((f) => !besideSet(f.name))
|
||
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
|
||
found.set(String(c.id), c)
|
||
files += Number(c.files)
|
||
}
|
||
if (found.size < 2) return null
|
||
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
|
||
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
|
||
return found
|
||
}
|
||
|
||
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
|
||
async mergeByName(setId: string): Promise<number> {
|
||
return db.transaction(async (trx) => {
|
||
const found = await this.nameMatches(trx, setId)
|
||
return found ? this.merge(trx, found, setId, 'name') : 0
|
||
})
|
||
}
|
||
|
||
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
|
||
async mergeStem(stem: string): Promise<number> {
|
||
return db.transaction(async (trx) => {
|
||
const { rows } = await trx.rawQuery(
|
||
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
|
||
FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
|
||
[stem]
|
||
)
|
||
const sorted: Member[] = rows.sort((a: Member, b: Member) => (a.posted_at ? new Date(a.posted_at).getTime() : 0) - (b.posted_at ? new Date(b.posted_at).getTime() : 0))
|
||
const anchor = sorted[0]
|
||
if (!anchor) return 0
|
||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
|
||
const near = (r: Member) =>
|
||
r.group_id === anchor.group_id &&
|
||
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
|
||
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
|
||
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||
const found = new Map<string, Member>()
|
||
for (const r of sorted.filter(near)) found.set(String(r.id), r)
|
||
if (found.size < 2) return 0
|
||
// No more files than the counter says, with room for par2 volumes posted under it.
|
||
const counted = Math.max(0, ...[...found.values()].map((r) => total(r) ?? 0))
|
||
const files = [...found.values()].reduce((n, m) => n + Number(m.files), 0)
|
||
if (counted > 0 && files > counted + SLACK_FILES) return 0
|
||
return this.merge(trx, found, stem, 'inner')
|
||
})
|
||
}
|
||
|
||
/** Merges the releases of one set; how many went into another. */
|
||
async mergeSet(setId: string): Promise<number> {
|
||
return db.transaction(async (trx) => {
|
||
const { rows: holders } = await trx.rawQuery(
|
||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||
WHERE s.set_id = ? ORDER BY r.id`,
|
||
[setId]
|
||
)
|
||
const anchor: Member | undefined = holders[0]
|
||
if (!anchor) return 0
|
||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||
const { rows: matches } = await trx.rawQuery(
|
||
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
|
||
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
|
||
WHERE f.set_id = ?`,
|
||
[setId]
|
||
)
|
||
// A file of the set that matches more than one release nearby is ambiguous: files that share their first
|
||
// 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
|
||
// one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
|
||
const byFile = new Map<string, Map<string, Member>>()
|
||
for (const m of matches.filter(near)) {
|
||
const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
|
||
releases.set(String(m.id), m)
|
||
byFile.set(m.file_id, releases)
|
||
}
|
||
const nearHolders = holders.filter(near)
|
||
if (nearHolders.length > MAX_HOLDERS) return 0
|
||
const found = new Map<string, Member>()
|
||
for (const r of nearHolders) found.set(String(r.id), r)
|
||
for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
|
||
if (found.size < 2) return 0
|
||
const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId])
|
||
const setFiles = Math.max(Number(described[0].n), 1)
|
||
if ([...found.values()].reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0
|
||
return this.merge(trx, found, setId, 'par2')
|
||
})
|
||
}
|
||
|
||
/**
|
||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||
*/
|
||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
|
||
{
|
||
// Locked in id order (the batch store and other rounds take the same rows).
|
||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||
const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
|
||
if (members.length < 2) return 0
|
||
const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
|
||
const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
|
||
const name = via === 'par2' && members.every((m) => !m.readable) ? await this.setName(trx, key) : undefined
|
||
|
||
await trx.rawQuery(
|
||
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
|
||
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||
[target.id, key, via, others]
|
||
)
|
||
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
|
||
// isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
|
||
await trx.rawQuery(
|
||
`DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
|
||
AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
|
||
OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
|
||
[others, target.id, others]
|
||
)
|
||
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
|
||
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||
}
|
||
// An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
|
||
await trx.rawQuery(
|
||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||
[target.id]
|
||
)
|
||
// The members' 7z tails together: those below the highest volume go.
|
||
await pruneSevenZipTails(trx, [target.id])
|
||
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
|
||
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
|
||
await trx.rawQuery(
|
||
`DELETE FROM par2_sets p WHERE p.release_id = ANY(?)
|
||
AND (EXISTS (SELECT 1 FROM par2_sets t WHERE t.set_id = p.set_id AND t.release_id = ?)
|
||
OR EXISTS (SELECT 1 FROM par2_sets o WHERE o.set_id = p.set_id AND o.release_id = ANY(?) AND o.release_id < p.release_id))`,
|
||
[others, target.id, others]
|
||
)
|
||
await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||
await trx.rawQuery(
|
||
`UPDATE releases t SET
|
||
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
|
||
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
|
||
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
|
||
first_seen_at = least(t.first_seen_at, s.first_seen_at),
|
||
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
|
||
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
|
||
name = coalesce(nullif(?, ''), t.name)
|
||
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
|
||
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
|
||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
|
||
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
|
||
max(file_total) AS file_total, max(inner_name) AS inner_name
|
||
FROM releases WHERE id = ANY(?)) s
|
||
WHERE t.id = ?`,
|
||
[name ?? '', others, target.id]
|
||
)
|
||
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
|
||
await trx.rawQuery(
|
||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||
[target.id]
|
||
)
|
||
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
||
await decidePar2Completeness(trx, [target.id])
|
||
return others.length
|
||
}
|
||
}
|
||
|
||
/** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
|
||
private async setName(trx: any, setId: string): Promise<string | undefined> {
|
||
const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||
const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
|
||
// A par2 made after the renaming names the random names again.
|
||
return name && !looksRandom(name) ? name.slice(0, 255) : undefined
|
||
}
|
||
}
|