Daan MeijerandClaude Opus 5.5
ebe9f8bb36
59% of settled releases had complete = NULL: without an [i/N] counter the settler left it unknown, and the "phase 2 reads par2 sets for those" step was never built. A release is now complete when every file its par2 sets describe is there by name or by its first-16-KiB hash, or, for an upload renamed after its par2 was made, when it has as many non-par2 files and bytes. Decided when a par2 index is probed, after a par2 merge, and before the pruner drops a set's file list; the batch store, settler and merger keep a decided value instead of resetting it to NULL. releases:par2-complete decides the releases probed before this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
54 lines
2.8 KiB
TypeScript
54 lines
2.8 KiB
TypeScript
import NzbService from '#services/nzb_service'
|
|
|
|
/** A file a par2 set describes (par2_files). */
|
|
export interface SetFile {
|
|
name: string
|
|
length: number
|
|
hash16k: string
|
|
}
|
|
|
|
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
|
|
|
/**
|
|
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
|
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
|
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
|
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
|
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
|
*/
|
|
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
|
const names = new Set(files.map((f) => base(f.name)))
|
|
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
|
const byName = set.filter((f) => names.has(base(f.name))).length
|
|
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
|
if (byName > 0) return false
|
|
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
|
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
|
}
|
|
|
|
/**
|
|
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
|
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
|
* a transaction that moved chunks (a merge) reads them as moved.
|
|
*/
|
|
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
|
if (releaseIds.length === 0) return 0
|
|
const { rows: releases } = await client.rawQuery(
|
|
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
|
[releaseIds.map(String)]
|
|
)
|
|
let decided = 0
|
|
for (const { id } of releases) {
|
|
const { rows: set } = await client.rawQuery(
|
|
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
|
[id]
|
|
)
|
|
if (set.length === 0) continue
|
|
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
|
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
|
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
|
decided++
|
|
}
|
|
return decided
|
|
}
|