Decide completeness by par2 set for releases without a counter
59% of settled releases had complete = NULL: without an [i/N] counter the settler left it unknown, and the "phase 2 reads par2 sets for those" step was never built. A release is now complete when every file its par2 sets describe is there by name or by its first-16-KiB hash, or, for an upload renamed after its par2 was made, when it has as many non-par2 files and bytes. Decided when a par2 index is probed, after a par2 merge, and before the pruner drops a set's file list; the batch store, settler and merger keep a decided value instead of resetting it to NULL. releases:par2-complete decides the releases probed before this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f75097d7ac
commit
ebe9f8bb36
10 files changed
+187
-12
No files matched your search
@@ -0,0 +1,53 @@
|
||||
import NzbService from '#services/nzb_service'
|
||||
|
||||
/** A file a par2 set describes (par2_files). */
|
||||
export interface SetFile {
|
||||
name: string
|
||||
length: number
|
||||
hash16k: string
|
||||
}
|
||||
|
||||
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
||||
|
||||
/**
|
||||
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
||||
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
||||
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
||||
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
||||
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
||||
*/
|
||||
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
||||
const names = new Set(files.map((f) => base(f.name)))
|
||||
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
||||
const byName = set.filter((f) => names.has(base(f.name))).length
|
||||
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
||||
if (byName > 0) return false
|
||||
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
||||
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
||||
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
||||
* a transaction that moved chunks (a merge) reads them as moved.
|
||||
*/
|
||||
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const { rows: releases } = await client.rawQuery(
|
||||
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
let decided = 0
|
||||
for (const { id } of releases) {
|
||||
const { rows: set } = await client.rawQuery(
|
||||
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (set.length === 0) continue
|
||||
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
||||
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
||||
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
||||
decided++
|
||||
}
|
||||
return decided
|
||||
}
|
||||
Reference in new issue
Block a user