Join an upload's releases by its par2 set's file names
The files of one upload often became several releases whose names
differ slightly: scene tracks apart from their album's nfo/sfv/par2,
"Vermaechtnis" beside a garbled "Vermächtnis", an index named
Die.Bergretter.S18E05... beside its video wishtv-diebergretter-s18e05.
The par2 merge joined releases only by first-16-KiB hashes, which
readable files never get. A name pass now looks, for every set the
merger finds, at the set poster's releases in its group posted within
12 hours: those whose files the set all names (without case or
punctuation, Latin-1-read UTF-8 read back; sidecars, par2 and images
aside) join the set's release, within the hash pass's file bound.
Merges are kept in release_merges as via 'name'. releases:merge-names
runs it over the sets found before; on 300 recent sets 22 merges of 147
releases, all one upload each.
Also: volumes split after .wmv, .mov, .mpg and other video extensions
share a stem ("Morning Lovers.wmv.001" to ".058" were 58 releases), in
the Rust edge and the TypeScript identity.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e4044cb1d1
commit
e00e4f3988
7 files changed
+242
-11
No files matched your search
@@ -0,0 +1,61 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Merger } from '#services/par2_merger'
|
||||
|
||||
/**
|
||||
* The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
|
||||
* first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
|
||||
*/
|
||||
export default class ReleasesMergeNames extends BaseCommand {
|
||||
static commandName = 'releases:merge-names'
|
||||
static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Sets to look at (newest first); all without' })
|
||||
declare sets: number
|
||||
|
||||
@flags.boolean({ description: 'Count and show, merge nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const merger = new Par2Merger()
|
||||
let before = '9999-12-31'
|
||||
let looked = 0
|
||||
let merges = 0
|
||||
let releases = 0
|
||||
const examples: string[] = []
|
||||
while (this.sets === undefined || looked < this.sets) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE s.created_at < ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id ORDER BY at DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
[before]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
for (const { set_id: setId } of rows) {
|
||||
looked++
|
||||
if (this.dryRun) {
|
||||
const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
|
||||
if (found) {
|
||||
merges++
|
||||
releases += found.size
|
||||
if (examples.length < 15) {
|
||||
const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
|
||||
examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const merged = await merger.mergeByName(setId.trim())
|
||||
if (merged > 0) {
|
||||
merges++
|
||||
releases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
before = rows.at(-1).at
|
||||
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
for (const e of examples) this.logger.info(` ${e.slice(0, 300)}`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user