Join an upload's releases by its par2 set's file names
The files of one upload often became several releases whose names
differ slightly: scene tracks apart from their album's nfo/sfv/par2,
"Vermaechtnis" beside a garbled "Vermächtnis", an index named
Die.Bergretter.S18E05... beside its video wishtv-diebergretter-s18e05.
The par2 merge joined releases only by first-16-KiB hashes, which
readable files never get. A name pass now looks, for every set the
merger finds, at the set poster's releases in its group posted within
12 hours: those whose files the set all names (without case or
punctuation, Latin-1-read UTF-8 read back; sidecars, par2 and images
aside) join the set's release, within the hash pass's file bound.
Merges are kept in release_merges as via 'name'. releases:merge-names
runs it over the sets found before; on 300 recent sets 22 merges of 147
releases, all one upload each.
Also: volumes split after .wmv, .mov, .mpg and other video extensions
share a stem ("Morning Lovers.wmv.001" to ".058" were 58 releases), in
the Rust edge and the TypeScript identity.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
e4044cb1d1
commit
e00e4f3988
7 files changed
+242
-11
No files matched your search
@@ -2,7 +2,9 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { par2ReleaseName } from '#lib/release/index'
|
||||
import { looksRandom } from '#lib/release/random'
|
||||
import { SidecarFolder } from '#services/sidecar_folder'
|
||||
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
|
||||
import { randomPoster } from '#lib/release/identity'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
@@ -23,6 +25,14 @@ const SLACK_FILES = 10
|
||||
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
|
||||
*/
|
||||
const COMMON_FILE = 20
|
||||
/**
|
||||
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
|
||||
* NAME_CANDIDATES of them (a poster's uploads of a day).
|
||||
*/
|
||||
const NAME_SPREAD_MS = 12 * 3600 * 1000
|
||||
const NAME_CANDIDATES = 200
|
||||
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
|
||||
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
|
||||
/** Between rounds. */
|
||||
const ROUND_MS = 60_000
|
||||
/** A window looks back this far past the previous one's end (rows committed after it was read). */
|
||||
@@ -45,6 +55,9 @@ export interface MergeRound {
|
||||
sets: number
|
||||
merges: number
|
||||
releases: number
|
||||
/** The name pass: merges, releases in them. */
|
||||
nameMerges: number
|
||||
nameReleases: number
|
||||
/** The inner-name pass: stems checked, merges, releases in them. */
|
||||
stems: number
|
||||
stemMerges: number
|
||||
@@ -59,6 +72,19 @@ export interface MergeRound {
|
||||
*/
|
||||
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
|
||||
|
||||
/**
|
||||
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
|
||||
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
|
||||
*/
|
||||
export function fileKey(name: string): string {
|
||||
let n = name.replace(/^.*[\\/]/, '')
|
||||
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
|
||||
const read = Buffer.from(n, 'latin1').toString('utf8')
|
||||
if (!read.includes('\ufffd')) n = read
|
||||
}
|
||||
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
|
||||
const HOLDERS = `SELECT c.release_id FROM (
|
||||
SELECT l.release_id, count(*) OVER () AS holders
|
||||
@@ -121,6 +147,7 @@ export class Par2Merger {
|
||||
}
|
||||
const round = await this.round()
|
||||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||||
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
|
||||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||||
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||||
} catch (error: any) {
|
||||
@@ -155,7 +182,7 @@ export class Par2Merger {
|
||||
foldedAt = Date.now()
|
||||
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||||
}
|
||||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
while (this.since < end) {
|
||||
if (this.stopped) return round
|
||||
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
|
||||
@@ -176,6 +203,21 @@ export class Par2Merger {
|
||||
round.releases += merged + 1
|
||||
}
|
||||
}
|
||||
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
|
||||
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
|
||||
// 2026-10-23 (97)).
|
||||
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
|
||||
for (const { set_id: setId } of found) {
|
||||
if (this.stopped) return round
|
||||
const merged = await this.mergeByName(setId.trim()).catch((error) => {
|
||||
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.nameMerges++
|
||||
round.nameReleases += merged + 1
|
||||
}
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
|
||||
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
@@ -245,6 +287,55 @@ export class Par2Merger {
|
||||
return { touched: Number(rows[0].touched), sets: rows[0].sets }
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases a set's file names find (mergeByName without merging): its holders, and the releases of the set's
|
||||
* poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides sidecars, par2 and images.
|
||||
* Null when there's nothing to merge.
|
||||
*/
|
||||
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
|
||||
const { rows: holders } = await trx.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
WHERE s.set_id = ? ORDER BY r.id`,
|
||||
[setId]
|
||||
)
|
||||
const anchor: Member | undefined = holders[0]
|
||||
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
|
||||
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
|
||||
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||||
if (described.length === 0) return null
|
||||
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
|
||||
const at = new Date(anchor.posted_at).getTime()
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster FROM releases r
|
||||
WHERE left(r.poster, 200) = left(?, 200) AND r.poster = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
|
||||
AND r.files <= ?
|
||||
ORDER BY left(r.poster, 200), r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
|
||||
// More files than the set describes, with its sidecars, can't all be named in it.
|
||||
[anchor.poster, anchor.poster, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
|
||||
)
|
||||
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
|
||||
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
|
||||
for (const c of candidates) {
|
||||
const own = await NzbService.files(Number(c.id), trx)
|
||||
const content = own.filter((f) => !besideSet(f.name))
|
||||
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
|
||||
found.set(String(c.id), c)
|
||||
files += Number(c.files)
|
||||
}
|
||||
if (found.size < 2) return null
|
||||
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
|
||||
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
|
||||
return found
|
||||
}
|
||||
|
||||
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
|
||||
async mergeByName(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const found = await this.nameMatches(trx, setId)
|
||||
return found ? this.merge(trx, found, setId, 'name') : 0
|
||||
})
|
||||
}
|
||||
|
||||
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
|
||||
async mergeStem(stem: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
@@ -319,7 +410,7 @@ export class Par2Merger {
|
||||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||||
*/
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
|
||||
{
|
||||
// Locked in id order (the batch store and other rounds take the same rows).
|
||||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Merger } from '#services/par2_merger'
|
||||
|
||||
/**
|
||||
* The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
|
||||
* first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
|
||||
*/
|
||||
export default class ReleasesMergeNames extends BaseCommand {
|
||||
static commandName = 'releases:merge-names'
|
||||
static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Sets to look at (newest first); all without' })
|
||||
declare sets: number
|
||||
|
||||
@flags.boolean({ description: 'Count and show, merge nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const merger = new Par2Merger()
|
||||
let before = '9999-12-31'
|
||||
let looked = 0
|
||||
let merges = 0
|
||||
let releases = 0
|
||||
const examples: string[] = []
|
||||
while (this.sets === undefined || looked < this.sets) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE s.created_at < ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id ORDER BY at DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
[before]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
for (const { set_id: setId } of rows) {
|
||||
looked++
|
||||
if (this.dryRun) {
|
||||
const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
|
||||
if (found) {
|
||||
merges++
|
||||
releases += found.size
|
||||
if (examples.length < 15) {
|
||||
const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
|
||||
examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const merged = await merger.mergeByName(setId.trim())
|
||||
if (merged > 0) {
|
||||
merges++
|
||||
releases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
before = rows.at(-1).at
|
||||
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
for (const e of examples) this.logger.info(` ${e.slice(0, 300)}`)
|
||||
}
|
||||
}
|
||||
+10
-4
@@ -40,11 +40,11 @@ static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||||
(re(r"(?i)\.tar$"), ""),
|
||||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$"), ""),
|
||||
]
|
||||
});
|
||||
|
||||
@@ -147,10 +147,13 @@ fn strip_volume_digits(s: &[u8]) -> Option<usize> {
|
||||
((2..=3).contains(&d) && n > d && s[n - d - 1] == b'.').then(|| n - d - 1)
|
||||
}
|
||||
|
||||
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||||
const SPLIT_EXTS: [&[u8]; 23] = [
|
||||
b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob", b"wmv", b"mov", b"mpg", b"mpeg", b"divx", b"webm", b"flv", b"m2v",
|
||||
b"mts",
|
||||
];
|
||||
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||||
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||||
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||||
const MEDIA_EXTS: [&[u8]; 16] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf", b"wmv", b"mov", b"mpg", b"mpeg"];
|
||||
|
||||
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||||
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||||
@@ -1326,6 +1329,9 @@ mod tests {
|
||||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||||
assert_eq!(file_stem("a.7z.002"), "a");
|
||||
assert_eq!(file_stem("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
assert_eq!(file_stem("clip.mov.12"), "clip");
|
||||
assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,12 +40,13 @@ const SUFFIXES = [
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r\d{2,3}))\.\d{2,3}$/i,
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts), and other video
|
||||
// ("Morning Lovers.wmv.001" to ".058" were 58 releases; LOG.md 2026-10-23 (97)).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r\d{2,3}))\.\d{2,3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
|
||||
@@ -49,7 +49,7 @@ test.group('Par2Merger', (group) => {
|
||||
await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
|
||||
|
||||
const round = await new Par2Merger().round()
|
||||
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, stems: 0, stemMerges: 0, stemReleases: 0, folded: { looked: 0, sidecars: 0, folded: 0 } })
|
||||
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded: { looked: 0, sidecars: 0, folded: 0 } })
|
||||
const target = await db.from('releases').where('id', b).firstOrFail()
|
||||
assert.equal(target.files, 4)
|
||||
assert.equal(target.segments, 40)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import { Par2Merger, fileKey } from '#services/par2_merger'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||
import type { Overview } from '#lib/nntp/index'
|
||||
|
||||
const GROUP = 'alt.binaries.test'
|
||||
|
||||
/** One file without a counter, from `poster`, posted `minutes` after the first. */
|
||||
function file(name: string, at: number, minutes: number, poster = 'up <up@load.er>'): Overview[] {
|
||||
const date = new Date(Date.parse('2026-10-20T10:00:00Z') + minutes * 60_000).toUTCString().replace('GMT', '+0000')
|
||||
return [[at, { 'subject': `"${name}" yEnc (1/1)`, 'from': poster, 'date': date, 'message-id': `<${at}@test>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at}` }] as Overview]
|
||||
}
|
||||
|
||||
let stores = 0
|
||||
async function store(overviews: Overview[]) {
|
||||
stores++
|
||||
const g = (await db.from('groups').where('name', GROUP).first()) ?? (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('*'))[0]
|
||||
const [n] = await db.table('nodes').insert({ name: `n${stores}` }).returning('id')
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: stores * 100_000, end_id: stores * 100_000 + 99_999, next_id: stores * 100_000, status: 'leased', node_id: n.id }).returning('id')
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: stores * 100_000, end: stores * 100_000 + 99_999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||
indexer.add(overviews, stores * 100_000 + 100_000)
|
||||
indexer.finish()
|
||||
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
||||
}
|
||||
|
||||
test.group('Par2Merger / by file names', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('a name as compared: no case, spaces or punctuation, UTF-8 read as Latin-1 read back', ({ assert }) => {
|
||||
assert.equal(fileKey('Some/Dir/Ein.dunkles.Vermächtnis - 1080p.MKV'), fileKey('Ein dunkles Vermächtnis 1080p.mkv'))
|
||||
assert.notEqual(fileKey('Track One.flac'), fileKey('Track Two.flac'))
|
||||
})
|
||||
|
||||
test("a set's poster's releases nearby whose files it all names join its release; others stay", async ({ assert }) => {
|
||||
await store([
|
||||
// The album's index files, its tracks apart (no counter: a release each), and the same poster's next album.
|
||||
...file('Some.Album-WEB-2026.par2', 100_010, 0),
|
||||
...file('Some.Album-WEB-2026.nfo', 100_011, 0),
|
||||
...file('Track One.flac', 100_020, 1),
|
||||
...file('Track Two.flac', 100_030, 2),
|
||||
...file('Ein.dunkles.Vermächtnis.mkv', 100_040, 3),
|
||||
...file('Another Song.flac', 100_050, 4),
|
||||
// A track the set names, but posted by someone else, and one a day later.
|
||||
...file('Track One.flac', 100_060, 5, 'other <o@x>'),
|
||||
])
|
||||
await store(file('Track Two.flac', 200_070, 24 * 60 + 10))
|
||||
const ids = Object.fromEntries((await db.from('releases').select('id', 'name', 'poster', 'posted_at')).map((r) => [`${r.name}${r.poster.startsWith('other') ? ' (other)' : ''}${new Date(r.posted_at).getUTCDate() === 21 ? ' (later)' : ''}`, Number(r.id)]))
|
||||
const index = ids['Some.Album-WEB-2026']
|
||||
const setId = 'a'.repeat(32)
|
||||
await db.table('par2_files').multiInsert(
|
||||
['Track One.flac', 'Track Two.flac', 'Ein.dunkles.Vermächtnis.mkv'].map((name, i) => ({ set_id: setId, file_id: String(i).repeat(32), hash16k: 'f'.repeat(32), length: 600_000, name }))
|
||||
)
|
||||
await db.table('par2_sets').insert({ set_id: setId, release_id: index, files: 3 })
|
||||
|
||||
assert.equal(await new Par2Merger().mergeByName(setId), 3)
|
||||
const left = (await db.from('releases').orderBy('name').select('name', 'files', 'poster')).map((r) => `${r.name} ${r.files}${r.poster.startsWith('other') ? ' (other)' : ''}`)
|
||||
assert.includeMembers(left, ['Another Song 1', 'Track One 1 (other)', 'Track Two 1'])
|
||||
assert.lengthOf(left, 4)
|
||||
const merged = await db.from('release_merges').where('via', 'name').count('* as n')
|
||||
assert.equal(Number(merged[0].n), 3)
|
||||
// Nothing more to find.
|
||||
assert.equal(await new Par2Merger().mergeByName(setId), 0)
|
||||
})
|
||||
})
|
||||
@@ -16,6 +16,10 @@ test.group('release / fileStem', () => {
|
||||
assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0')
|
||||
assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024')
|
||||
// Other video split too, one release (58 were).
|
||||
assert.equal(fileStem('Morning Lovers.wmv.001'), 'Morning Lovers')
|
||||
assert.equal(fileStem('Morning Lovers.wmv.058'), 'Morning Lovers')
|
||||
assert.equal(fileStem('clip.mov.12'), 'clip')
|
||||
assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265')
|
||||
assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264')
|
||||
assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024')
|
||||
|
||||
Reference in new issue
Block a user