Daan MeijerandClaude Opus 5.5
59481037fb
Release 145637 held four albums: the poster puts '[KiLL.dA.FuCkIn.
EaStErRaBbIt]----' before every upload, and the text outside the quotes
(when it names a release) was the key with the poster. The same made
one release of a poster's albums under '(PowBeertje80 Posted)' (a post
id, with the same [i/M]), of masterposts' films, and of one volume of
every set in a DVD9 post.
Now, when the file's stem describes something itself (12+ letters and
digits, two words of 3+ letters) and the text outside the quotes shares
no word with it (4+ letters as part of the stem, 3-letter words and
numbers whole; boilerplate like www/net/posted not counted; no 5+ digit
per-post id), that text is taken for the poster's tag and the stem
says which release the file is. Albums of track files ('FTA … Ketchup'),
'DIE_ZEIT_35' with '35_China_dl', 'FTD769997' posts, scene names with
abbreviated file names ('grp-sm') and generic sidecars ('file_id.diz')
keep their grouping.
releases:regroup re-keys a sample of the catalog with the current rule:
on homelab's test data 145637 → 4 albums, 111566 → 17 albums of 27
files, the rest of the subject releases that change mostly lose an ad
image ('Flyer-download-2-Day.gif'); stem releases unchanged. Ported to
edge-bench-rs: check.ts still finds all captures identical (ath 2013:
16 → 76 releases on both sides).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
2.7 KiB
TypeScript
62 lines
2.7 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
import { releaseIdentity } from '#lib/release/index'
|
|
|
|
/**
|
|
* What lib/release's identity makes of releases already in the catalog: a
|
|
* sample of releases (by source), their files decoded from the chunks and
|
|
* each given its release key again. Shows how many would now split, and
|
|
* into what: for checking an identity change on real data.
|
|
*/
|
|
export default class ReleasesRegroup extends BaseCommand {
|
|
static commandName = 'releases:regroup'
|
|
static description = 'Shows how the current release identity would regroup a sample of the catalog'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.number({ description: 'Releases sampled per source', default: 2000 })
|
|
declare sample: number
|
|
|
|
@flags.array({ description: 'Release ids to show in full' })
|
|
declare show?: string[]
|
|
|
|
async run() {
|
|
for (const id of this.show ?? []) {
|
|
const groups = await this.regroup(Number(id))
|
|
this.logger.info(`${id}: ${groups.size} release(s) now`)
|
|
for (const [, g] of groups) this.logger.info(` ${g.files} files: ${g.name} (${g.source})`)
|
|
}
|
|
if (this.show?.length) return
|
|
for (const source of ['subject', 'post', 'stem']) {
|
|
const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
|
|
let split = 0
|
|
let pieces = 0
|
|
const examples: string[] = []
|
|
for (const r of rows) {
|
|
const groups = await this.regroup(Number(r.id))
|
|
if (groups.size <= 1) continue
|
|
split++
|
|
pieces += groups.size
|
|
if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
|
|
}
|
|
this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
|
|
for (const e of examples) this.logger.info(` ${e.slice(0, 260)}`)
|
|
}
|
|
}
|
|
|
|
private async regroup(id: number) {
|
|
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
|
const groups = new Map<string, { name: string; source: string; files: number }>()
|
|
for (const c of chunks) {
|
|
for (const f of decodeChunk(c.payload)) {
|
|
const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
|
|
const g = groups.get(r.key)
|
|
if (g) g.files++
|
|
else groups.set(r.key, { name: r.name, source: r.source, files: 1 })
|
|
}
|
|
}
|
|
return groups
|
|
}
|
|
}
|