Rebuild: image sets, documents and audio albums as releases
Picture groups made a release per image: 501k single-file releases in
the first 24 ranges of the bottom 500 groups (a.b.models 385k), ~120M
for all of them. Counters written out ('File 027 of 132', '[12of42]',
'104 of 472', '[2239-6389]', '02/17', '0097-1574'; only when the first
number is at most the second, so '[1948-12]' stays) and per-file sizes
('76.88 MB', '335555 Bytes') kept every subject unique.
- subjectRemainder removes those counters and sizes, and empty brackets.
- A subject-named release is bounded by its counter's total
(counterTotal: the first form found): uploads a poster names alike
stay apart, an upload's files (par2, nfo) together.
- Images and audio tracks don't take the poster-tag rule: the subject
names the set or album. An image set without a counter: the image's
name up to its last number, without numbers.
- Documents and comics (pdf, epub, mobi, azw3, djvu, cbr, cbz, cb7, chm,
lit, doc, docx) are a release each: collections post hundreds under
one subject ('cw-horror&mystery').
releases:regroup --group re-keys a group's indexed releases together:
cheerleaders 55,741 -> 3,165 releases, models 385,298 -> 40,313, German
audiobooks 4,106 -> 1,724, playstation 3,309 -> 2,345; comics and
e-books a release per item; boneless unchanged. Ported to edge-bench-rs;
check.ts: identical on all captures.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
59481037fb
commit
14758a5b7a
4 files changed
+272
-15
No files matched your search
@@ -21,7 +21,32 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
@flags.array({ description: 'Release ids to show in full' })
|
||||
declare show?: string[]
|
||||
|
||||
@flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
|
||||
declare group?: string[]
|
||||
|
||||
async run() {
|
||||
for (const name of this.group ?? []) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
|
||||
[name]
|
||||
)
|
||||
const ids = rows.map((r: { release_id: string }) => r.release_id)
|
||||
const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
|
||||
for (const id of ids) {
|
||||
for (const [key, g] of await this.regroup(Number(id))) {
|
||||
const known = all.get(key)
|
||||
if (known) known.files += g.files
|
||||
else all.set(key, { ...g })
|
||||
}
|
||||
}
|
||||
const sizes = [...all.values()].map((g) => g.files)
|
||||
const single = sizes.filter((n) => n === 1).length
|
||||
this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
|
||||
for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(` ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
|
||||
const singles = [...all.values()].filter((g) => g.files === 1)
|
||||
for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(` single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
|
||||
}
|
||||
if (this.group?.length) return
|
||||
for (const id of this.show ?? []) {
|
||||
const groups = await this.regroup(Number(id))
|
||||
this.logger.info(`${id}: ${groups.size} release(s) now`)
|
||||
@@ -47,13 +72,13 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
|
||||
private async regroup(id: number) {
|
||||
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
||||
const groups = new Map<string, { name: string; source: string; files: number }>()
|
||||
const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
|
||||
for (const c of chunks) {
|
||||
for (const f of decodeChunk(c.payload)) {
|
||||
const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
|
||||
const g = groups.get(r.key)
|
||||
if (g) g.files++
|
||||
else groups.set(r.key, { name: r.name, source: r.source, files: 1 })
|
||||
else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
|
||||
}
|
||||
}
|
||||
return groups
|
||||
|
||||
Reference in new issue
Block a user