Daan MeijerandClaude Opus 5.5
0613984de9
A poster re-uploading under the same name made one release of all uploads: in a sample, 24% of releases with 2+ files had a file name more than once (4.3% of files extra copies), 70% of those copies posted a day or more apart; their NZBs listed each copy. - File assembly keeps a re-upload's copy apart: a part already in every open copy, posted more than 3 h from them, starts another copy; a missing part fills the nearest copy (a re-send of what was missing), a re-sent part near a copy that has it is ignored as before. - A key's files in a batch split into uploads where posting pauses for more than 3 h, counted from the latest article posted so far (a file that takes hours to post doesn't split it); each upload is judged and chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry their upload). - The catalog matches a chunk to the release of its key whose articles the range's group owns and whose posting window is within 3 h, else starts one (key, key:2, …): releases get identity_key, group_id, last_posted_at. Keys are locked in order (advisory locks), so concurrent batches neither deadlock nor both start one; the merger writes in key order too. The same upload multi-posted to other groups is a release per group. - The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies come later (releases:upload-gaps). releases:duplicates measures the duplicate names. Both implementations; check.ts compares the windows and uploads too (identical on all captures; Rust tests mirror the TypeScript cases). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
83 lines
4.4 KiB
TypeScript
83 lines
4.4 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
|
|
/**
|
|
* How often a release holds a file name more than once (decoded from its
|
|
* chunks): copies of one upload posted again (multi-posting, reposts that
|
|
* share a key) or different files that share a name. Copies of the same
|
|
* size are counted as the same file posted again.
|
|
*/
|
|
export default class ReleasesDuplicates extends BaseCommand {
|
|
static commandName = 'releases:duplicates'
|
|
static description = 'Counts file names a release holds more than once, in a sample'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
|
|
declare sample: number
|
|
|
|
async run() {
|
|
const ids: string[] = (
|
|
await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])
|
|
).rows.map((r: { id: string }) => r.id)
|
|
const kind = (name: string) =>
|
|
/\.(jpe?g|png|gif|bmp|webp)$/i.test(name) ? 'image'
|
|
: /\.(mp3|flac|m4a|ogg|wav)\b/i.test(name) ? 'audio'
|
|
: /\.(par2)$/i.test(name) ? 'par2'
|
|
: /\.(rar|r\d\d|7z|zip|\d{3})$/i.test(name) ? 'archive'
|
|
: /\.(pdf|epub|mobi|cbr|cbz|djvu|chm)$/i.test(name) ? 'document'
|
|
: /\.(nfo|sfv|nzb|txt|diz)$/i.test(name) ? 'sidecar'
|
|
: 'other'
|
|
let releases = 0
|
|
let withDuplicates = 0
|
|
let files = 0
|
|
let extra = 0
|
|
let completeOnlyByCopies = 0
|
|
const byKind = new Map<string, number>()
|
|
const how = { sameFile: 0, differentFile: 0, sameDay: 0, laterRepost: 0 }
|
|
const examples: string[] = []
|
|
for (const id of ids) {
|
|
const release = await db.from('releases').where('id', id).select('name', 'file_total', 'files').first()
|
|
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
|
const byName = new Map<string, Map<string, { parts: number; bytes: number; at: number | null }>>()
|
|
for (const c of chunks) {
|
|
for (const f of decodeChunk(c.payload)) {
|
|
const copies = byName.get(f.name) ?? new Map()
|
|
copies.set(f.segments[0]?.[1] ?? '', { parts: f.segments.length, bytes: f.segments.reduce((n, [, , b]) => n + b, 0), at: f.postedAt })
|
|
byName.set(f.name, copies)
|
|
}
|
|
}
|
|
releases++
|
|
let dup = false
|
|
for (const [name, copies] of byName) {
|
|
files += copies.size
|
|
if (copies.size < 2) continue
|
|
dup = true
|
|
extra += copies.size - 1
|
|
byKind.set(kind(name), (byKind.get(kind(name)) ?? 0) + copies.size - 1)
|
|
const list = [...copies.values()]
|
|
// The same file again: as many parts, within 3% in bytes (article sizes include headers).
|
|
const same = list.every((c) => c.parts === list[0].parts && Math.abs(c.bytes - list[0].bytes) <= 0.03 * list[0].bytes)
|
|
if (same) how.sameFile++
|
|
else how.differentFile++
|
|
const times = list.map((c) => c.at).filter((t): t is number => t !== null)
|
|
if (times.length > 1 && Math.max(...times) - Math.min(...times) > 86_400_000) how.laterRepost++
|
|
else how.sameDay++
|
|
if (!same && examples.length < 8) examples.push(`${id} ${release.name.slice(0, 60)} | ${name.slice(0, 60)} | ${list.map((c) => `${c.parts}p ${(c.bytes / 1e6).toFixed(1)}MB`).join(', ')}`)
|
|
}
|
|
if (!dup) continue
|
|
withDuplicates++
|
|
const total = Number(release.file_total ?? 0)
|
|
if (total && Number(release.files) >= total && byName.size < total) completeOnlyByCopies++
|
|
}
|
|
const pct = (n: number, of: number) => `${((100 * n) / Math.max(1, of)).toFixed(2)}%`
|
|
this.logger.info(`${releases} releases with 2+ files sampled`)
|
|
this.logger.info(` with a file name more than once: ${withDuplicates} (${pct(withDuplicates, releases)})`)
|
|
this.logger.info(` extra copies: ${extra} of ${files} files (${pct(extra, files)}); by type: ${[...byKind].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ${n}`).join(', ')}`)
|
|
this.logger.info(` duplicated names: same file again ${how.sameFile}, different (parts or size) ${how.differentFile}; copies the same day ${how.sameDay}, a day or more apart ${how.laterRepost}`)
|
|
this.logger.info(` 'complete' only by counting copies: ${completeOnlyByCopies}`)
|
|
for (const e of examples) this.logger.info(` different: ${e}`)
|
|
}
|
|
}
|