import { BaseCommand, flags } from '@adonisjs/core/ace' import type { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' import { decodeChunk } from '#lib/chunks/index' /** * How often a release holds a file name more than once (decoded from its * chunks): copies of one upload posted again (multi-posting, reposts that * share a key) or different files that share a name. Copies of the same * size are counted as the same file posted again. */ export default class ReleasesDuplicates extends BaseCommand { static commandName = 'releases:duplicates' static description = 'Counts file names a release holds more than once, in a sample' static options: CommandOptions = { startApp: true } @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 }) declare sample: number async run() { const ids: string[] = ( await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample]) ).rows.map((r: { id: string }) => r.id) const kind = (name: string) => /\.(jpe?g|png|gif|bmp|webp)$/i.test(name) ? 'image' : /\.(mp3|flac|m4a|ogg|wav)\b/i.test(name) ? 'audio' : /\.(par2)$/i.test(name) ? 'par2' : /\.(rar|r\d\d|7z|zip|\d{3})$/i.test(name) ? 'archive' : /\.(pdf|epub|mobi|cbr|cbz|djvu|chm)$/i.test(name) ? 'document' : /\.(nfo|sfv|nzb|txt|diz)$/i.test(name) ? 'sidecar' : 'other' let releases = 0 let withDuplicates = 0 let files = 0 let extra = 0 let completeOnlyByCopies = 0 const byKind = new Map() const how = { sameFile: 0, differentFile: 0, sameDay: 0, laterRepost: 0 } const examples: string[] = [] for (const id of ids) { const release = await db.from('releases').where('id', id).select('name', 'file_total', 'files').first() const chunks = await db.from('release_chunks').where('release_id', id).select('payload') const byName = new Map>() for (const c of chunks) { for (const f of decodeChunk(c.payload)) { const copies = byName.get(f.name) ?? new Map() copies.set(f.segments[0]?.[1] ?? '', { parts: f.segments.length, bytes: f.segments.reduce((n, [, , b]) => n + b, 0), at: f.postedAt }) byName.set(f.name, copies) } } releases++ let dup = false for (const [name, copies] of byName) { files += copies.size if (copies.size < 2) continue dup = true extra += copies.size - 1 byKind.set(kind(name), (byKind.get(kind(name)) ?? 0) + copies.size - 1) const list = [...copies.values()] // The same file again: as many parts, within 3% in bytes (article sizes include headers). const same = list.every((c) => c.parts === list[0].parts && Math.abs(c.bytes - list[0].bytes) <= 0.03 * list[0].bytes) if (same) how.sameFile++ else how.differentFile++ const times = list.map((c) => c.at).filter((t): t is number => t !== null) if (times.length > 1 && Math.max(...times) - Math.min(...times) > 86_400_000) how.laterRepost++ else how.sameDay++ if (!same && examples.length < 8) examples.push(`${id} ${release.name.slice(0, 60)} | ${name.slice(0, 60)} | ${list.map((c) => `${c.parts}p ${(c.bytes / 1e6).toFixed(1)}MB`).join(', ')}`) } if (!dup) continue withDuplicates++ const total = Number(release.file_total ?? 0) if (total && Number(release.files) >= total && byName.size < total) completeOnlyByCopies++ } const pct = (n: number, of: number) => `${((100 * n) / Math.max(1, of)).toFixed(2)}%` this.logger.info(`${releases} releases with 2+ files sampled`) this.logger.info(` with a file name more than once: ${withDuplicates} (${pct(withDuplicates, releases)})`) this.logger.info(` extra copies: ${extra} of ${files} files (${pct(extra, files)}); by type: ${[...byKind].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ${n}`).join(', ')}`) this.logger.info(` duplicated names: same file again ${how.sameFile}, different (parts or size) ${how.differentFile}; copies the same day ${how.sameDay}, a day or more apart ${how.laterRepost}`) this.logger.info(` 'complete' only by counting copies: ${completeOnlyByCopies}`) for (const e of examples) this.logger.info(` different: ${e}`) } }