Daan MeijerandClaude Opus 5.5
0613984de9
A poster re-uploading under the same name made one release of all uploads: in a sample, 24% of releases with 2+ files had a file name more than once (4.3% of files extra copies), 70% of those copies posted a day or more apart; their NZBs listed each copy. - File assembly keeps a re-upload's copy apart: a part already in every open copy, posted more than 3 h from them, starts another copy; a missing part fills the nearest copy (a re-send of what was missing), a re-sent part near a copy that has it is ignored as before. - A key's files in a batch split into uploads where posting pauses for more than 3 h, counted from the latest article posted so far (a file that takes hours to post doesn't split it); each upload is judged and chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry their upload). - The catalog matches a chunk to the release of its key whose articles the range's group owns and whose posting window is within 3 h, else starts one (key, key:2, …): releases get identity_key, group_id, last_posted_at. Keys are locked in order (advisory locks), so concurrent batches neither deadlock nor both start one; the merger writes in key order too. The same upload multi-posted to other groups is a release per group. - The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies come later (releases:upload-gaps). releases:duplicates measures the duplicate names. Both implementations; check.ts compares the windows and uploads too (identical on all captures; Rust tests mirror the TypeScript cases). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
2.8 KiB
TypeScript
59 lines
2.8 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
|
|
/**
|
|
* For telling uploads apart (re-uploads under one release key): how long
|
|
* one upload takes to post (the span of a release's files' dates, in
|
|
* releases without a file name twice), and how far apart the copies of a
|
|
* file name are in releases that have one twice.
|
|
*/
|
|
export default class ReleasesUploadGaps extends BaseCommand {
|
|
static commandName = 'releases:upload-gaps'
|
|
static description = 'Posting spans of uploads and gaps between re-uploads, in a sample'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
|
|
declare sample: number
|
|
|
|
async run() {
|
|
const ids: string[] = (await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])).rows.map((r: { id: string }) => r.id)
|
|
const bins = [10, 60, 180, 360, 720, 1440, 4320, Number.POSITIVE_INFINITY]
|
|
const label = (m: number) => (m === Number.POSITIVE_INFINITY ? '>3d' : m >= 1440 ? `≤${m / 1440}d` : m >= 60 ? `≤${m / 60}h` : `≤${m}min`)
|
|
const spans = new Array(bins.length).fill(0)
|
|
const gaps = new Array(bins.length).fill(0)
|
|
const bin = (minutes: number) => bins.findIndex((b) => minutes <= b)
|
|
for (const id of ids) {
|
|
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
|
const byName = new Map<string, number[]>()
|
|
const times: number[] = []
|
|
for (const c of chunks) {
|
|
for (const f of decodeChunk(c.payload)) {
|
|
if (f.postedAt === null) continue
|
|
times.push(f.postedAt)
|
|
byName.set(f.name, [...(byName.get(f.name) ?? []), f.postedAt])
|
|
}
|
|
}
|
|
if (times.length < 2) continue
|
|
const twice = [...byName.values()].filter((t) => t.length > 1)
|
|
if (twice.length === 0) {
|
|
spans[bin((Math.max(...times) - Math.min(...times)) / 60_000)]++
|
|
continue
|
|
}
|
|
// The gap between a name's copies (sorted; each step), so a release reposted 3 times counts 2 gaps per name.
|
|
for (const t of twice) {
|
|
t.sort((a, b) => a - b)
|
|
for (let i = 1; i < t.length; i++) gaps[bin((t[i] - t[i - 1]) / 60_000)]++
|
|
}
|
|
}
|
|
const show = (counts: number[]) => {
|
|
const total = counts.reduce((a, b) => a + b, 0)
|
|
let cumulative = 0
|
|
return bins.map((b, i) => `${label(b)} ${((100 * (cumulative += counts[i])) / Math.max(1, total)).toFixed(1)}%`).join(', ') + ` (n=${total})`
|
|
}
|
|
this.logger.info(`one upload's posting span (cumulative): ${show(spans)}`)
|
|
this.logger.info(`gap between copies of a name (cumulative): ${show(gaps)}`)
|
|
}
|
|
}
|