Rebuild: re-uploads are releases of their own
A poster re-uploading under the same name made one release of all uploads: in a sample, 24% of releases with 2+ files had a file name more than once (4.3% of files extra copies), 70% of those copies posted a day or more apart; their NZBs listed each copy. - File assembly keeps a re-upload's copy apart: a part already in every open copy, posted more than 3 h from them, starts another copy; a missing part fills the nearest copy (a re-send of what was missing), a re-sent part near a copy that has it is ignored as before. - A key's files in a batch split into uploads where posting pauses for more than 3 h, counted from the latest article posted so far (a file that takes hours to post doesn't split it); each upload is judged and chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry their upload). - The catalog matches a chunk to the release of its key whose articles the range's group owns and whose posting window is within 3 h, else starts one (key, key:2, …): releases get identity_key, group_id, last_posted_at. Keys are locked in order (advisory locks), so concurrent batches neither deadlock nor both start one; the merger writes in key order too. The same upload multi-posted to other groups is a release per group. - The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies come later (releases:upload-gaps). releases:duplicates measures the duplicate names. Both implementations; check.ts compares the windows and uploads too (identical on all captures; Rust tests mirror the TypeScript cases). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d0f3497ec9
commit
0613984de9
15 files changed
+715
-140
No files matched your search
@@ -0,0 +1,58 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
|
||||
/**
|
||||
* For telling uploads apart (re-uploads under one release key): how long
|
||||
* one upload takes to post (the span of a release's files' dates, in
|
||||
* releases without a file name twice), and how far apart the copies of a
|
||||
* file name are in releases that have one twice.
|
||||
*/
|
||||
export default class ReleasesUploadGaps extends BaseCommand {
|
||||
static commandName = 'releases:upload-gaps'
|
||||
static description = 'Posting spans of uploads and gaps between re-uploads, in a sample'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
|
||||
declare sample: number
|
||||
|
||||
async run() {
|
||||
const ids: string[] = (await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])).rows.map((r: { id: string }) => r.id)
|
||||
const bins = [10, 60, 180, 360, 720, 1440, 4320, Number.POSITIVE_INFINITY]
|
||||
const label = (m: number) => (m === Number.POSITIVE_INFINITY ? '>3d' : m >= 1440 ? `≤${m / 1440}d` : m >= 60 ? `≤${m / 60}h` : `≤${m}min`)
|
||||
const spans = new Array(bins.length).fill(0)
|
||||
const gaps = new Array(bins.length).fill(0)
|
||||
const bin = (minutes: number) => bins.findIndex((b) => minutes <= b)
|
||||
for (const id of ids) {
|
||||
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
||||
const byName = new Map<string, number[]>()
|
||||
const times: number[] = []
|
||||
for (const c of chunks) {
|
||||
for (const f of decodeChunk(c.payload)) {
|
||||
if (f.postedAt === null) continue
|
||||
times.push(f.postedAt)
|
||||
byName.set(f.name, [...(byName.get(f.name) ?? []), f.postedAt])
|
||||
}
|
||||
}
|
||||
if (times.length < 2) continue
|
||||
const twice = [...byName.values()].filter((t) => t.length > 1)
|
||||
if (twice.length === 0) {
|
||||
spans[bin((Math.max(...times) - Math.min(...times)) / 60_000)]++
|
||||
continue
|
||||
}
|
||||
// The gap between a name's copies (sorted; each step), so a release reposted 3 times counts 2 gaps per name.
|
||||
for (const t of twice) {
|
||||
t.sort((a, b) => a - b)
|
||||
for (let i = 1; i < t.length; i++) gaps[bin((t[i] - t[i - 1]) / 60_000)]++
|
||||
}
|
||||
}
|
||||
const show = (counts: number[]) => {
|
||||
const total = counts.reduce((a, b) => a + b, 0)
|
||||
let cumulative = 0
|
||||
return bins.map((b, i) => `${label(b)} ${((100 * (cumulative += counts[i])) / Math.max(1, total)).toFixed(1)}%`).join(', ') + ` (n=${total})`
|
||||
}
|
||||
this.logger.info(`one upload's posting span (cumulative): ${show(spans)}`)
|
||||
this.logger.info(`gap between copies of a name (cumulative): ${show(gaps)}`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user