From 3737ea15424ddab7d40b46aef0ea363b04b5e202 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 13:31:55 +0200 Subject: [PATCH] Rebuild: tools to rank groups by real span (GROUP's low) and sample how full they are Co-Authored-By: Claude Opus 5.5 --- .adonisjs/server/routes.d.ts | 11 ++++++++++ tools/group_fill.ts | 42 ++++++++++++++++++++++++++++++++++++ tools/group_sizes.ts | 36 +++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 tools/group_fill.ts create mode 100644 tools/group_sizes.ts diff --git a/.adonisjs/server/routes.d.ts b/.adonisjs/server/routes.d.ts index b676ade..df73287 100644 --- a/.adonisjs/server/routes.d.ts +++ b/.adonisjs/server/routes.d.ts @@ -4,29 +4,39 @@ type ParamValue = string | number | bigint | boolean export type ScannedRoutes = { ALL: { + 'newznab': { paramsTuple?: []; params?: {} } 'auth.new_account.store': { paramsTuple?: []; params?: {} } 'auth.access_tokens.store': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } 'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} } 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'node.node.lease': { paramsTuple?: []; params?: {} } 'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } } GET: { + 'newznab': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } } HEAD: { + 'newznab': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } } POST: { @@ -38,6 +48,7 @@ export type ScannedRoutes = { 'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } } } declare module '@adonisjs/core/types/http' { diff --git a/tools/group_fill.ts b/tools/group_fill.ts new file mode 100644 index 0000000..f464385 --- /dev/null +++ b/tools/group_fill.ts @@ -0,0 +1,42 @@ +/** + * How full a list of groups' spans are: for a sample of groups (from + * group_sizes.ts's TSV), XOVER windows of `window` articles at evenly spaced + * points; articles found / asked, weighted by span. + * + * node --env-file=.env --import=@poppinss/ts-exec tools/group_fill.ts groups.tsv [groups] [points] [window] + */ +import { readFileSync } from 'node:fs' +import { XoverClient } from '#lib/nntp/index' + +const [file, fromRank, toRank] = process.argv.slice(2, 5) +const sampleSize = Number(process.argv[5] ?? 25) +const points = Number(process.argv[6] ?? 5) +const window = Number(process.argv[7] ?? 1000) +const rows = readFileSync(file, 'utf8').trim().split('\n').map((l) => l.split('\t')).filter((r) => Number(r[0]) >= Number(fromRank) && Number(r[0]) <= Number(toRank)) +const step = rows.length / sampleSize +const sample = Array.from({ length: sampleSize }, (_, i) => rows[Math.floor(i * step)]) +const env = process.env +const tasks = sample.flatMap(([, name, low, high]) => Array.from({ length: points }, (_, p) => ({ name, at: Math.floor(Number(low) + ((Number(high) - Number(low)) * (p + 0.5)) / points) }))) +const found = new Map() +await Promise.all( + Array.from({ length: 4 }, async () => { + const client = await XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress: true, timeoutMs: 60_000 }) + let current = '' + while (tasks.length) { + const t = tasks.shift()! + if (t.name !== current) await client.group((current = t.name)) + const n = (await client.xover(t.at, t.at + window - 1)).length + found.set(t.name, (found.get(t.name) ?? 0) + n) + } + client.close() + }) +) +let spanAll = 0 +let existing = 0 +for (const [, name, , , span] of sample) { + const fill = (found.get(name) ?? 0) / (points * window) + spanAll += Number(span) + existing += Number(span) * fill + console.log(`${name}\t${(100 * fill).toFixed(0)}%`) +} +console.log(`weighted fill ${(100 * existing / spanAll).toFixed(1)}% of the span`) diff --git a/tools/group_sizes.ts b/tools/group_sizes.ts new file mode 100644 index 0000000..9e7b2bd --- /dev/null +++ b/tools/group_sizes.ts @@ -0,0 +1,36 @@ +/** + * Every group matching a pattern with its real span: LIST ACTIVE for the + * names (its low is always 2 on Eweka), GROUP per group for the real low and + * count. TSV, largest span first: rank, name, low, high, span, count. + * + * node --env-file=.env --import=@poppinss/ts-exec tools/group_sizes.ts 'alt.binaries.*' [connections] > groups.tsv + */ +import { XoverClient } from '#lib/nntp/index' + +const pattern = process.argv[2] ?? 'alt.binaries.*' +const connections = Number(process.argv[3] ?? 4) +const env = process.env +const connect = () => + XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress: true, timeoutMs: 60_000 }) + +const first = await connect() +const names = (await first.listActive(pattern)).map((g) => g.name) +const queue = [...names] +const out: { name: string; low: number; high: number; count: number }[] = [] +await Promise.all( + Array.from({ length: connections }, async (_, i) => { + const client = i === 0 ? first : await connect() + while (queue.length) { + const name = queue.shift()! + try { + const g = await client.group(name) + out.push({ name, ...g }) + } catch (error: any) { + console.error(`${name}: ${error.message}`) + } + } + client.close() + }) +) +out.sort((a, b) => b.high - b.low - (a.high - a.low)) +out.forEach((g, i) => console.log([i + 1, g.name, g.low, g.high, g.high - g.low, g.count].join('\t')))