Rebuild: tools to rank groups by real span (GROUP's low) and sample how full they are

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.adonisjs/server/routes.d.ts

@@ -4,29 +4,39 @@ type ParamValue = string | number | bigint | boolean
 
 export type ScannedRoutes = {
   ALL: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'auth.new_account.store': { paramsTuple?: []; params?: {} }
     'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.lease': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   GET: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   HEAD: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   POST: {
@@ -38,6 +48,7 @@ export type ScannedRoutes = {
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
 }
 declare module '@adonisjs/core/types/http' {

tools/group_fill.ts

@@ -0,0 +1,42 @@
+/**
+ * How full a list of groups' spans are: for a sample of groups (from
+ * group_sizes.ts's TSV), XOVER windows of `window` articles at evenly spaced
+ * points; articles found / asked, weighted by span.
+ *
+ *   node --env-file=.env --import=@poppinss/ts-exec tools/group_fill.ts groups.tsv <fromRank> <toRank> [groups] [points] [window]
+ */
+import { readFileSync } from 'node:fs'
+import { XoverClient } from '#lib/nntp/index'
+
+const [file, fromRank, toRank] = process.argv.slice(2, 5)
+const sampleSize = Number(process.argv[5] ?? 25)
+const points = Number(process.argv[6] ?? 5)
+const window = Number(process.argv[7] ?? 1000)
+const rows = readFileSync(file, 'utf8').trim().split('\n').map((l) => l.split('\t')).filter((r) => Number(r[0]) >= Number(fromRank) && Number(r[0]) <= Number(toRank))
+const step = rows.length / sampleSize
+const sample = Array.from({ length: sampleSize }, (_, i) => rows[Math.floor(i * step)])
+const env = process.env
+const tasks = sample.flatMap(([, name, low, high]) => Array.from({ length: points }, (_, p) => ({ name, at: Math.floor(Number(low) + ((Number(high) - Number(low)) * (p + 0.5)) / points) })))
+const found = new Map<string, number>()
+await Promise.all(
+  Array.from({ length: 4 }, async () => {
+    const client = await XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress: true, timeoutMs: 60_000 })
+    let current = ''
+    while (tasks.length) {
+      const t = tasks.shift()!
+      if (t.name !== current) await client.group((current = t.name))
+      const n = (await client.xover(t.at, t.at + window - 1)).length
+      found.set(t.name, (found.get(t.name) ?? 0) + n)
+    }
+    client.close()
+  })
+)
+let spanAll = 0
+let existing = 0
+for (const [, name, , , span] of sample) {
+  const fill = (found.get(name) ?? 0) / (points * window)
+  spanAll += Number(span)
+  existing += Number(span) * fill
+  console.log(`${name}\t${(100 * fill).toFixed(0)}%`)
+}
+console.log(`weighted fill ${(100 * existing / spanAll).toFixed(1)}% of the span`)

tools/group_sizes.ts

@@ -0,0 +1,36 @@
+/**
+ * Every group matching a pattern with its real span: LIST ACTIVE for the
+ * names (its low is always 2 on Eweka), GROUP per group for the real low and
+ * count. TSV, largest span first: rank, name, low, high, span, count.
+ *
+ *   node --env-file=.env --import=@poppinss/ts-exec tools/group_sizes.ts 'alt.binaries.*' [connections] > groups.tsv
+ */
+import { XoverClient } from '#lib/nntp/index'
+
+const pattern = process.argv[2] ?? 'alt.binaries.*'
+const connections = Number(process.argv[3] ?? 4)
+const env = process.env
+const connect = () =>
+  XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress: true, timeoutMs: 60_000 })
+
+const first = await connect()
+const names = (await first.listActive(pattern)).map((g) => g.name)
+const queue = [...names]
+const out: { name: string; low: number; high: number; count: number }[] = []
+await Promise.all(
+  Array.from({ length: connections }, async (_, i) => {
+    const client = i === 0 ? first : await connect()
+    while (queue.length) {
+      const name = queue.shift()!
+      try {
+        const g = await client.group(name)
+        out.push({ name, ...g })
+      } catch (error: any) {
+        console.error(`${name}: ${error.message}`)
+      }
+    }
+    client.close()
+  })
+)
+out.sort((a, b) => b.high - b.low - (a.high - a.low))
+out.forEach((g, i) => console.log([i + 1, g.name, g.low, g.high, g.high - g.low, g.count].join('\t')))