implemented a lot of features

lib/split/index.ts

@@ -0,0 +1,4 @@
+/**
+ * Separating equal-sized obfuscated volumes by posting order, independent of AdonisJS.
+ */
+export { splitInstances, splitByRank, splitVerified, verifyVolume, type SplitArticle, type SplitOptions } from './instances.js'

lib/split/instances.ts

@@ -0,0 +1,178 @@
+import { combineCrc32 } from '../yenc/decode.js'
+
+export interface SplitArticle {
+  number: number
+  messageId: string
+  bytes: number
+  /** Article number in the group; posting order. */
+  articleNumber: number | null
+  /** yEnc part CRC and decoded part length, for verification. */
+  pcrc32?: string | null
+  partSize?: number | null
+  /** Whole-file CRC32, usually only in the last part. */
+  fileCrc32?: string | null
+}
+
+export interface SplitOptions {
+  /** Total parts per volume; a volume that has them all takes no more articles. */
+  totalParts: number
+  /**
+   * Maximum distance in article numbers between consecutive articles of one
+   * volume. Separates different posts that happen to share a size.
+   */
+  window?: number
+  /**
+   * How far (in posting positions) an article may be from where its volume
+   * predicts it; covers out-of-order finishing across connections.
+   */
+  tolerance?: number
+}
+
+interface Volume {
+  articles: SplitArticle[]
+  /** Posting rank of each part number present. */
+  ranks: Map<number, number>
+  lastArticle: number
+  opened: number
+}
+
+/** ~1 minute of posting in a busy group. */
+const DEFAULT_WINDOW = 20_000
+
+/**
+ * Separates the articles of several equal-sized files that share one
+ * identity (obfuscated volumes with random names, possibly from different
+ * posts) into one list per file. See CONCEPT.md (D20).
+ *
+ * Posting tools upload a queue (volume 1 parts 1..n, volume 2 parts 1..n,
+ * ...) over several connections, so within a volume part k+1 lands about one
+ * position after part k in posting order, give or take the connection count.
+ * Each article, in posting order, goes to the eligible volume (lacks this
+ * part, not complete, active within `window` article numbers) whose
+ * predicted position for it is closest: taken from that volume's article with
+ * the nearest part number, which keeps the prediction local and unaffected
+ * by missing articles elsewhere. A deviation above the tolerance starts a new
+ * volume. The result is recomputed from all articles, so it improves as late
+ * articles arrive.
+ */
+export function splitInstances(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
+  const window = options.window ?? DEFAULT_WINDOW
+  const tolerance = options.tolerance ?? Math.max(16, Math.ceil(options.totalParts * 0.1))
+  const seen = new Set<string>()
+  const ordered = articles
+    .filter((a) => (seen.has(a.messageId) ? false : (seen.add(a.messageId), true)))
+    .sort(
+      (a, b) =>
+        (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
+        a.messageId.localeCompare(b.messageId)
+    )
+
+  const volumes: Volume[] = []
+  ordered.forEach((article, rank) => {
+    const position = article.articleNumber ?? Number.MAX_SAFE_INTEGER
+    let best: Volume | undefined
+    let bestScore: [number, number] | undefined
+    for (const volume of volumes) {
+      if (volume.ranks.has(article.number) || volume.ranks.size >= options.totalParts) continue
+      if (position - volume.lastArticle > window) continue
+      const deviation = Math.abs(rank - predictedRank(volume, article.number))
+      if (deviation > tolerance) continue
+      const score: [number, number] = [deviation, volume.opened]
+      if (!bestScore || score[0] < bestScore[0] || (score[0] === bestScore[0] && score[1] < bestScore[1])) {
+        best = volume
+        bestScore = score
+      }
+    }
+    if (!best) {
+      best = { articles: [], ranks: new Map(), lastArticle: position, opened: volumes.length }
+      volumes.push(best)
+    }
+    best.articles.push(article)
+    best.ranks.set(article.number, rank)
+    best.lastArticle = Math.max(best.lastArticle, position)
+  })
+
+  return volumes.map((v) => v.articles.sort((a, b) => a.number - b.number))
+}
+
+/** Where part `number` should be in posting order, judged from the volume's nearest part. */
+function predictedRank(volume: Volume, number: number): number {
+  let anchorPart = 0
+  let anchorRank = 0
+  let nearest = Number.MAX_SAFE_INTEGER
+  for (const [part, rank] of volume.ranks) {
+    const distance = Math.abs(part - number)
+    if (distance < nearest) {
+      nearest = distance
+      anchorPart = part
+      anchorRank = rank
+    }
+  }
+  return anchorRank + (number - anchorPart)
+}
+
+/**
+ * Simpler split: for every part number, the i-th article in posting order
+ * belongs to volume i. Right for a single, complete post; used as a second
+ * proposal for verification.
+ */
+export function splitByRank(articles: SplitArticle[]): SplitArticle[][] {
+  const byNumber = new Map<number, SplitArticle[]>()
+  const seen = new Set<string>()
+  for (const article of articles) {
+    if (seen.has(article.messageId)) continue
+    seen.add(article.messageId)
+    byNumber.set(article.number, [...(byNumber.get(article.number) ?? []), article])
+  }
+  const volumes: SplitArticle[][] = []
+  for (const group of byNumber.values()) {
+    group.sort(
+      (a, b) =>
+        (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
+        a.messageId.localeCompare(b.messageId)
+    )
+    group.forEach((article, i) => (volumes[i] ??= []).push(article))
+  }
+  return volumes.map((v) => v.sort((a, b) => a.number - b.number))
+}
+
+/**
+ * True when a volume has every part and its combined part CRCs equal the
+ * whole-file CRC32 its last part announces: then it is exactly one file.
+ */
+export function verifyVolume(volume: SplitArticle[], totalParts: number): boolean {
+  if (volume.length !== totalParts) return false
+  const sorted = [...volume].sort((a, b) => a.number - b.number)
+  if (sorted.some((a, i) => a.number !== i + 1 || !a.pcrc32 || !a.partSize)) return false
+  const expected = sorted[sorted.length - 1].fileCrc32
+  if (!expected) return false
+  const combined = combineCrc32(sorted.map((a) => ({ crc: a.pcrc32!, length: a.partSize! })))
+  return combined === expected.toLowerCase().padStart(8, '0')
+}
+
+/**
+ * Splits and keeps only volumes that verify (verifyVolume). Proposals from
+ * both methods are tried; articles of verified volumes are taken out and the
+ * rest is tried again, until nothing new verifies. Unverified articles are
+ * left out: guessing wrong would produce a corrupt file.
+ */
+export function splitVerified(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
+  const verified: SplitArticle[][] = []
+  let remaining = articles
+  for (let round = 0; round < 4 && remaining.length >= options.totalParts; round++) {
+    const found: SplitArticle[][] = []
+    const used = new Set<string>()
+    for (const propose of [(a: SplitArticle[]) => splitInstances(a, options), splitByRank]) {
+      for (const volume of propose(remaining.filter((a) => !used.has(a.messageId)))) {
+        if (verifyVolume(volume, options.totalParts)) {
+          found.push(volume)
+          for (const article of volume) used.add(article.messageId)
+        }
+      }
+    }
+    if (found.length === 0) break
+    verified.push(...found)
+    remaining = remaining.filter((a) => !used.has(a.messageId))
+  }
+  return verified
+}