diff --git a/.dockerignore b/.dockerignore index e680f30..3c9fa69 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,4 @@ build tmp .env .git -tools/edge-bench-rs/target +edge/target diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 6c8ad7b..627f112 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -66,8 +66,28 @@ jobs: - name: Test run: node ace test + # The edge worker (edge/, Rust): its tests, and the Rust and TypeScript + # RangeIndexers compared on the committed captures (edge/fixtures). + edge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - name: Rust toolchain + run: | + command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: cargo test + run: cargo test --release --manifest-path edge/Cargo.toml + - name: Rust and TypeScript identical on the fixtures + run: edge/check-fixtures.sh + deploy: - needs: test + needs: [test, edge] if: github.ref == 'refs/heads/rebuild' runs-on: ubuntu-latest steps: diff --git a/Dockerfile b/Dockerfile index 7448349..e3e5859 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # # Voyager 2's images (deploy/compose.yaml): # --target app the AdonisJS app: web server, coordinator, migrations, ace commands -# --target edge the Rust edge worker (tools/edge-bench-rs, bin edge-node) +# --target edge the Rust edge worker (edge/, bin edge-node) # Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to # validate commands) throws "Invalid URL" on $ref, so no ace command starts. @@ -29,7 +29,7 @@ CMD ["node", "ace", "list"] # The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed. FROM rust:1-bookworm AS edge-build WORKDIR /src -COPY tools/edge-bench-rs ./ +COPY edge ./ RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/src/target \ cargo build --release --locked --bin edge-node \ diff --git a/REBUILD.md b/REBUILD.md index 6415663..0bf6c1f 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -119,7 +119,7 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal: 2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`, 2026-10-09, after two rounds of work): ~200–215 k headers/s per core on recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak - per process. **Rust** (`tools/edge-bench-rs`, the same RangeIndexer path + per process. **Rust** (`edge/`, the same RangeIndexer path over the same captures; `check.ts` finds every output identical, the decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1 CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory @@ -130,7 +130,7 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal: cost: a second implementation of lib/headers, lib/release, lib/rules and the chunk format, kept identical (check.ts) for as long as both exist. Findings, how to run both and how to compare their outputs: - `tools/edge-bench-rs/README.md`. Since 2026-10-09 the Rust worker + `edge/README.md`. Since 2026-10-09 the Rust worker (`edge-node`, same crate) is the one running on homelab: ~8× less CPU per header and 20× less memory than `edge:node` in production. Per lease: compressed chunks of 10–50 k lines, in order → diff --git a/tools/edge-bench-rs/.gitignore b/edge/.gitignore similarity index 100% rename from tools/edge-bench-rs/.gitignore rename to edge/.gitignore diff --git a/tools/edge-bench-rs/Cargo.lock b/edge/Cargo.lock similarity index 99% rename from tools/edge-bench-rs/Cargo.lock rename to edge/Cargo.lock index 93df9e0..d56c87d 100644 --- a/tools/edge-bench-rs/Cargo.lock +++ b/edge/Cargo.lock @@ -141,7 +141,7 @@ dependencies = [ ] [[package]] -name = "edge-bench-rs" +name = "edge" version = "0.1.0" dependencies = [ "base64 0.22.1", diff --git a/tools/edge-bench-rs/Cargo.toml b/edge/Cargo.toml similarity index 63% rename from tools/edge-bench-rs/Cargo.toml rename to edge/Cargo.toml index ea31165..54c3e53 100644 --- a/tools/edge-bench-rs/Cargo.toml +++ b/edge/Cargo.toml @@ -1,8 +1,16 @@ [package] -name = "edge-bench-rs" +name = "edge" version = "0.1.0" edition = "2021" -description = "The edge worker's indexing (lib/edge RangeIndexer) in Rust, to compare with edge:bench on captured ranges" +description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)" + +[[bin]] +name = "edge-bench" +path = "src/main.rs" + +[[bin]] +name = "edge-node" +path = "src/bin/edge-node.rs" [dependencies] base64 = "0.22" diff --git a/tools/edge-bench-rs/README.md b/edge/README.md similarity index 81% rename from tools/edge-bench-rs/README.md rename to edge/README.md index 8ec00c5..06e4de2 100644 --- a/tools/edge-bench-rs/README.md +++ b/edge/README.md @@ -1,36 +1,40 @@ -# edge-bench-rs: the edge worker's indexing in Rust +# edge: voyager 2's edge worker (Rust) -Since 2026-10-09 this crate is also the production worker: `edge-node` -(`src/bin/edge-node.rs`) leases ranges from the coordinator and posts -batches like `node ace edge:node`, with the same indexing as the -benchmark (`src/lib.rs`). On homelab it indexed 122M headers in 10 -minutes with 0.37 cores and 565 MB, next to the TypeScript workers' 24M -with 0.61 cores and 11.3 GB (8 threads vs 8 processes): about 8× less -CPU per header and 20× less memory. See "The worker" below. +The product: `edge-node` (`src/bin/edge-node.rs`) is the worker that runs +on homelab (compose service `edge`, image target `edge`). It leases ranges +from the node API, reads their prefetched XOVER chunks, finds releases and +posts batches. `edge-bench` (`src/main.rs`) replays a captured range through +the same indexing (`src/lib.rs`) for timing and for comparing. -A measurement, not a product. It answers whether a Rust edge worker would be -worth a second implementation of the TypeScript one (`edge:node`, REBUILD.md -part 2). It ports the path that `edge:bench` replays over a captured range: +The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the +reference: `check.ts` runs both on a capture and compares every output field +by field. An identity or rule change goes into both and has to come out +identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test` +and `check-fixtures.sh`, which compares both on the committed captures in +`fixtures/` (25k overview lines each of a.b.models, a.b.ath and +a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they +match. -- inflate each XOVER chunk; -- scan the overview lines (no object per line); -- apply the owner rule from the Xref; -- build segments (file key, release identity); -- aggregate segments into files, and cut off idle ones as fragments, - per-article or incomplete files; -- every N chunks, judge the completed releases (rules: executables, - readable names, garbage tools and signatures, per-article random - subjects), encode their chunks (format v2) and pick probe candidates. +```sh +cargo test --release --manifest-path edge/Cargo.toml +edge/check-fixtures.sh # from the repository root +``` -It does not port NNTP, the coordinator API, batch posting or anything on the -server side. +In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores +and 565 MB, next to the TypeScript workers' 24M with 0.61 cores and 11.3 GB +(8 threads vs 8 processes): about 8× less CPU per header and 20× less +memory. See "The worker" below. + +It started as a measurement (moved from `tools/edge-bench-rs` on +2026-10-11): whether a Rust worker would be worth a second implementation. +The findings below are from then. ## Findings (2026-10-09) Same machine (the workstation, otherwise idle), same captures, 2M headers each: -| Capture | TypeScript (`edge:bench`) | Rust (`edge-bench-rs`) | Less CPU | +| Capture | TypeScript (`edge:bench`) | Rust (`edge-bench`) | Less CPU | |---|---|---|---| | a.b.boneless 2026 (160950000000–160951999999) | 11.1 CPU-s (~215k headers/s), 0.70 GB peak | 2.9 CPU-s (~690k/s), 0.24 GB | 3.8× | | a.b.boneless 2026 (160952000000–160953999999) | 11.8 CPU-s (~205k/s), 0.70 GB | 3.1 CPU-s (~640k/s), 0.24 GB | 3.8× | @@ -92,8 +96,8 @@ Benchmark both on it: ```sh node ace edge:bench --dir /path/to/capture # TypeScript: headers/s, CPU, inflate/index, peak RSS -cd tools/edge-bench-rs && cargo build --release -./target/release/edge-bench-rs /path/to/capture # Rust: the same summary +cd edge && cargo build --release +./target/release/edge-bench /path/to/capture # Rust: the same summary ``` Both take results every 10 chunks (`--batch-chunks`) and give a file up after @@ -109,8 +113,8 @@ themselves. ## Comparing the output ```sh -./target/release/edge-bench-rs /path/to/capture --dump /tmp/out.jsonl -cd ../.. && node --import=@poppinss/ts-exec tools/edge-bench-rs/check.ts /path/to/capture /tmp/out.jsonl +./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl +cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl ``` `--dump` writes every output of every batch as JSON lines: @@ -223,12 +227,12 @@ COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-nod node's download cap, shared by its threads. Plus a heartbeat when no batch went out for a minute. - **Wire format:** `src/wire.rs` writes a batch as `toWire` does. - `edge-bench-rs --wire` and `check.ts --wire` compare it with the + `edge-bench --wire` and `check.ts --wire` compare it with the TypeScript's, field by field. - **End to end:** both workers indexed the a.b.models capture (spooled as a range) through the web server; what the server stored was the same (releases with every column, chunks, candidates, rule counts, fragments with their JSON). -- **On homelab:** container `usenet-rebuild-edge-rs` runs - `/mnt/bulk/usenet-rebuild/bin/edge-node --threads 8` in - node:22-bookworm-slim (the binary needs glibc 2.34+). +- **On homelab:** compose service `edge` (deploy/compose.yaml), the + `edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8); + it talks to `node-api`. diff --git a/edge/check-fixtures.sh b/edge/check-fixtures.sh new file mode 100755 index 0000000..5bf4193 --- /dev/null +++ b/edge/check-fixtures.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k +# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared +# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from +# the repository root after any change to either side. +set -eu +cd "$(dirname "$0")/.." +cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +status=0 +for capture in edge/fixtures/*/; do + name=$(basename "$capture") + edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1 + echo "== $name" + node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1 +done +exit $status diff --git a/tools/edge-bench-rs/check.ts b/edge/check.ts similarity index 96% rename from tools/edge-bench-rs/check.ts rename to edge/check.ts index 45fb705..8919a5f 100644 --- a/tools/edge-bench-rs/check.ts +++ b/edge/check.ts @@ -1,12 +1,12 @@ /** - * Compares edge-bench-rs's output (--dump) with the TypeScript RangeIndexer + * Compares edge-bench's output (--dump) with the TypeScript RangeIndexer * on the same captured range, item by item: releases (every field, and the * payload decoded with lib/chunks), samples, probe candidates, rule counts, * fragments and batch boundaries. * - * node --import=@poppinss/ts-exec tools/edge-bench-rs/check.ts [--wire batches.jsonl] [--batch-chunks 10] [--idle 5] + * node --import=@poppinss/ts-exec edge/check.ts [--wire batches.jsonl] [--batch-chunks 10] [--idle 5] * - * With --wire (edge-bench-rs --wire): also each batch as the Rust worker + * With --wire (edge-bench --wire): also each batch as the Rust worker * posts it, against toWire of the TypeScript batch, field by field. */ import { readFile, readdir } from 'node:fs/promises' diff --git a/edge/fixtures/ath-2013/0000-2549992990-2549997989.gz b/edge/fixtures/ath-2013/0000-2549992990-2549997989.gz new file mode 100644 index 0000000..2642e51 Binary files /dev/null and b/edge/fixtures/ath-2013/0000-2549992990-2549997989.gz differ diff --git a/edge/fixtures/ath-2013/0001-2549997990-2550002989.gz b/edge/fixtures/ath-2013/0001-2549997990-2550002989.gz new file mode 100644 index 0000000..011dcbf Binary files /dev/null and b/edge/fixtures/ath-2013/0001-2549997990-2550002989.gz differ diff --git a/edge/fixtures/ath-2013/0002-2550002990-2550007989.gz b/edge/fixtures/ath-2013/0002-2550002990-2550007989.gz new file mode 100644 index 0000000..4ebb1e5 Binary files /dev/null and b/edge/fixtures/ath-2013/0002-2550002990-2550007989.gz differ diff --git a/edge/fixtures/ath-2013/0003-2550007990-2550012989.gz b/edge/fixtures/ath-2013/0003-2550007990-2550012989.gz new file mode 100644 index 0000000..e583dfa Binary files /dev/null and b/edge/fixtures/ath-2013/0003-2550007990-2550012989.gz differ diff --git a/edge/fixtures/ath-2013/0004-2550012990-2550017989.gz b/edge/fixtures/ath-2013/0004-2550012990-2550017989.gz new file mode 100644 index 0000000..2e01d96 Binary files /dev/null and b/edge/fixtures/ath-2013/0004-2550012990-2550017989.gz differ diff --git a/edge/fixtures/ath-2013/range.json b/edge/fixtures/ath-2013/range.json new file mode 100644 index 0000000..0a4290d --- /dev/null +++ b/edge/fixtures/ath-2013/range.json @@ -0,0 +1 @@ +{"group":"alt.binaries.ath","start":2549992990,"end":2550017989} \ No newline at end of file diff --git a/edge/fixtures/boneless-2026/0000-160950000000-160950004999.gz b/edge/fixtures/boneless-2026/0000-160950000000-160950004999.gz new file mode 100644 index 0000000..e0bd3c7 Binary files /dev/null and b/edge/fixtures/boneless-2026/0000-160950000000-160950004999.gz differ diff --git a/edge/fixtures/boneless-2026/0001-160950005000-160950009999.gz b/edge/fixtures/boneless-2026/0001-160950005000-160950009999.gz new file mode 100644 index 0000000..77a4e9c Binary files /dev/null and b/edge/fixtures/boneless-2026/0001-160950005000-160950009999.gz differ diff --git a/edge/fixtures/boneless-2026/0002-160950010000-160950014999.gz b/edge/fixtures/boneless-2026/0002-160950010000-160950014999.gz new file mode 100644 index 0000000..eb511cc Binary files /dev/null and b/edge/fixtures/boneless-2026/0002-160950010000-160950014999.gz differ diff --git a/edge/fixtures/boneless-2026/0003-160950015000-160950019999.gz b/edge/fixtures/boneless-2026/0003-160950015000-160950019999.gz new file mode 100644 index 0000000..4dad040 Binary files /dev/null and b/edge/fixtures/boneless-2026/0003-160950015000-160950019999.gz differ diff --git a/edge/fixtures/boneless-2026/0004-160950020000-160950024999.gz b/edge/fixtures/boneless-2026/0004-160950020000-160950024999.gz new file mode 100644 index 0000000..aa9d78d Binary files /dev/null and b/edge/fixtures/boneless-2026/0004-160950020000-160950024999.gz differ diff --git a/edge/fixtures/boneless-2026/range.json b/edge/fixtures/boneless-2026/range.json new file mode 100644 index 0000000..0945214 --- /dev/null +++ b/edge/fixtures/boneless-2026/range.json @@ -0,0 +1 @@ +{"group":"alt.binaries.boneless","start":160950000000,"end":160950024999} \ No newline at end of file diff --git a/edge/fixtures/models-a/0000-2017215-2022214.gz b/edge/fixtures/models-a/0000-2017215-2022214.gz new file mode 100644 index 0000000..5a912d2 Binary files /dev/null and b/edge/fixtures/models-a/0000-2017215-2022214.gz differ diff --git a/edge/fixtures/models-a/0001-2022215-2027214.gz b/edge/fixtures/models-a/0001-2022215-2027214.gz new file mode 100644 index 0000000..66a20e3 Binary files /dev/null and b/edge/fixtures/models-a/0001-2022215-2027214.gz differ diff --git a/edge/fixtures/models-a/0002-2027215-2032214.gz b/edge/fixtures/models-a/0002-2027215-2032214.gz new file mode 100644 index 0000000..64bb98c Binary files /dev/null and b/edge/fixtures/models-a/0002-2027215-2032214.gz differ diff --git a/edge/fixtures/models-a/0003-2032215-2037214.gz b/edge/fixtures/models-a/0003-2032215-2037214.gz new file mode 100644 index 0000000..60fa2d3 Binary files /dev/null and b/edge/fixtures/models-a/0003-2032215-2037214.gz differ diff --git a/edge/fixtures/models-a/0004-2037215-2042214.gz b/edge/fixtures/models-a/0004-2037215-2042214.gz new file mode 100644 index 0000000..7b842ce Binary files /dev/null and b/edge/fixtures/models-a/0004-2037215-2042214.gz differ diff --git a/edge/fixtures/models-a/range.json b/edge/fixtures/models-a/range.json new file mode 100644 index 0000000..d796b7a --- /dev/null +++ b/edge/fixtures/models-a/range.json @@ -0,0 +1 @@ +{"group":"alt.binaries.models","start":2017215,"end":2042214} \ No newline at end of file diff --git a/tools/edge-bench-rs/src/bin/edge-node.rs b/edge/src/bin/edge-node.rs similarity index 99% rename from tools/edge-bench-rs/src/bin/edge-node.rs rename to edge/src/bin/edge-node.rs index 5c7b3b0..915ac7a 100644 --- a/tools/edge-bench-rs/src/bin/edge-node.rs +++ b/edge/src/bin/edge-node.rs @@ -1,6 +1,6 @@ //! The edge worker in Rust (REBUILD.md, part 2; commands/edge_node.ts): leases //! ranges from the coordinator, reads their XOVER chunks from its stream, -//! indexes them (the same RangeIndexer as edge-bench-rs, kept identical to +//! indexes them (the same RangeIndexer as edge-bench, kept identical to //! the TypeScript by check.ts) and posts the batches. One range per thread. //! //! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M] @@ -8,9 +8,9 @@ //! SIGTERM/SIGINT: each thread gives its range back after the current chunk //! (the next worker resumes at the last stored batch). -use edge_bench_rs::indexer::{Batch, RangeIndexer}; -use edge_bench_rs::names::Rules; -use edge_bench_rs::wire; +use edge::indexer::{Batch, RangeIndexer}; +use edge::names::Rules; +use edge::wire; use flate2::write::GzEncoder; use serde::Deserialize; use std::collections::HashMap; diff --git a/tools/edge-bench-rs/src/codec.rs b/edge/src/codec.rs similarity index 100% rename from tools/edge-bench-rs/src/codec.rs rename to edge/src/codec.rs diff --git a/tools/edge-bench-rs/src/indexer.rs b/edge/src/indexer.rs similarity index 100% rename from tools/edge-bench-rs/src/indexer.rs rename to edge/src/indexer.rs diff --git a/tools/edge-bench-rs/src/js.rs b/edge/src/js.rs similarity index 100% rename from tools/edge-bench-rs/src/js.rs rename to edge/src/js.rs diff --git a/tools/edge-bench-rs/src/lib.rs b/edge/src/lib.rs similarity index 96% rename from tools/edge-bench-rs/src/lib.rs rename to edge/src/lib.rs index 3b8ca99..540a70a 100644 --- a/tools/edge-bench-rs/src/lib.rs +++ b/edge/src/lib.rs @@ -1,5 +1,5 @@ //! The edge worker's indexing (lib/edge RangeIndexer and what it uses) in -//! Rust, kept identical to the TypeScript (check.ts): the edge-bench-rs +//! Rust, kept identical to the TypeScript (check.ts): the edge-bench //! benchmark and the edge-node worker share it. pub mod codec; diff --git a/tools/edge-bench-rs/src/main.rs b/edge/src/main.rs similarity index 95% rename from tools/edge-bench-rs/src/main.rs rename to edge/src/main.rs index f7f8264..57fb5c7 100644 --- a/tools/edge-bench-rs/src/main.rs +++ b/edge/src/main.rs @@ -2,14 +2,14 @@ //! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as //! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off //! idle ones, and every N chunks judge and encode the releases. `--dump` -//! writes every output as JSON lines for tools/edge-bench-rs/check.ts. +//! writes every output as JSON lines for edge/check.ts. //! -//! edge-bench-rs [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl] +//! edge-bench [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl] use base64::Engine; -use edge_bench_rs::indexer::{self, Batch, RangeIndexer}; -use edge_bench_rs::js::json_str; -use edge_bench_rs::{names, wire}; +use edge::indexer::{self, Batch, RangeIndexer}; +use edge::js::json_str; +use edge::{names, wire}; use rustc_hash::FxHashMap; use std::io::Write; use std::time::Instant; @@ -150,7 +150,7 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) { fn main() { let args: Vec = std::env::args().collect(); - let dir = args.get(1).expect("usage: edge-bench-rs [--batch-chunks N] [--idle N] [--dump file]").clone(); + let dir = args.get(1).expect("usage: edge-bench [--batch-chunks N] [--idle N] [--dump file]").clone(); let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned(); let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10); let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5); diff --git a/tools/edge-bench-rs/src/names.rs b/edge/src/names.rs similarity index 98% rename from tools/edge-bench-rs/src/names.rs rename to edge/src/names.rs index ad2e411..db00429 100644 --- a/tools/edge-bench-rs/src/names.rs +++ b/edge/src/names.rs @@ -591,6 +591,11 @@ pub fn looks_random(name: &str) -> bool { if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 { return true; } + // 51294P1CUKR2R35X: capitals and digits only, switching between them all over the token + // (COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice). + if b.len() >= 12 && b.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) && b.windows(2).filter(|w| w[0].is_ascii_digit() != w[1].is_ascii_digit()).count() >= 6 { + return true; + } if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 { return false; } @@ -985,6 +990,9 @@ mod tests { fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4")); assert!(looks_random("RmISakytFwXSCEzjVEpaKk")); + assert!(looks_random("51294P1CUKR2R35X")); + assert!(!looks_random("COD4MW2REMASTERED")); + assert!(!looks_random("BEATLES1964REMASTER")); assert!(!looks_random("Some.Show.S01E01.1080p")); assert!(looks_random("VIDEO_TS")); } diff --git a/tools/edge-bench-rs/src/wire.rs b/edge/src/wire.rs similarity index 100% rename from tools/edge-bench-rs/src/wire.rs rename to edge/src/wire.rs diff --git a/lib/release/random.ts b/lib/release/random.ts index 1f9318d..68f0ef0 100644 --- a/lib/release/random.ts +++ b/lib/release/random.ts @@ -5,6 +5,17 @@ import { isGenericName } from './identity.js' * separators, and the other generated shapes seen on Usenet (LOG.md * 2026-10-01 (3)). Kept in step with SQL's is_readable_name(). */ +/** How often a token goes from a letter to a digit or back. */ +function letterDigitSwitches(token: string): number { + let switches = 0 + for (let i = 1; i < token.length; i++) { + const a = token.charCodeAt(i - 1) <= 57 + const b = token.charCodeAt(i) <= 57 + if (a !== b) switches++ + } + return switches +} + export function looksRandom(name: string): boolean { // An unknown short extension (e.g. ".bin") says nothing about the name. name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') @@ -21,6 +32,9 @@ export function looksRandom(name: string): boolean { if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true + // 51294P1CUKR2R35X: capitals and digits only, switching between them all over the token + // (COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice). + if (/^[A-Z0-9]{12,}$/.test(name) && letterDigitSwitches(name) >= 6) return true if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts index d205877..7d09724 100644 --- a/tests/unit/release/identity.spec.ts +++ b/tests/unit/release/identity.spec.ts @@ -78,6 +78,15 @@ test.group('release / releaseIdentity', () => { assert.equal(a.key, b.key) }) + test('capitals and digits switching all over a token are random', async ({ assert }) => { + const { looksRandom } = await import('#lib/release/index') + assert.isTrue(looksRandom('51294P1CUKR2R35X')) + assert.isTrue(looksRandom('51294P1CUKR2R35X.part01.rar'.replace(/\.part01\.rar$/, ''))) + assert.isFalse(looksRandom('COD4MW2REMASTERED')) + assert.isFalse(looksRandom('BEATLES1964REMASTER')) + assert.isFalse(looksRandom('HDTV1080P')) + }) + test('random extensionless names cannot be grouped by name', ({ assert }) => { const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231' assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true }))