diff --git a/tools/edge-bench-rs/.gitignore b/tools/edge-bench-rs/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/tools/edge-bench-rs/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tools/edge-bench-rs/Cargo.lock b/tools/edge-bench-rs/Cargo.lock new file mode 100644 index 0000000..1239213 --- /dev/null +++ b/tools/edge-bench-rs/Cargo.lock @@ -0,0 +1,554 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "edge-bench-rs" +version = "0.1.0" +dependencies = [ + "base64", + "chrono", + "flate2", + "md-5", + "regex", + "rustc-hash", + "sha1", + "zstd", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/tools/edge-bench-rs/Cargo.toml b/tools/edge-bench-rs/Cargo.toml new file mode 100644 index 0000000..ce3c056 --- /dev/null +++ b/tools/edge-bench-rs/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "edge-bench-rs" +version = "0.1.0" +edition = "2021" +description = "The edge worker's indexing (lib/edge RangeIndexer) in Rust, to compare with edge:bench on captured ranges" + +[dependencies] +base64 = "0.22" +chrono = "0.4" +flate2 = { version = "1", default-features = false, features = ["zlib-rs"] } +md-5 = "0.10" +regex = "1" +rustc-hash = "2" +sha1 = "0.10" +zstd = "0.13" + +[profile.release] +lto = true +codegen-units = 1 diff --git a/tools/edge-bench-rs/check.ts b/tools/edge-bench-rs/check.ts new file mode 100644 index 0000000..f4d0187 --- /dev/null +++ b/tools/edge-bench-rs/check.ts @@ -0,0 +1,100 @@ +/** + * Compares edge-bench-rs's output (--dump) with the TypeScript RangeIndexer + * on the same captured range, item by item: releases (every field, and the + * payload decoded with lib/chunks), samples, probe candidates, rule counts, + * fragments and batch boundaries. + * + * node --import=@poppinss/ts-exec tools/edge-bench-rs/check.ts [--batch-chunks 10] [--idle 5] + */ +import { readFile, readdir } from 'node:fs/promises' +import { join } from 'node:path' +import { unzipSync } from 'node:zlib' +import { isDeepStrictEqual } from 'node:util' +import { RangeIndexer, type EdgeBatch } from '#lib/edge/index' +import { DEFAULT_RULES, type Rules } from '#lib/rules/index' +import { decodeChunk } from '#lib/chunks/index' + +const [dir, dumpPath] = process.argv.slice(2) +const flag = (name: string, fallback: number) => { + const i = process.argv.indexOf(name) + return i === -1 ? fallback : Number(process.argv[i + 1]) +} +const batchChunks = flag('--batch-chunks', 10) +const idle = flag('--idle', 5) + +const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8')) +const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] } +const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle }) +const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort() +const batches: EdgeBatch[] = [] +for (const [i, file] of files.entries()) { + const next = Number(file.split('-')[2].replace('.gz', '')) + 1 + indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next) + const last = i === files.length - 1 + if (last) indexer.finish() + if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take()) +} + +/** Canonical lines per kind, in a stable order, from both sides. */ +type Item = Record +const sortFiles = (decoded: ReturnType) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1)) +const payloadBytes = { ts: 0, rs: 0 } +let side: 'ts' | 'rs' = 'ts' +const release = (r: Item, b: number) => { + const { payload, ...rest } = r as Item & { payload: Buffer } + // The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little + // differently); what the payload decodes to is. + payloadBytes[side] += payload.length + return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) } +} +const ts: Record = { b: [], r: [], s: [], c: [], n: [], f: [] } +batches.forEach((batch, b) => { + ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers }) + for (const r of batch.releases) ts.r.push(release({ key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b)) + for (const s of batch.samples) ts.s.push({ b, key: s.key, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments }) + for (const c of batch.candidates) ts.c.push({ b, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file }) + for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files }) + for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments }) +}) + +const rs: Record = { b: [], r: [], s: [], c: [], n: [], f: [] } +side = 'rs' +let b = -1 +for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) { + if (!line) continue + const { t, ...item } = JSON.parse(line) + if (t === 'b') b++ + if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b)) + else if (t === 's') rs.s.push({ b, key: item.key, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments }) + else rs[t].push({ b, ...item }) +} + +const keyOf: Record string> = { + b: (i) => String(i.b), + r: (i) => `${i.b} ${i.key}`, + s: (i) => `${i.b} ${i.key}`, + c: (i) => `${i.b} ${i.releaseKey} ${i.kind} ${i.messageId}`, + n: (i) => `${i.b} ${i.rule} ${i.key}`, + f: (i) => `${i.b} ${i.fileKey}`, +} +const names: Record = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' } +let failed = false +for (const kind of Object.keys(keyOf)) { + const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i])) + const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i])) + const missing = [...a.keys()].filter((k) => !z.has(k)) + const extra = [...z.keys()].filter((k) => !a.has(k)) + const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k))) + const ok = missing.length === 0 && extra.length === 0 && differ.length === 0 + failed ||= !ok + console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`) + for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`) + for (const k of differ.slice(0, 2)) { + const x = a.get(k)! + const y = z.get(k)! + const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f])) + console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`) + } +} +console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`) +process.exit(failed ? 1 : 0) diff --git a/tools/edge-bench-rs/src/codec.rs b/tools/edge-bench-rs/src/codec.rs new file mode 100644 index 0000000..8d16dcb --- /dev/null +++ b/tools/edge-bench-rs/src/codec.rs @@ -0,0 +1,161 @@ +//! lib/chunks/codec.ts encodeChunk, format v2: a version byte, the length of +//! the compressed part, zstd(JSON head + text ids), then the nyuu and hex ids +//! as raw bytes. + +use crate::js::json_str; + +pub struct ChunkFile<'a> { + pub name: &'a str, + pub poster: &'a str, + pub subject: &'a str, + pub posted_at: Option, + pub groups: &'a [String], + /// [part, message-id, bytes] + pub segments: Vec<(u32, &'a str, i64)>, +} + +fn is_nyuu(id: &str, suffix: &str) -> bool { + let b = id.as_bytes(); + if b.len() != 38 + suffix.len() || b[24] != b'-' || !id.ends_with(suffix) { + return false; + } + (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[25..38].iter().all(u8::is_ascii_digit) +} + +fn is_hex(id: &str, suffix: &str) -> bool { + let b = id.as_bytes(); + b.len() == 32 + suffix.len() && id.ends_with(suffix) && b[..32].iter().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(c)) +} + +fn pack_letters(id: &[u8], out: &mut Vec) { + let mut acc: u32 = 0; + let mut bits = 0; + for i in (0..24).step_by(2) { + acc = (acc << 10) | ((id[i] - b'A') as u32 * 26 + (id[i + 1] - b'a') as u32); + bits += 10; + while bits >= 8 { + bits -= 8; + out.push(((acc >> bits) & 0xff) as u8); + } + acc &= (1 << bits) - 1; + } +} + +fn hex_val(c: u8) -> u8 { + if c.is_ascii_digit() { + c - b'0' + } else { + c - b'a' + 10 + } +} + +fn push_num(out: &mut String, n: i64) { + use std::fmt::Write; + write!(out, "{n}").unwrap(); +} + +pub fn encode_chunk(files: &[ChunkFile]) -> Vec { + let mut head = String::with_capacity(256 * files.len()); + let mut text: Vec<&str> = Vec::new(); + let mut raw: Vec = Vec::new(); + head.push('['); + for (fi, f) in files.iter().enumerate() { + let ids = &f.segments; + let code = match ids.first() { + None => 't', + Some(&(_, first, _)) => { + let nyuu = first.get(38..).filter(|s| s.starts_with('@')); + let hex = first.get(32..).filter(|s| s.starts_with('@')); + if nyuu.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_nyuu(id, suffix))) { + 'n' + } else if hex.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_hex(id, suffix))) { + 'h' + } else { + 't' + } + } + }; + if fi > 0 { + head.push(','); + } + head.push_str("{\"n\":"); + json_str(f.name, &mut head); + head.push_str(",\"p\":"); + json_str(f.poster, &mut head); + head.push_str(",\"j\":"); + json_str(f.subject, &mut head); + head.push_str(",\"d\":"); + match f.posted_at { + Some(t) => push_num(&mut head, t), + None => head.push_str("null"), + } + head.push_str(",\"g\":["); + for (i, g) in f.groups.iter().enumerate() { + if i > 0 { + head.push(','); + } + json_str(g, &mut head); + } + head.push_str("],\"s\":["); + for (i, &(_, _, b)) in ids.iter().enumerate() { + if i > 0 { + head.push(','); + } + push_num(&mut head, b); + } + head.push(']'); + if !ids.iter().enumerate().all(|(k, &(n, _, _))| n as usize == k + 1) { + head.push_str(",\"k\":["); + for (i, &(n, _, _)) in ids.iter().enumerate() { + if i > 0 { + head.push(','); + } + push_num(&mut head, n as i64); + } + head.push(']'); + } + match code { + 'n' => { + head.push_str(",\"c\":\"n\",\"x\":"); + json_str(&ids[0].1[38..], &mut head); + head.push_str(",\"t\":["); + let mut previous = 0i64; + for (i, &(_, id, _)) in ids.iter().enumerate() { + pack_letters(id.as_bytes(), &mut raw); + let t: i64 = id[25..38].parse().unwrap(); + if i > 0 { + head.push(','); + } + push_num(&mut head, t - previous); + previous = t; + } + head.push(']'); + } + 'h' => { + head.push_str(",\"c\":\"h\",\"x\":"); + json_str(&ids[0].1[32..], &mut head); + for &(_, id, _) in ids { + let b = id.as_bytes(); + for i in (0..32).step_by(2) { + raw.push(hex_val(b[i]) << 4 | hex_val(b[i + 1])); + } + } + } + _ => text.extend(ids.iter().map(|&(_, id, _)| id)), + } + head.push('}'); + } + head.push(']'); + let mut plain = head.into_bytes(); + for id in text { + plain.push(b'\n'); + plain.extend_from_slice(id.as_bytes()); + } + let packed = zstd::bulk::compress(&plain, 3).unwrap(); + let mut out = Vec::with_capacity(5 + packed.len() + raw.len()); + out.push(2); + out.extend_from_slice(&(packed.len() as u32).to_be_bytes()); + out.extend_from_slice(&packed); + out.extend_from_slice(&raw); + out +} diff --git a/tools/edge-bench-rs/src/indexer.rs b/tools/edge-bench-rs/src/indexer.rs new file mode 100644 index 0000000..c2765dd --- /dev/null +++ b/tools/edge-bench-rs/src/indexer.rs @@ -0,0 +1,601 @@ +//! lib/edge/scan.ts (SegmentScanner, scanOverviewText), lib/files/aggregator.ts +//! and lib/edge/range_indexer.ts (RangeIndexer, chunkOf, candidatesOf), in the +//! same order of operations, so the outputs can be compared one to one. + +use crate::codec::{encode_chunk, ChunkFile}; +use crate::js::{self, sha1_hex}; +use crate::names::{self, is_random_file, JudgedFile, Release, Verdict}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::rc::Rc; + +pub struct FileMeta { + pub file_key: String, + pub name: String, + pub poster: String, + pub subject: String, + pub total_parts: u32, + pub groups: Vec, + pub release: Release, +} + +pub struct Seg { + pub file: Rc, + pub number: u32, + pub message_id: String, + pub bytes: i64, + pub article: i64, + pub posted_at: Option, +} + +pub struct ReleaseChunk { + pub key: String, + pub name: String, + pub source: &'static str, + pub file_total: Option, + pub poster: String, + pub groups: Vec, + pub posted_at: Option, + pub files: usize, + pub segments: usize, + pub bytes: i64, + pub first_article: i64, + pub last_article: i64, + pub readable: bool, + pub has_par2: bool, + pub payload: Vec, +} + +pub struct Candidate { + pub release_key: String, + pub kind: &'static str, + pub message_id: String, + pub file: String, +} + +pub struct Fragment { + pub file_key: String, + pub name: String, + pub total_parts: u32, + pub posted_at: Option, + pub segments: Vec<(u32, String, i64, i64)>, +} + +#[derive(Default)] +pub struct Batch { + pub from: i64, + pub next: i64, + pub headers: u64, + pub releases: Vec, + pub samples: Vec<(ReleaseChunk, &'static str, String)>, + pub candidates: Vec, + /// (rule, key) → (articles, files) + pub counts: Vec<(&'static str, String, u64, u64)>, + pub fragments: Vec, +} + +struct Counter { + index: usize, + length: usize, + segment: i64, + total: i64, +} + +/// The last "(k/N)" after `after`, scanning back from the end (findCounter). +fn find_counter(subject: &[u8], after: usize) -> Option { + let mut open = subject.len(); + loop { + // lastIndexOf('(', open - 1): at most open - 1. + if open == 0 { + return None; + } + open = subject[..open].iter().rposition(|&c| c == b'(')?; + if open <= after { + return None; + } + let mut i = open + 1; + let mut segment: i64 = 0; + let mut digits = 0; + while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) { + segment = segment * 10 + (c - b'0') as i64; + digits += 1; + i += 1; + } + if digits == 0 || subject.get(i) != Some(&b'/') { + continue; + } + i += 1; + let mut total: i64 = 0; + digits = 0; + while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) { + total = total * 10 + (c - b'0') as i64; + digits += 1; + i += 1; + } + if digits == 0 || subject.get(i) != Some(&b')') { + continue; + } + return Some(Counter { index: open, length: i + 1 - open, segment, total }); + } +} + +fn xref_groups(xref: &str, group: &str) -> Vec { + let mut groups = Vec::new(); + for token in js::split_spaces(js::trim(xref)).into_iter().skip(1) { + if let Some(colon) = token.rfind(':') { + if colon > 0 { + groups.push(token[..colon].to_string()); + } + } + } + if groups.is_empty() { + groups.push(group.to_string()); + } + groups +} + +#[derive(Default)] +struct Scanner { + files: FxHashMap>>, + key: String, +} + +impl Scanner { + fn owner<'a>(&self, xref: &'a str, group: &'a str, priorities: &FxHashMap) -> &'a str { + let b = xref.as_bytes(); + let Some(mut at) = xref.find(' ') else { return group }; + let mut owner: Option<&str> = None; + let mut best = u32::MAX; + loop { + while b.get(at) == Some(&b' ') { + at += 1; + } + if at >= b.len() { + break; + } + let end = xref[at..].find(' ').map(|i| i + at); + // lastIndexOf(':', end ?? length): a colon at or before `end`. + let limit = end.unwrap_or(b.len()); + let colon = xref[..limit.min(b.len())].rfind(':'); + if let Some(colon) = colon.filter(|&c| c > at) { + let name = &xref[at..colon]; + if let Some(&p) = priorities.get(name) { + if p < best { + best = p; + owner = Some(name); + } + } + } + match end { + Some(e) => at = e, + None => break, + } + } + owner.unwrap_or(group) + } + + #[allow(clippy::too_many_arguments)] + fn segment(&mut self, article: i64, subject: &str, from: &str, date: &str, message_id: &str, bytes: &str, xref: &str, group: &str) -> Option { + if subject.is_empty() || message_id.is_empty() { + return None; + } + let sb = subject.as_bytes(); + let name_end = subject.rfind('"')?; + if name_end == 0 { + return None; + } + let counter = find_counter(sb, name_end)?; + self.key.clear(); + self.key.push_str(from); + self.key.push('\u{1}'); + self.key.push_str(&subject[..counter.index]); + self.key.push('\u{0}'); + self.key.push_str(&subject[counter.index + counter.length..]); + let file = match self.files.get(self.key.as_str()) { + Some(f) => f.clone(), + None => { + let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new); + self.files.insert(self.key.clone(), f.clone()); + f + } + }; + let file = file?; + if counter.segment < 1 || counter.segment > file.total_parts as i64 { + return None; + } + let id = if message_id.starts_with('<') { message_id.get(1..message_id.len().saturating_sub(1)).unwrap_or("") } else { message_id }; + Some(Seg { + number: counter.segment as u32, + message_id: id.to_string(), + bytes: js::parse_int(bytes).unwrap_or(0), + article, + posted_at: js::parse_date(date), + file, + }) + } +} + +fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option { + let name_start = subject[..name_end].rfind('"')?; + let name = &subject[name_start + 1..name_end]; + if name.is_empty() || counter.total < 1 { + return None; + } + let joined = format!("{}{}", &subject[..counter.index], &subject[counter.index + counter.length..]); + let base = js::trim(&js::collapse_spaces(&joined)).to_string(); + let file_key = sha1_hex(&format!("{from}\n{base}\n{}", counter.total)); + let release = names::release_identity(from, &base, name); + Some(FileMeta { + file_key, + name: name.to_string(), + poster: from.to_string(), + subject: base, + total_parts: counter.total as u32, + groups: xref_groups(xref, group), + release, + }) +} + +/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText). +fn scan_overview_text(text: &str, mut line: impl FnMut(Option, &str, &str, &str, &str, &str, &str)) { + let b = text.as_bytes(); + let mut start = 0; + while start < b.len() { + let end = memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len()); + let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end }; + if stop > start { + let l = &text[start..stop]; + let lb = l.as_bytes(); + let mut tabs = [usize::MAX; 9]; + let mut n = 0; + for (i, &c) in lb.iter().enumerate() { + if c == b'\t' { + tabs[n] = i; + n += 1; + if n == 9 { + break; + } + } + } + if n >= 5 { + let field = |a: usize, b: usize| -> &str { + if a == usize::MAX { + "" + } else { + &l[a + 1..if b == usize::MAX { l.len() } else { b }] + } + }; + let mut xref = if n >= 8 { field(tabs[7], tabs[8]) } else { "" }; + if let Some(rest) = xref.strip_prefix("Xref:") { + xref = rest.strip_prefix(' ').unwrap_or(rest); + } + line(js::parse_int(&l[..tabs[0]]), &l[tabs[0] + 1..tabs[1]], field(tabs[1], tabs[2]), field(tabs[2], tabs[3]), field(tabs[3], tabs[4]), field(tabs[5], tabs[6]), xref); + } + } + start = end + 1; + } +} + +fn memchr(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&c| c == needle) +} + +struct OpenFile { + parts: Vec, + seen: FxHashSet, + total: u32, + touched: u64, +} + +#[derive(Default)] +struct Aggregator { + open: FxHashMap, + batch: u64, +} + +impl Aggregator { + fn add(&mut self, segments: Vec) -> Vec> { + self.batch += 1; + let mut touched: Vec = Vec::new(); + let mut touched_set: FxHashSet = FxHashSet::default(); + for s in segments { + let key = &s.file.file_key; + let file = match self.open.get_mut(key.as_str()) { + Some(f) => f, + None => self.open.entry(key.clone()).or_insert(OpenFile { parts: Vec::new(), seen: FxHashSet::default(), total: s.file.total_parts, touched: self.batch }), + }; + if !touched_set.contains(key.as_str()) { + touched_set.insert(key.clone()); + touched.push(key.clone()); + } + file.touched = self.batch; + if file.seen.insert(s.number) { + file.parts.push(s); + } + } + let mut complete = Vec::new(); + for key in touched { + let f = self.open.get(&key).unwrap(); + if (f.parts.len() as u32) < f.total { + continue; + } + let mut f = self.open.remove(&key).unwrap(); + f.parts.sort_by_key(|s| s.number); + complete.push(f.parts); + } + complete + } + + /// Files idle for `idle` batches (all when 0): their segments, file by file in arrival order. + fn evict(&mut self, idle: u64) -> Vec> { + let batch = self.batch; + let gone: Vec = self.open.iter().filter(|(_, f)| !(idle > 0 && batch - f.touched < idle)).map(|(k, _)| k.clone()).collect(); + gone.into_iter().map(|k| self.open.remove(&k).unwrap().parts).collect() + } +} + +pub struct RangeIndexer { + group: String, + start: i64, + end: i64, + priorities: FxHashMap, + idle: u64, + chunk_articles: i64, + scanner: Scanner, + aggregator: Aggregator, + batch: Batch, + counts: FxHashMap<(&'static str, String), (u64, u64)>, + count_order: Vec<(&'static str, String)>, + completed: FxHashMap>>, + completed_order: Vec, +} + +impl RangeIndexer { + pub fn new(group: &str, start: i64, end: i64, priorities: FxHashMap, idle: u64) -> Self { + RangeIndexer { + group: group.to_string(), + start, + end, + priorities, + idle, + chunk_articles: 0, + scanner: Scanner::default(), + aggregator: Aggregator::default(), + batch: Batch { from: start, next: start, ..Default::default() }, + counts: FxHashMap::default(), + count_order: Vec::new(), + completed: FxHashMap::default(), + completed_order: Vec::new(), + } + } + + pub fn add_text(&mut self, text: &str, next: i64) { + let mut segments: Vec = Vec::new(); + let mut crossposts: Vec = Vec::new(); + let mut obfuscated = 0u64; + let mut headers = 0u64; + { + let group = self.group.clone(); + let scanner = &mut self.scanner; + let priorities = &self.priorities; + scan_overview_text(text, |n, subject, from, date, message_id, bytes, xref| { + let Some(n) = n else { return }; + if message_id.is_empty() { + return; + } + headers += 1; + let owner = if xref.is_empty() { group.as_str() } else { scanner.owner(xref, &group, priorities) }; + if owner != group { + crossposts.push(owner.to_string()); + return; + } + match scanner.segment(n, subject, from, date, message_id, bytes, xref, &group) { + Some(s) => segments.push(s), + None => obfuscated += 1, + } + }); + } + self.batch.headers += headers; + for owner in crossposts { + self.count("crosspost", owner, 1, 0); + } + if obfuscated > 0 { + self.count("obfuscated", String::new(), obfuscated, 0); + } + self.end_chunk(segments, next); + } + + fn end_chunk(&mut self, segments: Vec, next: i64) { + let done = self.aggregator.add(segments); + self.files(done); + if self.chunk_articles == 0 { + self.chunk_articles = next - self.batch.from; + } + let gone = self.aggregator.evict(self.idle); + self.cut_off(gone, false); + self.batch.next = next; + } + + pub fn finish(&mut self) { + let gone = self.aggregator.evict(0); + self.cut_off(gone, true); + self.batch.next = self.end + 1; + } + + pub fn take(&mut self) -> Batch { + self.judge_releases(); + self.scanner.files.clear(); + let next = self.batch.next; + let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() }); + for id in self.count_order.drain(..) { + let (articles, files) = self.counts.remove(&id).unwrap(); + out.counts.push((id.0, id.1, articles, files)); + } + out + } + + fn files(&mut self, files: Vec>) { + for file in files { + let key = release_key_of(&file[0]); + match self.completed.get_mut(&key) { + Some(list) => list.push(file), + None => { + self.completed_order.push(key.clone()); + self.completed.insert(key, vec![file]); + } + } + } + } + + fn judge_releases(&mut self) { + let order = std::mem::take(&mut self.completed_order); + let mut by_release = std::mem::take(&mut self.completed); + for key in order { + let list = by_release.remove(&key).unwrap(); + let judged: Vec = list + .iter() + .map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id }) + .collect(); + match names::judge(&key, &judged) { + Verdict::Keep => { + let candidates = candidates_of(&key, &list); + self.batch.releases.push(chunk_of(&key, &list)); + self.batch.candidates.extend(candidates); + } + Verdict::Drop { rule, key: rule_key, sample } => { + let segments: usize = list.iter().map(|f| f.len()).sum(); + self.count(rule, rule_key.clone(), segments as u64, list.len() as u64); + if sample { + self.batch.samples.push((chunk_of(&key, &list), rule, rule_key)); + } + } + } + } + } + + fn cut_off(&mut self, files: Vec>, at_end: bool) { + let window = self.idle as i64 * self.chunk_articles.max(1); + for list in files { + if list.len() == 1 && is_random_file(&list[0].file.name) { + let tool = names::upload_tool(&list[0].message_id); + self.count("per-article", tool.to_string(), 1, 0); + continue; + } + let min_part = list.iter().map(|s| s.number).min().unwrap(); + let max_part = list.iter().map(|s| s.number).max().unwrap(); + let first_article = list.iter().map(|s| s.article).min().unwrap(); + let cut_at_start = min_part > 1 && first_article < self.start + window; + let cut_at_end = at_end && max_part < list[0].file.total_parts; + if !cut_at_start && !cut_at_end { + self.count("incomplete", String::new(), list.len() as u64, 1); + continue; + } + let f = &list[0].file; + let mut fragment = Fragment { file_key: f.file_key.clone(), name: f.name.clone(), total_parts: f.total_parts, posted_at: None, segments: Vec::with_capacity(list.len()) }; + for s in &list { + fragment.segments.push((s.number, s.message_id.clone(), s.bytes, s.article)); + if let Some(t) = s.posted_at { + if fragment.posted_at.is_none_or(|p| t < p) { + fragment.posted_at = Some(t); + } + } + } + self.batch.fragments.push(fragment); + } + } + + fn count(&mut self, rule: &'static str, key: String, articles: u64, files: u64) { + match self.counts.get_mut(&(rule, key.clone())) { + Some(c) => { + c.0 += articles; + c.1 += files; + } + None => { + self.count_order.push((rule, key.clone())); + self.counts.insert((rule, key), (articles, files)); + } + } + } +} + +fn release_key_of(s: &Seg) -> String { + // A named file always has a release (releaseIdentity isn't null unless weak). + s.file.release.key.clone() +} + +fn posted_of(segments: &[Seg]) -> Option { + segments.iter().filter_map(|s| s.posted_at).min() +} + +fn chunk_of(key: &str, files: &[Vec]) -> ReleaseChunk { + let first = &files[0][0].file; + let chunk_files: Vec = files + .iter() + .map(|f| { + let m = &f[0].file; + ChunkFile { name: &m.name, poster: &m.poster, subject: &m.subject, posted_at: posted_of(f), groups: &m.groups, segments: f.iter().map(|s| (s.number, s.message_id.as_str(), s.bytes)).collect() } + }) + .collect(); + let mut segments = 0; + let mut bytes = 0; + let mut first_article = i64::MAX; + let mut last_article = 0; + let mut posted_at: Option = None; + for f in files { + for s in f { + segments += 1; + bytes += s.bytes; + first_article = first_article.min(s.article); + last_article = last_article.max(s.article); + } + if let Some(t) = posted_of(f) { + if posted_at.is_none_or(|p| t < p) { + posted_at = Some(t); + } + } + } + ReleaseChunk { + key: key.to_string(), + name: first.release.name.clone(), + source: first.release.source, + file_total: files.iter().find_map(|f| f[0].file.release.file_total_truthy()), + poster: first.poster.clone(), + groups: first.groups.clone(), + posted_at, + files: files.len(), + segments, + bytes, + first_article, + last_article, + readable: chunk_files.iter().any(|f| !is_random_file(f.name)), + has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)), + payload: encode_chunk(&chunk_files), + } +} + +fn candidates_of(key: &str, files: &[Vec]) -> Vec { + let mut out = Vec::new(); + let mut tail: Option<(i64, &Vec)> = None; + let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), kind, message_id: s.message_id.clone(), file: s.file.name.clone() }; + for f in files { + let name = &f[0].file.name; + if let Some(volume) = names::seven_zip_volume(name).filter(|&v| v > 1) { + if tail.is_none_or(|(v, _)| volume > v) { + tail = Some((volume, f)); + } + continue; + } + if names::is_archive_start(name) { + out.push(c("archive", &f[0])); + } else if names::is_par2(name) && !names::is_later_volume(name) { + out.push(c("par2", &f[0])); + } + } + if let Some((_, f)) = tail { + out.push(Candidate { release_key: key.to_string(), kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() }); + } + if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) { + out.push(c("lone", &files[0][0])); + } + out.truncate(4); + out +} diff --git a/tools/edge-bench-rs/src/js.rs b/tools/edge-bench-rs/src/js.rs new file mode 100644 index 0000000..8c31083 --- /dev/null +++ b/tools/edge-bench-rs/src/js.rs @@ -0,0 +1,174 @@ +//! JavaScript semantics the TypeScript worker relies on: `\s` and `trim()`, +//! UTF-16 lengths, `parseInt`, `Date.parse` (for Usenet dates) and sha1 hex. + +use sha1::{Digest, Sha1}; + +/// JS `\s` / `trim()` whitespace (WhiteSpace and LineTerminator). +pub fn is_space(c: char) -> bool { + matches!( + c, + '\t' | '\n' | '\u{0B}' | '\u{0C}' | '\r' | ' ' | '\u{A0}' | '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{2028}' | '\u{2029}' | '\u{202F}' | '\u{205F}' | '\u{3000}' | '\u{FEFF}' + ) +} + +pub fn trim(s: &str) -> &str { + s.trim_matches(is_space) +} + +/// `s.replace(/\s+/g, ' ')`. +pub fn collapse_spaces(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_space = false; + for c in s.chars() { + if is_space(c) { + if !in_space { + out.push(' '); + in_space = true; + } + } else { + out.push(c); + in_space = false; + } + } + out +} + +/// `s.split(/\s+/)`: like JS, a leading or trailing run gives an empty word. +pub fn split_spaces(s: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut start = 0; + let mut in_space = false; + let mut run_start = 0; + for (i, c) in s.char_indices() { + if is_space(c) { + if !in_space { + run_start = i; + in_space = true; + } + } else if in_space { + out.push(&s[start..run_start]); + start = i; + in_space = false; + } + } + if in_space { + out.push(&s[start..run_start]); + out.push(""); + } else { + out.push(&s[start..]); + } + out +} + +/// A string's `.length` in JavaScript (UTF-16 code units). +pub fn len16(s: &str) -> usize { + if s.is_ascii() { + s.len() + } else { + s.encode_utf16().count() + } +} + +/// `Number.parseInt(s, 10)`, None for NaN. +pub fn parse_int(s: &str) -> Option { + let s = s.trim_start_matches(is_space); + let (neg, digits) = match s.as_bytes().first() { + Some(b'-') => (true, &s[1..]), + Some(b'+') => (false, &s[1..]), + _ => (false, s), + }; + let mut n: i64 = 0; + let mut any = false; + for b in digits.bytes() { + if !b.is_ascii_digit() { + break; + } + n = n.wrapping_mul(10).wrapping_add((b - b'0') as i64); + any = true; + } + if any { + Some(if neg { -n } else { n }) + } else { + None + } +} + +/// `Date.parse` for the Date header (RFC 5322 and the obsolete forms); None for NaN. +pub fn parse_date(s: &str) -> Option { + let s = trim(s); + // "UTC" isn't an RFC 5322 zone (UT and GMT are), but Date.parse takes it (~a quarter of a.b.boneless). + let owned; + let s = match s.strip_suffix(" UTC") { + Some(rest) => { + owned = format!("{rest} GMT"); + owned.as_str() + } + None => s, + }; + if let Ok(d) = chrono::DateTime::parse_from_rfc2822(s) { + return Some(d.timestamp_millis()); + } + // "Mon, 5 Oct 2026 10:00:00 +0000 (UTC)" and other trailing comments. + if let Some(i) = s.find(" (") { + if let Ok(d) = chrono::DateTime::parse_from_rfc2822(&s[..i]) { + return Some(d.timestamp_millis()); + } + } + None +} + +/// sha1 of the UTF-8 text, as lowercase hex (node's `hash('sha1', text)`). +pub fn sha1_hex(text: &str) -> String { + let digest = Sha1::digest(text.as_bytes()); + let mut out = String::with_capacity(40); + for b in digest { + out.push(char::from_digit((b >> 4) as u32, 16).unwrap()); + out.push(char::from_digit((b & 15) as u32, 16).unwrap()); + } + out +} + +/// JSON.stringify of a string. +pub fn json_str(s: &str, out: &mut String) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\u{08}' => out.push_str("\\b"), + '\u{0C}' => out.push_str("\\f"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_like_js() { + assert_eq!(split_spaces("a b"), vec!["a", "b"]); + assert_eq!(split_spaces(" a b "), vec!["", "a", "b", ""]); + assert_eq!(split_spaces(""), vec![""]); + } + + #[test] + fn parse_int_like_js() { + assert_eq!(parse_int("123abc"), Some(123)); + assert_eq!(parse_int(" -5"), Some(-5)); + assert_eq!(parse_int("x"), None); + } + + #[test] + fn dates() { + assert_eq!(parse_date("Mon, 05 Oct 2026 10:00:00 +0000"), Some(1_791_194_400_000)); + assert_eq!(parse_date("5 Oct 2026 10:00:00 GMT"), Some(1_791_194_400_000)); + assert_eq!(parse_date("Mon, 05 Oct 26 10:00:00 UTC"), Some(1_791_194_400_000)); + } +} diff --git a/tools/edge-bench-rs/src/main.rs b/tools/edge-bench-rs/src/main.rs new file mode 100644 index 0000000..a795135 --- /dev/null +++ b/tools/edge-bench-rs/src/main.rs @@ -0,0 +1,224 @@ +//! The edge worker's indexing in Rust, on a range captured by `edge:bench +//! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as +//! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off +//! idle ones, and every N chunks judge and encode the releases. `--dump` +//! writes every output as JSON lines for tools/edge-bench-rs/check.ts. +//! +//! edge-bench-rs [--batch-chunks 10] [--idle 5] [--dump out.jsonl] + +mod codec; +mod indexer; +mod js; +mod names; + +use base64::Engine; +use indexer::{Batch, RangeIndexer}; +use js::json_str; +use rustc_hash::FxHashMap; +use std::io::{Read, Write}; +use std::time::Instant; + +fn inflate(raw: &[u8]) -> Vec { + let mut out = Vec::with_capacity(raw.len() * 6); + if raw.starts_with(&[0x1f, 0x8b]) { + flate2::read::MultiGzDecoder::new(raw).read_to_end(&mut out).unwrap(); + } else { + flate2::read::ZlibDecoder::new(raw).read_to_end(&mut out).unwrap(); + } + out +} + +fn json_field(json: &str, name: &str) -> String { + let at = json.find(&format!("\"{name}\":")).unwrap() + name.len() + 3; + let rest = json[at..].trim_start(); + if let Some(s) = rest.strip_prefix('"') { + s[..s.find('"').unwrap()].to_string() + } else { + rest[..rest.find([',', '}']).unwrap()].trim().to_string() + } +} + +fn peak_rss_mb() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); + status.lines().find(|l| l.starts_with("VmHWM:")).and_then(|l| l.split_whitespace().nth(1)).and_then(|k| k.parse::().ok()).unwrap_or(0) / 1024 +} + +fn opt_num(n: Option, out: &mut String) { + match n { + Some(n) => out.push_str(&n.to_string()), + None => out.push_str("null"), + } +} + +fn dump_batch(batch: &Batch, out: &mut impl Write) { + let b64 = base64::engine::general_purpose::STANDARD; + let mut line = String::new(); + let mut emit = |line: &mut String| { + line.push('\n'); + out.write_all(line.as_bytes()).unwrap(); + line.clear(); + }; + line.push_str(&format!("{{\"t\":\"b\",\"from\":{},\"next\":{},\"headers\":{}}}", batch.from, batch.next, batch.headers)); + emit(&mut line); + let release = |r: &indexer::ReleaseChunk, kind: &str, line: &mut String| { + line.push_str(&format!("{{\"t\":\"{kind}\",\"key\":")); + json_str(&r.key, line); + line.push_str(",\"name\":"); + json_str(&r.name, line); + line.push_str(&format!(",\"source\":\"{}\",\"fileTotal\":", r.source)); + opt_num(r.file_total, line); + line.push_str(",\"poster\":"); + json_str(&r.poster, line); + line.push_str(",\"groups\":["); + for (i, g) in r.groups.iter().enumerate() { + if i > 0 { + line.push(','); + } + json_str(g, line); + } + line.push_str("],\"postedAt\":"); + opt_num(r.posted_at, line); + line.push_str(&format!( + ",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"", + r.files, + r.segments, + r.bytes, + r.first_article, + r.last_article, + r.readable, + r.has_par2, + b64.encode(&r.payload) + )); + }; + for r in &batch.releases { + release(r, "r", &mut line); + line.push('}'); + emit(&mut line); + } + for (r, rule, key) in &batch.samples { + release(r, "s", &mut line); + line.push_str(&format!(",\"rule\":\"{rule}\",\"ruleKey\":")); + json_str(key, &mut line); + line.push('}'); + emit(&mut line); + } + for c in &batch.candidates { + line.push_str("{\"t\":\"c\",\"releaseKey\":"); + json_str(&c.release_key, &mut line); + line.push_str(&format!(",\"kind\":\"{}\",\"messageId\":", c.kind)); + json_str(&c.message_id, &mut line); + line.push_str(",\"file\":"); + json_str(&c.file, &mut line); + line.push('}'); + emit(&mut line); + } + for (rule, key, articles, files) in &batch.counts { + line.push_str(&format!("{{\"t\":\"n\",\"rule\":\"{rule}\",\"key\":")); + json_str(key, &mut line); + line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}")); + emit(&mut line); + } + for f in &batch.fragments { + line.push_str("{\"t\":\"f\",\"fileKey\":"); + json_str(&f.file_key, &mut line); + line.push_str(",\"name\":"); + json_str(&f.name, &mut line); + line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts)); + opt_num(f.posted_at, &mut line); + line.push_str(",\"segments\":["); + for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() { + if i > 0 { + line.push(','); + } + line.push_str(&format!("[{n},")); + json_str(id, &mut line); + line.push_str(&format!(",{bytes},{article}]")); + } + line.push_str("]}"); + emit(&mut line); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let dir = args.get(1).expect("usage: edge-bench-rs [--batch-chunks N] [--idle N] [--dump file]").clone(); + let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned(); + let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10); + let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5); + let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap())); + + let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap(); + let group = json_field(&range, "group"); + let start: i64 = json_field(&range, "start").parse().unwrap(); + let end: i64 = json_field(&range, "end").parse().unwrap(); + let mut files: Vec = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect(); + files.sort(); + + let mut priorities = FxHashMap::default(); + priorities.insert(group.clone(), 1); + let mut indexer = RangeIndexer::new(&group, start, end, priorities, idle); + + let (mut inflate_s, mut index_s) = (0.0, 0.0); + let (mut compressed, mut text_bytes) = (0usize, 0usize); + let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples + let mut keys = std::collections::HashSet::new(); + let wall = Instant::now(); + for (i, file) in files.iter().enumerate() { + let raw = std::fs::read(format!("{dir}/{file}")).unwrap(); + compressed += raw.len(); + let t = Instant::now(); + let data = inflate(&raw); + inflate_s += t.elapsed().as_secs_f64(); + text_bytes += data.len(); + let t = Instant::now(); + let text = String::from_utf8_lossy(&data); + let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::().unwrap() + 1; + indexer.add_text(&text, next); + let last = i + 1 == files.len(); + if last { + indexer.finish(); + } + if !last && (i + 1) % batch_chunks != 0 { + index_s += t.elapsed().as_secs_f64(); + continue; + } + let batch = indexer.take(); + index_s += t.elapsed().as_secs_f64(); + out.0 += batch.headers; + out.1 += batch.releases.len(); + for r in &batch.releases { + keys.insert(r.key.clone()); + out.2 += r.files; + out.3 += r.segments; + out.4 += r.payload.len(); + } + out.5 += batch.candidates.len(); + out.6 += batch.fragments.len(); + out.7 += batch.samples.len(); + if let Some(d) = dump.as_mut() { + dump_batch(&batch, d); + } + } + let total = inflate_s + index_s; + println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000); + println!( + "{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)", + out.0, + total, + out.0 as f64 / total, + wall.elapsed().as_secs_f64() + ); + println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb()); + println!( + " kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples", + out.1, + keys.len(), + out.2, + out.3, + out.4 as f64 / 1e6, + out.4 as f64 / out.3.max(1) as f64, + out.5, + out.6, + out.7 + ); +} diff --git a/tools/edge-bench-rs/src/names.rs b/tools/edge-bench-rs/src/names.rs new file mode 100644 index 0000000..936f581 --- /dev/null +++ b/tools/edge-bench-rs/src/names.rs @@ -0,0 +1,358 @@ +//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools. +//! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u` +//! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS +//! `\s` (which includes Unicode spaces) is spelled out as SP. + +use crate::js::{self, len16, sha1_hex}; +use md5::{Digest, Md5}; +use regex::bytes::Regex; +use std::sync::LazyLock; + +/// JS `\s` as a byte pattern. +const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)"; + +fn re(pattern: &str) -> Regex { + Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap() +} + +/// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all). +fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String { + let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) }; + // The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8. + String::from_utf8(out.into_owned()).unwrap() +} + +fn test(re: &Regex, s: &str) -> bool { + re.is_match(s.as_bytes()) +} + +static SUFFIXES: LazyLock> = LazyLock::new(|| { + vec![ + (re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""), + (re(r"(?i)\.par2$"), ""), + (re(r"(?i)\.part[0-9]+\.rar$"), ""), + (re(r"(?i)\.r[0-9]{2,3}$"), ""), + (re(r"(?i)\.rar$"), ""), + (re(r"(?i)\.7z(\.[0-9]{3})?$"), ""), + (re(r"(?i)\.zip(\.[0-9]{3})?$"), ""), + // JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back. + (re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.[0-9]{3}$"), "$1"), + (re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""), + (re(r"(?i)\.tar$"), ""), + (re(r"(?i)\.(zst|gz|xz|bz2)$"), ""), + (re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""), + ] +}); + +/// fileStem: per-file suffixes stripped, repeatedly. +pub fn file_stem(name: &str) -> String { + let mut stem = js::trim(name).to_string(); + let mut changed = true; + while changed { + changed = false; + for (suffix, with) in SUFFIXES.iter() { + if !suffix.is_match(stem.as_bytes()) { + continue; + } + let next = replace(suffix, &stem, with, false); + if next != stem && !next.is_empty() { + stem = next; + changed = true; + } + } + } + stem +} + +static QUOTED: LazyLock = LazyLock::new(|| re(r#""[^"]*""#)); +static COUNTERS: LazyLock = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)")); +static YENC_END: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$")); +static YENC: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b")); +static SEPARATORS: LazyLock = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+")); + +pub fn subject_remainder(subject: &str) -> String { + let s = replace("ED, subject, " ", true); + let s = replace(&COUNTERS, &s, " ", true); + let s = replace(&YENC_END, &s, " ", false); + let s = replace(&YENC, &s, " ", true); + let s = replace(&SEPARATORS, &s, " ", true); + js::trim(&s).to_string() +} + +fn is_random_token(token: &str) -> bool { + let t = token.strip_prefix('[').unwrap_or(token); + let t = t.strip_suffix(']').unwrap_or(t); + if len16(t) < 6 { + return false; + } + let digit = t.bytes().any(|b| b.is_ascii_digit()); + if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) { + return true; + } + digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-')) +} + +pub fn is_post_token(remainder: &str) -> bool { + let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect(); + let n = len16(&compact); + (6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric()) +} + +static NAME_LIKE: LazyLock = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}")); + +pub fn names_release(remainder: &str) -> bool { + let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect(); + test(&NAME_LIKE, &words.join(" ")) +} + +#[derive(Debug, Clone)] +pub struct Release { + pub key: String, + pub name: String, + pub source: &'static str, + /// The [i/M] total as parsed (0 is falsy in the TypeScript). + pub file_total: Option, +} + +impl Release { + pub fn file_total_truthy(&self) -> Option { + self.file_total.filter(|&n| n != 0) + } +} + +static FILE_COUNTER: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]")); + +fn hash(parts: &[&str]) -> String { + sha1_hex(&parts.join("\n")) +} + +/// releaseIdentity for a named (not weak) file: never null. +pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { + let file_total = FILE_COUNTER + .captures(subject.as_bytes()) + .and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap())); + let stem = file_stem(name); + let remainder = subject_remainder(subject); + if names_release(&remainder) { + return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total }; + } + if let Some(total) = file_total.filter(|&n| n != 0) { + if is_post_token(&remainder) { + return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total }; + } + } + Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total } +} + +static GENERIC: LazyLock = LazyLock::new(|| { + re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$") +}); + +fn is_generic_name(name: &str) -> bool { + test(&GENERIC, js::trim(name)) +} + +/// looksRandom. +pub fn looks_random(name: &str) -> bool { + // name.replace(/\.[A-Za-z0-9]{1,4}$/, '') + let name = match name.rfind('.') { + Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i], + _ => name, + }; + if is_generic_name(name) { + return true; + } + let b = name.as_bytes(); + let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit()); + let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit()); + if b.len() >= 16 && hex(b) { + return true; + } + if let Some(dot) = name.find('.') { + let (h, d) = (&b[..dot], &b[dot + 1..]); + if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) { + return true; + } + } + if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) { + return true; + } + { + let core = name.strip_suffix(" n").unwrap_or(name).as_bytes(); + if core.len() >= 6 && digits(core) { + return true; + } + } + let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count(); + let n_digit = count(u8::is_ascii_digit); + let n_lower = count(u8::is_ascii_lowercase); + let n_upper = count(u8::is_ascii_uppercase); + let alnum = b.iter().all(|c| c.is_ascii_alphanumeric()); + if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 { + return true; + } + if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 { + return true; + } + if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 { + return false; + } + n_digit > 0 || n_upper >= 6 +} + +pub fn is_random_file(name: &str) -> bool { + looks_random(&file_stem(name)) +} + +static SEVEN_SPLIT: LazyLock = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$")); +static SEVEN: LazyLock = LazyLock::new(|| re(r"(?i)\.7z$")); +static PART1: LazyLock = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$")); +static RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.rar$")); +static PART: LazyLock = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$")); +static BIN: LazyLock = LazyLock::new(|| re(r"(?i)\.bin$")); +static OLD_RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$")); +static PAR2_VOL: LazyLock = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$")); +pub static PAR2: LazyLock = LazyLock::new(|| re(r"(?i)\.par2$")); + +pub fn is_par2(name: &str) -> bool { + test(&PAR2, name) +} + +pub fn seven_zip_volume(name: &str) -> Option { + if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) { + return js::parse_int(std::str::from_utf8(&c[1]).unwrap()); + } + if test(&SEVEN, name) { + Some(0) + } else { + None + } +} + +pub fn is_archive_start(name: &str) -> bool { + if test(&BIN, name) { + return true; + } + if matches!(seven_zip_volume(name), Some(0) | Some(1)) { + return true; + } + test(&PART1, name) || (test(&RAR, name) && !test(&PART, name)) +} + +pub fn is_later_volume(name: &str) -> bool { + if let Some(c) = PART.captures(name.as_bytes()) { + return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1; + } + test(&OLD_RAR, name) || test(&PAR2_VOL, name) +} + +static TOOLS: LazyLock> = LazyLock::new(|| { + vec![ + ("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")), + ("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")), + ("ngPost", re(r"^[0-9a-f]{32}@ngPost$")), + ("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")), + ("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")), + ("JBinUp", re(r"@JBin(?:Up|Down)\.local$")), + ("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")), + ("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")), + ("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")), + ] +}); + +pub fn upload_tool(message_id: &str) -> &'static str { + if message_id.is_empty() { + return "other"; + } + for (tool, pattern) in TOOLS.iter() { + if test(pattern, message_id) { + return tool; + } + } + "other" +} + +/// explored: a release in the sampled share of a rule's drops. +pub fn explored(key: &str, percent: f64) -> bool { + if percent <= 0.0 { + return false; + } + let d = Md5::digest(key.as_bytes()); + let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]); + ((n % 10_000) as f64) < percent * 100.0 +} + +static SIDECAR: LazyLock = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$")); +static EXECUTABLE: LazyLock = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$")); + +pub struct JudgedFile<'a> { + pub name: &'a str, + pub total_parts: u32, + pub groups: &'a [String], + pub first_message_id: &'a str, +} + +pub enum Verdict { + Keep, + Drop { rule: &'static str, key: String, sample: bool }, +} + +/// RuleSet.judge with DEFAULT_RULES (no dropped signatures). +pub fn judge(release_key: &str, files: &[JudgedFile]) -> Verdict { + const DROPPED_TOOLS: [&str; 5] = ["ngPost", "hex32", "yuun", "JBinUp", "random"]; + let sample = explored(release_key, 1.0); + let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect(); + if !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) { + let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap(); + let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase(); + return Verdict::Drop { rule: "executable", key, sample }; + } + if files.iter().any(|f| !is_random_file(f.name)) { + return Verdict::Keep; + } + let first = &files[0]; + let tool = upload_tool(first.first_message_id); + let group = first.groups.first().map(String::as_str).unwrap_or(""); + if DROPPED_TOOLS.contains(&tool) { + return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample }; + } + if files.iter().all(|f| f.total_parts <= 3) { + return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample }; + } + Verdict::Keep +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stems() { + assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x"); + assert_eq!(file_stem("x.part07.rar"), "x"); + assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264"); + assert_eq!(file_stem("a.7z.002"), "a"); + } + + #[test] + fn remainders() { + assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release"); + assert!(names_release("Some Release")); + assert!(!names_release("elys de81bcabef5faf6a")); + } + + #[test] + fn random_names() { + assert!(looks_random("prvjr7hapd0i1ny14pz4")); + assert!(looks_random("RmISakytFwXSCEzjVEpaKk")); + assert!(!looks_random("Some.Show.S01E01.1080p")); + assert!(looks_random("VIDEO_TS")); + } + + #[test] + fn tools() { + assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu"); + assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost"); + assert_eq!(upload_tool("x@y"), "other"); + } +}