Compare commits
5
Commits
42b6cac3ee
...
b1a6f02de6
| Author | SHA1 | Date | |
|---|---|---|---|
 Daan MeijerandClaude Opus 5.5
|
b1a6f02de6 | ||
|
Daan MeijerandClaude Opus 5.5
|
c7b4ebe470 | ||
|
Daan MeijerandClaude Opus 5.5
|
b1c2db5984 | ||
|
Daan MeijerandClaude Opus 5.5
|
43e74199d4 | ||
|
Daan MeijerandClaude Opus 5.5
|
bacc9aa3b3 |
No files matched your search
+13
-4
@@ -99,10 +99,19 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
|||||||
settles).
|
settles).
|
||||||
|
|
||||||
2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
|
2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
|
||||||
2026-10-09): 73 k headers/s per core on an a.b.ath range from 2013,
|
2026-10-09, after two rounds of work): ~200–215 k headers/s per core on
|
||||||
103 k/s on a recent a.b.boneless range (2 M headers each), ~1.2 GB peak
|
recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak
|
||||||
per process; the 25 B lines/day goal needs ~3–4 cores over all nodes,
|
per process. **Rust** (`tools/edge-bench-rs`, the same RangeIndexer path
|
||||||
so no Rust. Per lease: compressed chunks of 10–50 k lines, in order →
|
over the same captures; `check.ts` finds every output identical, the
|
||||||
|
decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1
|
||||||
|
CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory
|
||||||
|
(0.20–0.24 GB). Its gains: no GC, hand-written matching instead of the
|
||||||
|
regexes, mimalloc, libdeflate (half of zlib's inflate time) and SIMD line
|
||||||
|
scanning. At the full target (~1.25 M/s) that is ~1.5–2 worker cores
|
||||||
|
instead of ~6–7; the speed is still set by the NNTP connections. The
|
||||||
|
cost: a second implementation of lib/headers, lib/release, lib/rules and
|
||||||
|
the chunk format, kept identical (check.ts) for as long as both exist.
|
||||||
|
Per lease: compressed chunks of 10–50 k lines, in order →
|
||||||
parse subjects → skip crossposts outside their owner group (of the
|
parse subjects → skip crossposts outside their owner group (of the
|
||||||
indexed groups in the article's Xref, the one with the lowest priority
|
indexed groups in the article's Xref, the one with the lowest priority
|
||||||
number; see the group list) → group articles into files → files
|
number; see the group list) → group articles into files → files
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
target/
|
||||||
Generated
+593
@@ -0,0 +1,593 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "1.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android_system_properties"
|
||||||
|
version = "0.1.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "autocfg"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "base64"
|
||||||
|
version = "0.22.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "block-buffer"
|
||||||
|
version = "0.10.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bumpalo"
|
||||||
|
version = "3.20.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cc"
|
||||||
|
version = "1.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630"
|
||||||
|
dependencies = [
|
||||||
|
"find-msvc-tools",
|
||||||
|
"jobserver",
|
||||||
|
"libc",
|
||||||
|
"shlex",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chrono"
|
||||||
|
version = "0.4.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||||
|
dependencies = [
|
||||||
|
"iana-time-zone",
|
||||||
|
"js-sys",
|
||||||
|
"num-traits",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation-sys"
|
||||||
|
version = "0.8.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crypto-common"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
"typenum",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "digest"
|
||||||
|
version = "0.10.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||||
|
dependencies = [
|
||||||
|
"block-buffer",
|
||||||
|
"crypto-common",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "edge-bench-rs"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"chrono",
|
||||||
|
"flate2",
|
||||||
|
"libdeflater",
|
||||||
|
"md-5",
|
||||||
|
"memchr",
|
||||||
|
"mimalloc",
|
||||||
|
"regex",
|
||||||
|
"rustc-hash",
|
||||||
|
"sha1",
|
||||||
|
"zstd",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "find-msvc-tools"
|
||||||
|
version = "0.1.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "flate2"
|
||||||
|
version = "1.1.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||||
|
dependencies = [
|
||||||
|
"zlib-rs",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-core"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-task"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-util"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"futures-task",
|
||||||
|
"pin-project-lite",
|
||||||
|
"slab",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "generic-array"
|
||||||
|
version = "0.14.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||||
|
dependencies = [
|
||||||
|
"typenum",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"r-efi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone"
|
||||||
|
version = "0.1.65"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||||
|
dependencies = [
|
||||||
|
"android_system_properties",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"iana-time-zone-haiku",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone-haiku"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jobserver"
|
||||||
|
version = "0.1.35"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "js-sys"
|
||||||
|
version = "0.3.106"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"futures-util",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libc"
|
||||||
|
version = "0.2.190"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libdeflate-sys"
|
||||||
|
version = "1.26.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libdeflater"
|
||||||
|
version = "1.26.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f"
|
||||||
|
dependencies = [
|
||||||
|
"libdeflate-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libmimalloc-sys"
|
||||||
|
version = "0.1.49"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "log"
|
||||||
|
version = "0.4.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "md-5"
|
||||||
|
version = "0.10.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mimalloc"
|
||||||
|
version = "0.1.52"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||||
|
dependencies = [
|
||||||
|
"libmimalloc-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-traits"
|
||||||
|
version = "0.2.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.21.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pin-project-lite"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pkg-config"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "r-efi"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.4.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.8.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-hash"
|
||||||
|
version = "2.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustversion"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha1"
|
||||||
|
version = "0.10.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shlex"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "slab"
|
||||||
|
version = "0.4.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.119"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "3.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "typenum"
|
||||||
|
version = "1.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.26"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "version_check"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen"
|
||||||
|
version = "0.2.129"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"once_cell",
|
||||||
|
"rustversion",
|
||||||
|
"wasm-bindgen-macro",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro"
|
||||||
|
version = "0.2.129"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||||
|
dependencies = [
|
||||||
|
"quote",
|
||||||
|
"wasm-bindgen-macro-support",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro-support"
|
||||||
|
version = "0.2.129"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||||
|
dependencies = [
|
||||||
|
"bumpalo",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 3.0.6",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-shared"
|
||||||
|
version = "0.2.129"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-core"
|
||||||
|
version = "0.62.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||||
|
dependencies = [
|
||||||
|
"windows-implement",
|
||||||
|
"windows-interface",
|
||||||
|
"windows-link",
|
||||||
|
"windows-result",
|
||||||
|
"windows-strings",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-implement"
|
||||||
|
version = "0.60.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-interface"
|
||||||
|
version = "0.59.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-link"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-result"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-strings"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zlib-rs"
|
||||||
|
version = "0.6.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd"
|
||||||
|
version = "0.13.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-safe",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-safe"
|
||||||
|
version = "7.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-sys"
|
||||||
|
version = "2.1.0+zstd.1.5.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"pkg-config",
|
||||||
|
]
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
[package]
|
||||||
|
name = "edge-bench-rs"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
description = "The edge worker's indexing (lib/edge RangeIndexer) in Rust, to compare with edge:bench on captured ranges"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
base64 = "0.22"
|
||||||
|
chrono = "0.4"
|
||||||
|
flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
|
||||||
|
libdeflater = "1.26.1"
|
||||||
|
md-5 = "0.10"
|
||||||
|
memchr = "2.8.3"
|
||||||
|
mimalloc = "0.1.52"
|
||||||
|
regex = "1"
|
||||||
|
rustc-hash = "2"
|
||||||
|
sha1 = "0.10"
|
||||||
|
zstd = "0.13"
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
lto = true
|
||||||
|
codegen-units = 1
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
/**
|
||||||
|
* Compares edge-bench-rs's output (--dump) with the TypeScript RangeIndexer
|
||||||
|
* on the same captured range, item by item: releases (every field, and the
|
||||||
|
* payload decoded with lib/chunks), samples, probe candidates, rule counts,
|
||||||
|
* fragments and batch boundaries.
|
||||||
|
*
|
||||||
|
* node --import=@poppinss/ts-exec tools/edge-bench-rs/check.ts <dir> <dump.jsonl> [--batch-chunks 10] [--idle 5]
|
||||||
|
*/
|
||||||
|
import { readFile, readdir } from 'node:fs/promises'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { unzipSync } from 'node:zlib'
|
||||||
|
import { isDeepStrictEqual } from 'node:util'
|
||||||
|
import { RangeIndexer, type EdgeBatch } from '#lib/edge/index'
|
||||||
|
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||||
|
import { decodeChunk } from '#lib/chunks/index'
|
||||||
|
|
||||||
|
const [dir, dumpPath] = process.argv.slice(2)
|
||||||
|
const flag = (name: string, fallback: number) => {
|
||||||
|
const i = process.argv.indexOf(name)
|
||||||
|
return i === -1 ? fallback : Number(process.argv[i + 1])
|
||||||
|
}
|
||||||
|
const batchChunks = flag('--batch-chunks', 10)
|
||||||
|
const idle = flag('--idle', 5)
|
||||||
|
|
||||||
|
const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
|
||||||
|
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
|
||||||
|
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
|
||||||
|
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
|
||||||
|
const batches: EdgeBatch[] = []
|
||||||
|
for (const [i, file] of files.entries()) {
|
||||||
|
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
|
||||||
|
indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
|
||||||
|
const last = i === files.length - 1
|
||||||
|
if (last) indexer.finish()
|
||||||
|
if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Canonical lines per kind, in a stable order, from both sides. */
|
||||||
|
type Item = Record<string, unknown>
|
||||||
|
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
|
||||||
|
const payloadBytes = { ts: 0, rs: 0 }
|
||||||
|
let side: 'ts' | 'rs' = 'ts'
|
||||||
|
const release = (r: Item, b: number) => {
|
||||||
|
const { payload, ...rest } = r as Item & { payload: Buffer }
|
||||||
|
// The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
|
||||||
|
// differently); what the payload decodes to is.
|
||||||
|
payloadBytes[side] += payload.length
|
||||||
|
return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
|
||||||
|
}
|
||||||
|
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||||
|
batches.forEach((batch, b) => {
|
||||||
|
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
|
||||||
|
for (const r of batch.releases) ts.r.push(release({ key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
||||||
|
for (const s of batch.samples) ts.s.push({ b, key: s.key, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
||||||
|
for (const c of batch.candidates) ts.c.push({ b, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
||||||
|
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
|
||||||
|
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
|
||||||
|
})
|
||||||
|
|
||||||
|
const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||||
|
side = 'rs'
|
||||||
|
let b = -1
|
||||||
|
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
|
||||||
|
if (!line) continue
|
||||||
|
const { t, ...item } = JSON.parse(line)
|
||||||
|
if (t === 'b') b++
|
||||||
|
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
|
||||||
|
else if (t === 's') rs.s.push({ b, key: item.key, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
|
||||||
|
else rs[t].push({ b, ...item })
|
||||||
|
}
|
||||||
|
|
||||||
|
const keyOf: Record<string, (i: Item) => string> = {
|
||||||
|
b: (i) => String(i.b),
|
||||||
|
r: (i) => `${i.b} ${i.key}`,
|
||||||
|
s: (i) => `${i.b} ${i.key}`,
|
||||||
|
c: (i) => `${i.b} ${i.releaseKey} ${i.kind} ${i.messageId}`,
|
||||||
|
n: (i) => `${i.b} ${i.rule} ${i.key}`,
|
||||||
|
f: (i) => `${i.b} ${i.fileKey}`,
|
||||||
|
}
|
||||||
|
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
|
||||||
|
let failed = false
|
||||||
|
for (const kind of Object.keys(keyOf)) {
|
||||||
|
const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
|
||||||
|
const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
|
||||||
|
const missing = [...a.keys()].filter((k) => !z.has(k))
|
||||||
|
const extra = [...z.keys()].filter((k) => !a.has(k))
|
||||||
|
const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
|
||||||
|
const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
|
||||||
|
failed ||= !ok
|
||||||
|
console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
|
||||||
|
for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`)
|
||||||
|
for (const k of differ.slice(0, 2)) {
|
||||||
|
const x = a.get(k)!
|
||||||
|
const y = z.get(k)!
|
||||||
|
const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
|
||||||
|
console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
|
||||||
|
process.exit(failed ? 1 : 0)
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
//! lib/chunks/codec.ts encodeChunk, format v2: a version byte, the length of
|
||||||
|
//! the compressed part, zstd(JSON head + text ids), then the nyuu and hex ids
|
||||||
|
//! as raw bytes.
|
||||||
|
|
||||||
|
use crate::js::json_str;
|
||||||
|
|
||||||
|
pub struct ChunkFile<'a> {
|
||||||
|
pub name: &'a str,
|
||||||
|
pub poster: &'a str,
|
||||||
|
pub subject: &'a str,
|
||||||
|
pub posted_at: Option<i64>,
|
||||||
|
pub groups: &'a [String],
|
||||||
|
/// [part, message-id, bytes]
|
||||||
|
pub segments: Vec<(u32, &'a str, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_nyuu(id: &str, suffix: &str) -> bool {
|
||||||
|
let b = id.as_bytes();
|
||||||
|
if b.len() != 38 + suffix.len() || b[24] != b'-' || !id.ends_with(suffix) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
(0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[25..38].iter().all(u8::is_ascii_digit)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_hex(id: &str, suffix: &str) -> bool {
|
||||||
|
let b = id.as_bytes();
|
||||||
|
b.len() == 32 + suffix.len() && id.ends_with(suffix) && b[..32].iter().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(c))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pack_letters(id: &[u8], out: &mut Vec<u8>) {
|
||||||
|
let mut acc: u32 = 0;
|
||||||
|
let mut bits = 0;
|
||||||
|
for i in (0..24).step_by(2) {
|
||||||
|
acc = (acc << 10) | ((id[i] - b'A') as u32 * 26 + (id[i + 1] - b'a') as u32);
|
||||||
|
bits += 10;
|
||||||
|
while bits >= 8 {
|
||||||
|
bits -= 8;
|
||||||
|
out.push(((acc >> bits) & 0xff) as u8);
|
||||||
|
}
|
||||||
|
acc &= (1 << bits) - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_val(c: u8) -> u8 {
|
||||||
|
if c.is_ascii_digit() {
|
||||||
|
c - b'0'
|
||||||
|
} else {
|
||||||
|
c - b'a' + 10
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_num(out: &mut String, n: i64) {
|
||||||
|
use std::fmt::Write;
|
||||||
|
write!(out, "{n}").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode_chunk(files: &[ChunkFile]) -> Vec<u8> {
|
||||||
|
let mut head = String::with_capacity(256 * files.len());
|
||||||
|
let mut text: Vec<&str> = Vec::new();
|
||||||
|
let mut raw: Vec<u8> = Vec::new();
|
||||||
|
head.push('[');
|
||||||
|
for (fi, f) in files.iter().enumerate() {
|
||||||
|
let ids = &f.segments;
|
||||||
|
let code = match ids.first() {
|
||||||
|
None => 't',
|
||||||
|
Some(&(_, first, _)) => {
|
||||||
|
let nyuu = first.get(38..).filter(|s| s.starts_with('@'));
|
||||||
|
let hex = first.get(32..).filter(|s| s.starts_with('@'));
|
||||||
|
if nyuu.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_nyuu(id, suffix))) {
|
||||||
|
'n'
|
||||||
|
} else if hex.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_hex(id, suffix))) {
|
||||||
|
'h'
|
||||||
|
} else {
|
||||||
|
't'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if fi > 0 {
|
||||||
|
head.push(',');
|
||||||
|
}
|
||||||
|
head.push_str("{\"n\":");
|
||||||
|
json_str(f.name, &mut head);
|
||||||
|
head.push_str(",\"p\":");
|
||||||
|
json_str(f.poster, &mut head);
|
||||||
|
head.push_str(",\"j\":");
|
||||||
|
json_str(f.subject, &mut head);
|
||||||
|
head.push_str(",\"d\":");
|
||||||
|
match f.posted_at {
|
||||||
|
Some(t) => push_num(&mut head, t),
|
||||||
|
None => head.push_str("null"),
|
||||||
|
}
|
||||||
|
head.push_str(",\"g\":[");
|
||||||
|
for (i, g) in f.groups.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
head.push(',');
|
||||||
|
}
|
||||||
|
json_str(g, &mut head);
|
||||||
|
}
|
||||||
|
head.push_str("],\"s\":[");
|
||||||
|
for (i, &(_, _, b)) in ids.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
head.push(',');
|
||||||
|
}
|
||||||
|
push_num(&mut head, b);
|
||||||
|
}
|
||||||
|
head.push(']');
|
||||||
|
if !ids.iter().enumerate().all(|(k, &(n, _, _))| n as usize == k + 1) {
|
||||||
|
head.push_str(",\"k\":[");
|
||||||
|
for (i, &(n, _, _)) in ids.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
head.push(',');
|
||||||
|
}
|
||||||
|
push_num(&mut head, n as i64);
|
||||||
|
}
|
||||||
|
head.push(']');
|
||||||
|
}
|
||||||
|
match code {
|
||||||
|
'n' => {
|
||||||
|
head.push_str(",\"c\":\"n\",\"x\":");
|
||||||
|
json_str(&ids[0].1[38..], &mut head);
|
||||||
|
head.push_str(",\"t\":[");
|
||||||
|
let mut previous = 0i64;
|
||||||
|
for (i, &(_, id, _)) in ids.iter().enumerate() {
|
||||||
|
pack_letters(id.as_bytes(), &mut raw);
|
||||||
|
let t: i64 = id[25..38].parse().unwrap();
|
||||||
|
if i > 0 {
|
||||||
|
head.push(',');
|
||||||
|
}
|
||||||
|
push_num(&mut head, t - previous);
|
||||||
|
previous = t;
|
||||||
|
}
|
||||||
|
head.push(']');
|
||||||
|
}
|
||||||
|
'h' => {
|
||||||
|
head.push_str(",\"c\":\"h\",\"x\":");
|
||||||
|
json_str(&ids[0].1[32..], &mut head);
|
||||||
|
for &(_, id, _) in ids {
|
||||||
|
let b = id.as_bytes();
|
||||||
|
for i in (0..32).step_by(2) {
|
||||||
|
raw.push(hex_val(b[i]) << 4 | hex_val(b[i + 1]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => text.extend(ids.iter().map(|&(_, id, _)| id)),
|
||||||
|
}
|
||||||
|
head.push('}');
|
||||||
|
}
|
||||||
|
head.push(']');
|
||||||
|
let mut plain = head.into_bytes();
|
||||||
|
for id in text {
|
||||||
|
plain.push(b'\n');
|
||||||
|
plain.extend_from_slice(id.as_bytes());
|
||||||
|
}
|
||||||
|
let packed = zstd::bulk::compress(&plain, 3).unwrap();
|
||||||
|
let mut out = Vec::with_capacity(5 + packed.len() + raw.len());
|
||||||
|
out.push(2);
|
||||||
|
out.extend_from_slice(&(packed.len() as u32).to_be_bytes());
|
||||||
|
out.extend_from_slice(&packed);
|
||||||
|
out.extend_from_slice(&raw);
|
||||||
|
out
|
||||||
|
}
|
||||||
@@ -0,0 +1,603 @@
|
|||||||
|
//! lib/edge/scan.ts (SegmentScanner, scanOverviewText), lib/files/aggregator.ts
|
||||||
|
//! and lib/edge/range_indexer.ts (RangeIndexer, chunkOf, candidatesOf), in the
|
||||||
|
//! same order of operations, so the outputs can be compared one to one.
|
||||||
|
|
||||||
|
use crate::codec::{encode_chunk, ChunkFile};
|
||||||
|
use crate::js::{self, sha1_hex};
|
||||||
|
use crate::names::{self, is_random_file, JudgedFile, Release, Verdict};
|
||||||
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
|
use std::rc::Rc;
|
||||||
|
|
||||||
|
pub static mut PROF: [u64; 8] = [0; 8];
|
||||||
|
|
||||||
|
pub struct FileMeta {
|
||||||
|
pub file_key: String,
|
||||||
|
pub name: String,
|
||||||
|
pub poster: String,
|
||||||
|
pub subject: String,
|
||||||
|
pub total_parts: u32,
|
||||||
|
pub groups: Vec<String>,
|
||||||
|
pub release: Release,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Seg {
|
||||||
|
pub file: Rc<FileMeta>,
|
||||||
|
pub number: u32,
|
||||||
|
pub message_id: String,
|
||||||
|
pub bytes: i64,
|
||||||
|
pub article: i64,
|
||||||
|
pub posted_at: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ReleaseChunk {
|
||||||
|
pub key: String,
|
||||||
|
pub name: String,
|
||||||
|
pub source: &'static str,
|
||||||
|
pub file_total: Option<i64>,
|
||||||
|
pub poster: String,
|
||||||
|
pub groups: Vec<String>,
|
||||||
|
pub posted_at: Option<i64>,
|
||||||
|
pub files: usize,
|
||||||
|
pub segments: usize,
|
||||||
|
pub bytes: i64,
|
||||||
|
pub first_article: i64,
|
||||||
|
pub last_article: i64,
|
||||||
|
pub readable: bool,
|
||||||
|
pub has_par2: bool,
|
||||||
|
pub payload: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Candidate {
|
||||||
|
pub release_key: String,
|
||||||
|
pub kind: &'static str,
|
||||||
|
pub message_id: String,
|
||||||
|
pub file: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Fragment {
|
||||||
|
pub file_key: String,
|
||||||
|
pub name: String,
|
||||||
|
pub total_parts: u32,
|
||||||
|
pub posted_at: Option<i64>,
|
||||||
|
pub segments: Vec<(u32, String, i64, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct Batch {
|
||||||
|
pub from: i64,
|
||||||
|
pub next: i64,
|
||||||
|
pub headers: u64,
|
||||||
|
pub releases: Vec<ReleaseChunk>,
|
||||||
|
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
||||||
|
pub candidates: Vec<Candidate>,
|
||||||
|
/// (rule, key) → (articles, files)
|
||||||
|
pub counts: Vec<(&'static str, String, u64, u64)>,
|
||||||
|
pub fragments: Vec<Fragment>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Counter {
|
||||||
|
index: usize,
|
||||||
|
length: usize,
|
||||||
|
segment: i64,
|
||||||
|
total: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The last "(k/N)" after `after`, scanning back from the end (findCounter).
|
||||||
|
fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
|
||||||
|
let mut open = subject.len();
|
||||||
|
loop {
|
||||||
|
// lastIndexOf('(', open - 1): at most open - 1.
|
||||||
|
if open == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
open = subject[..open].iter().rposition(|&c| c == b'(')?;
|
||||||
|
if open <= after {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut i = open + 1;
|
||||||
|
let mut segment: i64 = 0;
|
||||||
|
let mut digits = 0;
|
||||||
|
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||||||
|
segment = segment * 10 + (c - b'0') as i64;
|
||||||
|
digits += 1;
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
if digits == 0 || subject.get(i) != Some(&b'/') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
let mut total: i64 = 0;
|
||||||
|
digits = 0;
|
||||||
|
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||||||
|
total = total * 10 + (c - b'0') as i64;
|
||||||
|
digits += 1;
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
if digits == 0 || subject.get(i) != Some(&b')') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Some(Counter { index: open, length: i + 1 - open, segment, total });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn xref_groups(xref: &str, group: &str) -> Vec<String> {
|
||||||
|
let mut groups = Vec::new();
|
||||||
|
for token in js::split_spaces(js::trim(xref)).into_iter().skip(1) {
|
||||||
|
if let Some(colon) = token.rfind(':') {
|
||||||
|
if colon > 0 {
|
||||||
|
groups.push(token[..colon].to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if groups.is_empty() {
|
||||||
|
groups.push(group.to_string());
|
||||||
|
}
|
||||||
|
groups
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Scanner {
|
||||||
|
files: FxHashMap<String, Option<Rc<FileMeta>>>,
|
||||||
|
key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Scanner {
|
||||||
|
fn owner<'a>(&self, xref: &'a str, group: &'a str, priorities: &FxHashMap<String, u32>) -> &'a str {
|
||||||
|
let b = xref.as_bytes();
|
||||||
|
let Some(mut at) = xref.find(' ') else { return group };
|
||||||
|
let mut owner: Option<&str> = None;
|
||||||
|
let mut best = u32::MAX;
|
||||||
|
loop {
|
||||||
|
while b.get(at) == Some(&b' ') {
|
||||||
|
at += 1;
|
||||||
|
}
|
||||||
|
if at >= b.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let end = xref[at..].find(' ').map(|i| i + at);
|
||||||
|
// lastIndexOf(':', end ?? length): a colon at or before `end`.
|
||||||
|
let limit = end.unwrap_or(b.len());
|
||||||
|
let colon = xref[..limit.min(b.len())].rfind(':');
|
||||||
|
if let Some(colon) = colon.filter(|&c| c > at) {
|
||||||
|
let name = &xref[at..colon];
|
||||||
|
if let Some(&p) = priorities.get(name) {
|
||||||
|
if p < best {
|
||||||
|
best = p;
|
||||||
|
owner = Some(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match end {
|
||||||
|
Some(e) => at = e,
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
owner.unwrap_or(group)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn segment(&mut self, article: i64, subject: &str, from: &str, date: &str, message_id: &str, bytes: &str, xref: &str, group: &str) -> Option<Seg> {
|
||||||
|
if subject.is_empty() || message_id.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let sb = subject.as_bytes();
|
||||||
|
let name_end = subject.rfind('"')?;
|
||||||
|
if name_end == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let counter = find_counter(sb, name_end)?;
|
||||||
|
self.key.clear();
|
||||||
|
self.key.push_str(from);
|
||||||
|
self.key.push('\u{1}');
|
||||||
|
self.key.push_str(&subject[..counter.index]);
|
||||||
|
self.key.push('\u{0}');
|
||||||
|
self.key.push_str(&subject[counter.index + counter.length..]);
|
||||||
|
let file = match self.files.get(self.key.as_str()) {
|
||||||
|
Some(f) => f.clone(),
|
||||||
|
None => {
|
||||||
|
let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new);
|
||||||
|
self.files.insert(self.key.clone(), f.clone());
|
||||||
|
f
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let file = file?;
|
||||||
|
if counter.segment < 1 || counter.segment > file.total_parts as i64 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let id = if message_id.starts_with('<') { message_id.get(1..message_id.len().saturating_sub(1)).unwrap_or("") } else { message_id };
|
||||||
|
Some(Seg {
|
||||||
|
number: counter.segment as u32,
|
||||||
|
message_id: id.to_string(),
|
||||||
|
bytes: js::parse_int(bytes).unwrap_or(0),
|
||||||
|
article,
|
||||||
|
posted_at: js::parse_date(date),
|
||||||
|
file,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
||||||
|
let name_start = subject[..name_end].rfind('"')?;
|
||||||
|
let name = &subject[name_start + 1..name_end];
|
||||||
|
if name.is_empty() || counter.total < 1 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let joined = format!("{}{}", &subject[..counter.index], &subject[counter.index + counter.length..]);
|
||||||
|
let base = js::trim(&js::collapse_spaces(&joined)).to_string();
|
||||||
|
let file_key = sha1_hex(&format!("{from}\n{base}\n{}", counter.total));
|
||||||
|
let release = names::release_identity(from, &base, name);
|
||||||
|
Some(FileMeta {
|
||||||
|
file_key,
|
||||||
|
name: name.to_string(),
|
||||||
|
poster: from.to_string(),
|
||||||
|
subject: base,
|
||||||
|
total_parts: counter.total as u32,
|
||||||
|
groups: xref_groups(xref, group),
|
||||||
|
release,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
|
||||||
|
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
|
||||||
|
let b = text.as_bytes();
|
||||||
|
let mut start = 0;
|
||||||
|
while start < b.len() {
|
||||||
|
let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
|
||||||
|
let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end };
|
||||||
|
if stop > start {
|
||||||
|
let l = &text[start..stop];
|
||||||
|
let lb = l.as_bytes();
|
||||||
|
let mut tabs = [usize::MAX; 9];
|
||||||
|
let mut n = 0;
|
||||||
|
for i in memchr::memchr_iter(b'\t', lb) {
|
||||||
|
tabs[n] = i;
|
||||||
|
n += 1;
|
||||||
|
if n == 9 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n >= 5 {
|
||||||
|
let field = |a: usize, b: usize| -> &str {
|
||||||
|
if a == usize::MAX {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
&l[a + 1..if b == usize::MAX { l.len() } else { b }]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut xref = if n >= 8 { field(tabs[7], tabs[8]) } else { "" };
|
||||||
|
if let Some(rest) = xref.strip_prefix("Xref:") {
|
||||||
|
xref = rest.strip_prefix(' ').unwrap_or(rest);
|
||||||
|
}
|
||||||
|
line(js::parse_int(&l[..tabs[0]]), &l[tabs[0] + 1..tabs[1]], field(tabs[1], tabs[2]), field(tabs[2], tabs[3]), field(tabs[3], tabs[4]), field(tabs[5], tabs[6]), xref);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
start = end + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct OpenFile {
|
||||||
|
parts: Vec<Seg>,
|
||||||
|
seen: FxHashSet<u32>,
|
||||||
|
total: u32,
|
||||||
|
touched: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Aggregator {
|
||||||
|
open: FxHashMap<String, OpenFile>,
|
||||||
|
batch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Aggregator {
|
||||||
|
fn add(&mut self, segments: Vec<Seg>) -> Vec<Vec<Seg>> {
|
||||||
|
self.batch += 1;
|
||||||
|
let mut touched: Vec<String> = Vec::new();
|
||||||
|
let mut touched_set: FxHashSet<String> = FxHashSet::default();
|
||||||
|
for s in segments {
|
||||||
|
let key = &s.file.file_key;
|
||||||
|
let file = match self.open.get_mut(key.as_str()) {
|
||||||
|
Some(f) => f,
|
||||||
|
None => self.open.entry(key.clone()).or_insert(OpenFile { parts: Vec::new(), seen: FxHashSet::default(), total: s.file.total_parts, touched: self.batch }),
|
||||||
|
};
|
||||||
|
if !touched_set.contains(key.as_str()) {
|
||||||
|
touched_set.insert(key.clone());
|
||||||
|
touched.push(key.clone());
|
||||||
|
}
|
||||||
|
file.touched = self.batch;
|
||||||
|
if file.seen.insert(s.number) {
|
||||||
|
file.parts.push(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut complete = Vec::new();
|
||||||
|
for key in touched {
|
||||||
|
let f = self.open.get(&key).unwrap();
|
||||||
|
if (f.parts.len() as u32) < f.total {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut f = self.open.remove(&key).unwrap();
|
||||||
|
f.parts.sort_by_key(|s| s.number);
|
||||||
|
complete.push(f.parts);
|
||||||
|
}
|
||||||
|
complete
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Files idle for `idle` batches (all when 0): their segments, file by file in arrival order.
|
||||||
|
fn evict(&mut self, idle: u64) -> Vec<Vec<Seg>> {
|
||||||
|
let batch = self.batch;
|
||||||
|
let gone: Vec<String> = self.open.iter().filter(|(_, f)| !(idle > 0 && batch - f.touched < idle)).map(|(k, _)| k.clone()).collect();
|
||||||
|
gone.into_iter().map(|k| self.open.remove(&k).unwrap().parts).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct RangeIndexer {
|
||||||
|
group: String,
|
||||||
|
start: i64,
|
||||||
|
end: i64,
|
||||||
|
priorities: FxHashMap<String, u32>,
|
||||||
|
idle: u64,
|
||||||
|
chunk_articles: i64,
|
||||||
|
scanner: Scanner,
|
||||||
|
aggregator: Aggregator,
|
||||||
|
batch: Batch,
|
||||||
|
counts: FxHashMap<(&'static str, String), (u64, u64)>,
|
||||||
|
count_order: Vec<(&'static str, String)>,
|
||||||
|
completed: FxHashMap<String, Vec<Vec<Seg>>>,
|
||||||
|
completed_order: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RangeIndexer {
|
||||||
|
pub fn new(group: &str, start: i64, end: i64, priorities: FxHashMap<String, u32>, idle: u64) -> Self {
|
||||||
|
RangeIndexer {
|
||||||
|
group: group.to_string(),
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
priorities,
|
||||||
|
idle,
|
||||||
|
chunk_articles: 0,
|
||||||
|
scanner: Scanner::default(),
|
||||||
|
aggregator: Aggregator::default(),
|
||||||
|
batch: Batch { from: start, next: start, ..Default::default() },
|
||||||
|
counts: FxHashMap::default(),
|
||||||
|
count_order: Vec::new(),
|
||||||
|
completed: FxHashMap::default(),
|
||||||
|
completed_order: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_text(&mut self, text: &str, next: i64) {
|
||||||
|
let mut segments: Vec<Seg> = Vec::new();
|
||||||
|
let mut crossposts: Vec<String> = Vec::new();
|
||||||
|
let mut obfuscated = 0u64;
|
||||||
|
let mut headers = 0u64;
|
||||||
|
{
|
||||||
|
let group = self.group.clone();
|
||||||
|
let scanner = &mut self.scanner;
|
||||||
|
let priorities = &self.priorities;
|
||||||
|
scan_overview_text(text, |n, subject, from, date, message_id, bytes, xref| {
|
||||||
|
let Some(n) = n else { return };
|
||||||
|
if message_id.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
headers += 1;
|
||||||
|
let owner = if xref.is_empty() { group.as_str() } else { scanner.owner(xref, &group, priorities) };
|
||||||
|
if owner != group {
|
||||||
|
crossposts.push(owner.to_string());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match scanner.segment(n, subject, from, date, message_id, bytes, xref, &group) {
|
||||||
|
Some(s) => segments.push(s),
|
||||||
|
None => obfuscated += 1,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
self.batch.headers += headers;
|
||||||
|
for owner in crossposts {
|
||||||
|
self.count("crosspost", owner, 1, 0);
|
||||||
|
}
|
||||||
|
if obfuscated > 0 {
|
||||||
|
self.count("obfuscated", String::new(), obfuscated, 0);
|
||||||
|
}
|
||||||
|
self.end_chunk(segments, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn end_chunk(&mut self, segments: Vec<Seg>, next: i64) {
|
||||||
|
let t = std::time::Instant::now();
|
||||||
|
let done = self.aggregator.add(segments);
|
||||||
|
unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
|
||||||
|
self.files(done);
|
||||||
|
if self.chunk_articles == 0 {
|
||||||
|
self.chunk_articles = next - self.batch.from;
|
||||||
|
}
|
||||||
|
let t = std::time::Instant::now();
|
||||||
|
let gone = self.aggregator.evict(self.idle);
|
||||||
|
self.cut_off(gone, false);
|
||||||
|
unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
|
||||||
|
self.batch.next = next;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn finish(&mut self) {
|
||||||
|
let gone = self.aggregator.evict(0);
|
||||||
|
self.cut_off(gone, true);
|
||||||
|
self.batch.next = self.end + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn take(&mut self) -> Batch {
|
||||||
|
let t = std::time::Instant::now();
|
||||||
|
self.judge_releases();
|
||||||
|
unsafe { PROF[5] += t.elapsed().as_nanos() as u64; }
|
||||||
|
self.scanner.files.clear();
|
||||||
|
let next = self.batch.next;
|
||||||
|
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
||||||
|
for id in self.count_order.drain(..) {
|
||||||
|
let (articles, files) = self.counts.remove(&id).unwrap();
|
||||||
|
out.counts.push((id.0, id.1, articles, files));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn files(&mut self, files: Vec<Vec<Seg>>) {
|
||||||
|
for file in files {
|
||||||
|
let key = release_key_of(&file[0]);
|
||||||
|
match self.completed.get_mut(&key) {
|
||||||
|
Some(list) => list.push(file),
|
||||||
|
None => {
|
||||||
|
self.completed_order.push(key.clone());
|
||||||
|
self.completed.insert(key, vec![file]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn judge_releases(&mut self) {
|
||||||
|
let order = std::mem::take(&mut self.completed_order);
|
||||||
|
let mut by_release = std::mem::take(&mut self.completed);
|
||||||
|
for key in order {
|
||||||
|
let list = by_release.remove(&key).unwrap();
|
||||||
|
let judged: Vec<JudgedFile> = list
|
||||||
|
.iter()
|
||||||
|
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
||||||
|
.collect();
|
||||||
|
match names::judge(&key, &judged) {
|
||||||
|
Verdict::Keep => {
|
||||||
|
let candidates = candidates_of(&key, &list);
|
||||||
|
self.batch.releases.push(chunk_of(&key, &list));
|
||||||
|
self.batch.candidates.extend(candidates);
|
||||||
|
}
|
||||||
|
Verdict::Drop { rule, key: rule_key, sample } => {
|
||||||
|
let segments: usize = list.iter().map(|f| f.len()).sum();
|
||||||
|
self.count(rule, rule_key.clone(), segments as u64, list.len() as u64);
|
||||||
|
if sample {
|
||||||
|
self.batch.samples.push((chunk_of(&key, &list), rule, rule_key));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cut_off(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
|
||||||
|
let window = self.idle as i64 * self.chunk_articles.max(1);
|
||||||
|
for list in files {
|
||||||
|
if list.len() == 1 && is_random_file(&list[0].file.name) {
|
||||||
|
let tool = names::upload_tool(&list[0].message_id);
|
||||||
|
self.count("per-article", tool.to_string(), 1, 0);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let min_part = list.iter().map(|s| s.number).min().unwrap();
|
||||||
|
let max_part = list.iter().map(|s| s.number).max().unwrap();
|
||||||
|
let first_article = list.iter().map(|s| s.article).min().unwrap();
|
||||||
|
let cut_at_start = min_part > 1 && first_article < self.start + window;
|
||||||
|
let cut_at_end = at_end && max_part < list[0].file.total_parts;
|
||||||
|
if !cut_at_start && !cut_at_end {
|
||||||
|
self.count("incomplete", String::new(), list.len() as u64, 1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let f = &list[0].file;
|
||||||
|
let mut fragment = Fragment { file_key: f.file_key.clone(), name: f.name.clone(), total_parts: f.total_parts, posted_at: None, segments: Vec::with_capacity(list.len()) };
|
||||||
|
for s in &list {
|
||||||
|
fragment.segments.push((s.number, s.message_id.clone(), s.bytes, s.article));
|
||||||
|
if let Some(t) = s.posted_at {
|
||||||
|
if fragment.posted_at.is_none_or(|p| t < p) {
|
||||||
|
fragment.posted_at = Some(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.batch.fragments.push(fragment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count(&mut self, rule: &'static str, key: String, articles: u64, files: u64) {
|
||||||
|
match self.counts.get_mut(&(rule, key.clone())) {
|
||||||
|
Some(c) => {
|
||||||
|
c.0 += articles;
|
||||||
|
c.1 += files;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
self.count_order.push((rule, key.clone()));
|
||||||
|
self.counts.insert((rule, key), (articles, files));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_key_of(s: &Seg) -> String {
|
||||||
|
// A named file always has a release (releaseIdentity isn't null unless weak).
|
||||||
|
s.file.release.key.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn posted_of(segments: &[Seg]) -> Option<i64> {
|
||||||
|
segments.iter().filter_map(|s| s.posted_at).min()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chunk_of(key: &str, files: &[Vec<Seg>]) -> ReleaseChunk {
|
||||||
|
let first = &files[0][0].file;
|
||||||
|
let chunk_files: Vec<ChunkFile> = files
|
||||||
|
.iter()
|
||||||
|
.map(|f| {
|
||||||
|
let m = &f[0].file;
|
||||||
|
ChunkFile { name: &m.name, poster: &m.poster, subject: &m.subject, posted_at: posted_of(f), groups: &m.groups, segments: f.iter().map(|s| (s.number, s.message_id.as_str(), s.bytes)).collect() }
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut segments = 0;
|
||||||
|
let mut bytes = 0;
|
||||||
|
let mut first_article = i64::MAX;
|
||||||
|
let mut last_article = 0;
|
||||||
|
let mut posted_at: Option<i64> = None;
|
||||||
|
for f in files {
|
||||||
|
for s in f {
|
||||||
|
segments += 1;
|
||||||
|
bytes += s.bytes;
|
||||||
|
first_article = first_article.min(s.article);
|
||||||
|
last_article = last_article.max(s.article);
|
||||||
|
}
|
||||||
|
if let Some(t) = posted_of(f) {
|
||||||
|
if posted_at.is_none_or(|p| t < p) {
|
||||||
|
posted_at = Some(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReleaseChunk {
|
||||||
|
key: key.to_string(),
|
||||||
|
name: first.release.name.clone(),
|
||||||
|
source: first.release.source,
|
||||||
|
file_total: files.iter().find_map(|f| f[0].file.release.file_total_truthy()),
|
||||||
|
poster: first.poster.clone(),
|
||||||
|
groups: first.groups.clone(),
|
||||||
|
posted_at,
|
||||||
|
files: files.len(),
|
||||||
|
segments,
|
||||||
|
bytes,
|
||||||
|
first_article,
|
||||||
|
last_article,
|
||||||
|
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
||||||
|
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
||||||
|
payload: encode_chunk(&chunk_files),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn candidates_of(key: &str, files: &[Vec<Seg>]) -> Vec<Candidate> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut tail: Option<(i64, &Vec<Seg>)> = None;
|
||||||
|
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
||||||
|
for f in files {
|
||||||
|
let name = &f[0].file.name;
|
||||||
|
if let Some(volume) = names::seven_zip_volume(name).filter(|&v| v > 1) {
|
||||||
|
if tail.is_none_or(|(v, _)| volume > v) {
|
||||||
|
tail = Some((volume, f));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if names::is_archive_start(name) {
|
||||||
|
out.push(c("archive", &f[0]));
|
||||||
|
} else if names::is_par2(name) && !names::is_later_volume(name) {
|
||||||
|
out.push(c("par2", &f[0]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some((_, f)) = tail {
|
||||||
|
out.push(Candidate { release_key: key.to_string(), kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
|
||||||
|
}
|
||||||
|
if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
|
||||||
|
out.push(c("lone", &files[0][0]));
|
||||||
|
}
|
||||||
|
out.truncate(4);
|
||||||
|
out
|
||||||
|
}
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
//! JavaScript semantics the TypeScript worker relies on: `\s` and `trim()`,
|
||||||
|
//! UTF-16 lengths, `parseInt`, `Date.parse` (for Usenet dates) and sha1 hex.
|
||||||
|
|
||||||
|
use sha1::{Digest, Sha1};
|
||||||
|
|
||||||
|
/// JS `\s` / `trim()` whitespace (WhiteSpace and LineTerminator).
|
||||||
|
pub fn is_space(c: char) -> bool {
|
||||||
|
matches!(
|
||||||
|
c,
|
||||||
|
'\t' | '\n' | '\u{0B}' | '\u{0C}' | '\r' | ' ' | '\u{A0}' | '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{2028}' | '\u{2029}' | '\u{202F}' | '\u{205F}' | '\u{3000}' | '\u{FEFF}'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn trim(s: &str) -> &str {
|
||||||
|
s.trim_matches(is_space)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `s.replace(/\s+/g, ' ')`.
|
||||||
|
pub fn collapse_spaces(s: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(s.len());
|
||||||
|
let mut in_space = false;
|
||||||
|
for c in s.chars() {
|
||||||
|
if is_space(c) {
|
||||||
|
if !in_space {
|
||||||
|
out.push(' ');
|
||||||
|
in_space = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
out.push(c);
|
||||||
|
in_space = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `s.split(/\s+/)`: like JS, a leading or trailing run gives an empty word.
|
||||||
|
pub fn split_spaces(s: &str) -> Vec<&str> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut start = 0;
|
||||||
|
let mut in_space = false;
|
||||||
|
let mut run_start = 0;
|
||||||
|
for (i, c) in s.char_indices() {
|
||||||
|
if is_space(c) {
|
||||||
|
if !in_space {
|
||||||
|
run_start = i;
|
||||||
|
in_space = true;
|
||||||
|
}
|
||||||
|
} else if in_space {
|
||||||
|
out.push(&s[start..run_start]);
|
||||||
|
start = i;
|
||||||
|
in_space = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if in_space {
|
||||||
|
out.push(&s[start..run_start]);
|
||||||
|
out.push("");
|
||||||
|
} else {
|
||||||
|
out.push(&s[start..]);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A string's `.length` in JavaScript (UTF-16 code units).
|
||||||
|
pub fn len16(s: &str) -> usize {
|
||||||
|
if s.is_ascii() {
|
||||||
|
s.len()
|
||||||
|
} else {
|
||||||
|
s.encode_utf16().count()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Number.parseInt(s, 10)`, None for NaN.
|
||||||
|
pub fn parse_int(s: &str) -> Option<i64> {
|
||||||
|
let s = s.trim_start_matches(is_space);
|
||||||
|
let (neg, digits) = match s.as_bytes().first() {
|
||||||
|
Some(b'-') => (true, &s[1..]),
|
||||||
|
Some(b'+') => (false, &s[1..]),
|
||||||
|
_ => (false, s),
|
||||||
|
};
|
||||||
|
let mut n: i64 = 0;
|
||||||
|
let mut any = false;
|
||||||
|
for b in digits.bytes() {
|
||||||
|
if !b.is_ascii_digit() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
n = n.wrapping_mul(10).wrapping_add((b - b'0') as i64);
|
||||||
|
any = true;
|
||||||
|
}
|
||||||
|
if any {
|
||||||
|
Some(if neg { -n } else { n })
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Days since 1970-01-01 of a proleptic Gregorian date (month 1..=12; a day past the month's end runs on, as in JS).
|
||||||
|
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
||||||
|
let y = if m <= 2 { y - 1 } else { y };
|
||||||
|
let era = y.div_euclid(400);
|
||||||
|
let yoe = y - era * 400;
|
||||||
|
let mp = (m + 9) % 12;
|
||||||
|
let doy = (153 * mp + 2) / 5 + d - 1;
|
||||||
|
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
||||||
|
era * 146_097 + doe - 719_468
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The usual Date headers: [Day, ]D[D] Mon YY[YY] HH:MM:SS (GMT|UTC|UT|±hhmm). None for anything else.
|
||||||
|
fn fast_date(s: &[u8]) -> Option<i64> {
|
||||||
|
let mut i = 0;
|
||||||
|
if s.len() > 4 && s[3] == b',' && s[..3].iter().all(u8::is_ascii_alphabetic) {
|
||||||
|
i = 4;
|
||||||
|
while s.get(i) == Some(&b' ') {
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let num = |i: &mut usize, min: usize, max: usize| -> Option<i64> {
|
||||||
|
let start = *i;
|
||||||
|
while *i < s.len() && s[*i].is_ascii_digit() && *i - start < max {
|
||||||
|
*i += 1;
|
||||||
|
}
|
||||||
|
if *i - start < min || s.get(*i).is_some_and(u8::is_ascii_digit) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(std::str::from_utf8(&s[start..*i]).ok()?.parse().ok()?)
|
||||||
|
};
|
||||||
|
let expect = |i: &mut usize, c: u8| -> Option<()> {
|
||||||
|
(s.get(*i) == Some(&c)).then(|| *i += 1)
|
||||||
|
};
|
||||||
|
let day = num(&mut i, 1, 2)?;
|
||||||
|
expect(&mut i, b' ')?;
|
||||||
|
const MONTHS: [&[u8]; 12] = [b"jan", b"feb", b"mar", b"apr", b"may", b"jun", b"jul", b"aug", b"sep", b"oct", b"nov", b"dec"];
|
||||||
|
let mon = s.get(i..i + 3)?;
|
||||||
|
let month = MONTHS.iter().position(|m| mon.iter().zip(m.iter()).all(|(a, b)| a.to_ascii_lowercase() == *b))? as i64 + 1;
|
||||||
|
i += 3;
|
||||||
|
expect(&mut i, b' ')?;
|
||||||
|
let year_start = i;
|
||||||
|
let mut year = num(&mut i, 2, 4)?;
|
||||||
|
match i - year_start {
|
||||||
|
2 => year += if year < 50 { 2000 } else { 1900 },
|
||||||
|
4 => {}
|
||||||
|
_ => return None,
|
||||||
|
}
|
||||||
|
expect(&mut i, b' ')?;
|
||||||
|
let hour = num(&mut i, 2, 2)?;
|
||||||
|
expect(&mut i, b':')?;
|
||||||
|
let minute = num(&mut i, 2, 2)?;
|
||||||
|
expect(&mut i, b':')?;
|
||||||
|
let second = num(&mut i, 2, 2)?;
|
||||||
|
expect(&mut i, b' ')?;
|
||||||
|
if day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let offset_minutes = match &s[i..] {
|
||||||
|
b"GMT" | b"UTC" | b"UT" => 0,
|
||||||
|
[sign @ (b'+' | b'-'), h1, h2, m1, m2] if [h1, h2, m1, m2].iter().all(|c| c.is_ascii_digit()) => {
|
||||||
|
let minutes = ((h1 - b'0') as i64 * 10 + (h2 - b'0') as i64) * 60 + (m1 - b'0') as i64 * 10 + (m2 - b'0') as i64;
|
||||||
|
if *sign == b'+' { minutes } else { -minutes }
|
||||||
|
}
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let days = days_from_civil(year, month, day);
|
||||||
|
Some((((days * 24 + hour) * 60 + minute - offset_minutes) * 60 + second) * 1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Date.parse` for the Date header (RFC 5322 and the obsolete forms); None for NaN.
|
||||||
|
pub fn parse_date(s: &str) -> Option<i64> {
|
||||||
|
let s = trim(s);
|
||||||
|
if let Some(t) = fast_date(s.as_bytes()) {
|
||||||
|
return Some(t);
|
||||||
|
}
|
||||||
|
parse_date_chrono(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_date_chrono(s: &str) -> Option<i64> {
|
||||||
|
// "UTC" isn't an RFC 5322 zone (UT and GMT are), but Date.parse takes it (~a quarter of a.b.boneless).
|
||||||
|
let owned;
|
||||||
|
let s = match s.strip_suffix(" UTC") {
|
||||||
|
Some(rest) => {
|
||||||
|
owned = format!("{rest} GMT");
|
||||||
|
owned.as_str()
|
||||||
|
}
|
||||||
|
None => s,
|
||||||
|
};
|
||||||
|
if let Ok(d) = chrono::DateTime::parse_from_rfc2822(s) {
|
||||||
|
return Some(d.timestamp_millis());
|
||||||
|
}
|
||||||
|
// "Mon, 5 Oct 2026 10:00:00 +0000 (UTC)" and other trailing comments.
|
||||||
|
if let Some(i) = s.find(" (") {
|
||||||
|
if let Ok(d) = chrono::DateTime::parse_from_rfc2822(&s[..i]) {
|
||||||
|
return Some(d.timestamp_millis());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// sha1 of the UTF-8 text, as lowercase hex (node's `hash('sha1', text)`).
|
||||||
|
pub fn sha1_hex(text: &str) -> String {
|
||||||
|
let digest = Sha1::digest(text.as_bytes());
|
||||||
|
let mut out = String::with_capacity(40);
|
||||||
|
for b in digest {
|
||||||
|
out.push(char::from_digit((b >> 4) as u32, 16).unwrap());
|
||||||
|
out.push(char::from_digit((b & 15) as u32, 16).unwrap());
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON.stringify of a string.
|
||||||
|
pub fn json_str(s: &str, out: &mut String) {
|
||||||
|
out.push('"');
|
||||||
|
for c in s.chars() {
|
||||||
|
match c {
|
||||||
|
'"' => out.push_str("\\\""),
|
||||||
|
'\\' => out.push_str("\\\\"),
|
||||||
|
'\u{08}' => out.push_str("\\b"),
|
||||||
|
'\u{0C}' => out.push_str("\\f"),
|
||||||
|
'\n' => out.push_str("\\n"),
|
||||||
|
'\r' => out.push_str("\\r"),
|
||||||
|
'\t' => out.push_str("\\t"),
|
||||||
|
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
|
||||||
|
c => out.push(c),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push('"');
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn split_like_js() {
|
||||||
|
assert_eq!(split_spaces("a b"), vec!["a", "b"]);
|
||||||
|
assert_eq!(split_spaces(" a b "), vec!["", "a", "b", ""]);
|
||||||
|
assert_eq!(split_spaces(""), vec![""]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_int_like_js() {
|
||||||
|
assert_eq!(parse_int("123abc"), Some(123));
|
||||||
|
assert_eq!(parse_int(" -5"), Some(-5));
|
||||||
|
assert_eq!(parse_int("x"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dates() {
|
||||||
|
assert_eq!(parse_date("Mon, 05 Oct 2026 10:00:00 +0000"), Some(1_791_194_400_000));
|
||||||
|
assert_eq!(parse_date("5 Oct 2026 10:00:00 GMT"), Some(1_791_194_400_000));
|
||||||
|
assert_eq!(parse_date("Mon, 05 Oct 26 10:00:00 UTC"), Some(1_791_194_400_000));
|
||||||
|
for d in [
|
||||||
|
"Mon, 05 Oct 2026 10:00:00 +0000", "Mon, 5 Oct 2026 12:00:00 +0200", "05 Oct 2026 10:00:00 GMT", "Thu, 01 Jan 1970 00:00:00 GMT",
|
||||||
|
"Sat, 29 Feb 2020 23:59:59 -0130", "Fri, 31 Dec 99 00:00:00 UTC", "Mon, 05 Oct 2026 10:00:00 UT",
|
||||||
|
] {
|
||||||
|
assert_eq!(fast_date(d.as_bytes()), parse_date_chrono(d), "{d}");
|
||||||
|
}
|
||||||
|
assert_eq!(fast_date(b"Mon, 05 Oct 2026 10:00:00 EST"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
//! The edge worker's indexing in Rust, on a range captured by `edge:bench
|
||||||
|
//! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as
|
||||||
|
//! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off
|
||||||
|
//! idle ones, and every N chunks judge and encode the releases. `--dump`
|
||||||
|
//! writes every output as JSON lines for tools/edge-bench-rs/check.ts.
|
||||||
|
//!
|
||||||
|
//! edge-bench-rs <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl]
|
||||||
|
|
||||||
|
mod codec;
|
||||||
|
mod indexer;
|
||||||
|
mod js;
|
||||||
|
mod names;
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
use indexer::{Batch, RangeIndexer};
|
||||||
|
use js::json_str;
|
||||||
|
use rustc_hash::FxHashMap;
|
||||||
|
use std::io::Write;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
// Many small strings (message-ids, names): the system allocator was a measurable share.
|
||||||
|
#[global_allocator]
|
||||||
|
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||||
|
|
||||||
|
/// A chunk as the news server sent it (zlib; gzip also taken), whole: libdeflate wants the output
|
||||||
|
/// size, so it grows until the text fits (~3.6 times the compressed size on boneless).
|
||||||
|
fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Vec<u8> {
|
||||||
|
let mut out = vec![0u8; raw.len() * 4 + 4096];
|
||||||
|
loop {
|
||||||
|
let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) };
|
||||||
|
match result {
|
||||||
|
Ok(n) => {
|
||||||
|
out.truncate(n);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0),
|
||||||
|
Err(e) => panic!("inflate: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_field(json: &str, name: &str) -> String {
|
||||||
|
let at = json.find(&format!("\"{name}\":")).unwrap() + name.len() + 3;
|
||||||
|
let rest = json[at..].trim_start();
|
||||||
|
if let Some(s) = rest.strip_prefix('"') {
|
||||||
|
s[..s.find('"').unwrap()].to_string()
|
||||||
|
} else {
|
||||||
|
rest[..rest.find([',', '}']).unwrap()].trim().to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn peak_rss_mb() -> u64 {
|
||||||
|
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
|
||||||
|
status.lines().find(|l| l.starts_with("VmHWM:")).and_then(|l| l.split_whitespace().nth(1)).and_then(|k| k.parse::<u64>().ok()).unwrap_or(0) / 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
fn opt_num(n: Option<i64>, out: &mut String) {
|
||||||
|
match n {
|
||||||
|
Some(n) => out.push_str(&n.to_string()),
|
||||||
|
None => out.push_str("null"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||||
|
let b64 = base64::engine::general_purpose::STANDARD;
|
||||||
|
let mut line = String::new();
|
||||||
|
let mut emit = |line: &mut String| {
|
||||||
|
line.push('\n');
|
||||||
|
out.write_all(line.as_bytes()).unwrap();
|
||||||
|
line.clear();
|
||||||
|
};
|
||||||
|
line.push_str(&format!("{{\"t\":\"b\",\"from\":{},\"next\":{},\"headers\":{}}}", batch.from, batch.next, batch.headers));
|
||||||
|
emit(&mut line);
|
||||||
|
let release = |r: &indexer::ReleaseChunk, kind: &str, line: &mut String| {
|
||||||
|
line.push_str(&format!("{{\"t\":\"{kind}\",\"key\":"));
|
||||||
|
json_str(&r.key, line);
|
||||||
|
line.push_str(",\"name\":");
|
||||||
|
json_str(&r.name, line);
|
||||||
|
line.push_str(&format!(",\"source\":\"{}\",\"fileTotal\":", r.source));
|
||||||
|
opt_num(r.file_total, line);
|
||||||
|
line.push_str(",\"poster\":");
|
||||||
|
json_str(&r.poster, line);
|
||||||
|
line.push_str(",\"groups\":[");
|
||||||
|
for (i, g) in r.groups.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
line.push(',');
|
||||||
|
}
|
||||||
|
json_str(g, line);
|
||||||
|
}
|
||||||
|
line.push_str("],\"postedAt\":");
|
||||||
|
opt_num(r.posted_at, line);
|
||||||
|
line.push_str(&format!(
|
||||||
|
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
|
||||||
|
r.files,
|
||||||
|
r.segments,
|
||||||
|
r.bytes,
|
||||||
|
r.first_article,
|
||||||
|
r.last_article,
|
||||||
|
r.readable,
|
||||||
|
r.has_par2,
|
||||||
|
b64.encode(&r.payload)
|
||||||
|
));
|
||||||
|
};
|
||||||
|
for r in &batch.releases {
|
||||||
|
release(r, "r", &mut line);
|
||||||
|
line.push('}');
|
||||||
|
emit(&mut line);
|
||||||
|
}
|
||||||
|
for (r, rule, key) in &batch.samples {
|
||||||
|
release(r, "s", &mut line);
|
||||||
|
line.push_str(&format!(",\"rule\":\"{rule}\",\"ruleKey\":"));
|
||||||
|
json_str(key, &mut line);
|
||||||
|
line.push('}');
|
||||||
|
emit(&mut line);
|
||||||
|
}
|
||||||
|
for c in &batch.candidates {
|
||||||
|
line.push_str("{\"t\":\"c\",\"releaseKey\":");
|
||||||
|
json_str(&c.release_key, &mut line);
|
||||||
|
line.push_str(&format!(",\"kind\":\"{}\",\"messageId\":", c.kind));
|
||||||
|
json_str(&c.message_id, &mut line);
|
||||||
|
line.push_str(",\"file\":");
|
||||||
|
json_str(&c.file, &mut line);
|
||||||
|
line.push('}');
|
||||||
|
emit(&mut line);
|
||||||
|
}
|
||||||
|
for (rule, key, articles, files) in &batch.counts {
|
||||||
|
line.push_str(&format!("{{\"t\":\"n\",\"rule\":\"{rule}\",\"key\":"));
|
||||||
|
json_str(key, &mut line);
|
||||||
|
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
|
||||||
|
emit(&mut line);
|
||||||
|
}
|
||||||
|
for f in &batch.fragments {
|
||||||
|
line.push_str("{\"t\":\"f\",\"fileKey\":");
|
||||||
|
json_str(&f.file_key, &mut line);
|
||||||
|
line.push_str(",\"name\":");
|
||||||
|
json_str(&f.name, &mut line);
|
||||||
|
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
|
||||||
|
opt_num(f.posted_at, &mut line);
|
||||||
|
line.push_str(",\"segments\":[");
|
||||||
|
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
line.push(',');
|
||||||
|
}
|
||||||
|
line.push_str(&format!("[{n},"));
|
||||||
|
json_str(id, &mut line);
|
||||||
|
line.push_str(&format!(",{bytes},{article}]"));
|
||||||
|
}
|
||||||
|
line.push_str("]}");
|
||||||
|
emit(&mut line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let args: Vec<String> = std::env::args().collect();
|
||||||
|
let dir = args.get(1).expect("usage: edge-bench-rs <dir> [--batch-chunks N] [--idle N] [--dump file]").clone();
|
||||||
|
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||||
|
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||||
|
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
|
||||||
|
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||||
|
|
||||||
|
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
|
||||||
|
let group = json_field(&range, "group");
|
||||||
|
let start: i64 = json_field(&range, "start").parse().unwrap();
|
||||||
|
let end: i64 = json_field(&range, "end").parse().unwrap();
|
||||||
|
let mut files: Vec<String> = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||||
|
files.sort();
|
||||||
|
|
||||||
|
let mut priorities = FxHashMap::default();
|
||||||
|
priorities.insert(group.clone(), 1);
|
||||||
|
let mut indexer = RangeIndexer::new(&group, start, end, priorities, idle);
|
||||||
|
|
||||||
|
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||||
|
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||||
|
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
|
||||||
|
let mut keys = std::collections::HashSet::new();
|
||||||
|
let mut decompressor = libdeflater::Decompressor::new();
|
||||||
|
let wall = Instant::now();
|
||||||
|
for (i, file) in files.iter().enumerate() {
|
||||||
|
let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
|
||||||
|
compressed += raw.len();
|
||||||
|
let t = Instant::now();
|
||||||
|
let data = inflate(&raw, &mut decompressor);
|
||||||
|
inflate_s += t.elapsed().as_secs_f64();
|
||||||
|
text_bytes += data.len();
|
||||||
|
let t = Instant::now();
|
||||||
|
let text = String::from_utf8_lossy(&data);
|
||||||
|
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||||
|
indexer.add_text(&text, next);
|
||||||
|
let last = i + 1 == files.len();
|
||||||
|
if last {
|
||||||
|
indexer.finish();
|
||||||
|
}
|
||||||
|
if !last && (i + 1) % batch_chunks != 0 {
|
||||||
|
index_s += t.elapsed().as_secs_f64();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let batch = indexer.take();
|
||||||
|
index_s += t.elapsed().as_secs_f64();
|
||||||
|
out.0 += batch.headers;
|
||||||
|
out.1 += batch.releases.len();
|
||||||
|
for r in &batch.releases {
|
||||||
|
keys.insert(r.key.clone());
|
||||||
|
out.2 += r.files;
|
||||||
|
out.3 += r.segments;
|
||||||
|
out.4 += r.payload.len();
|
||||||
|
}
|
||||||
|
out.5 += batch.candidates.len();
|
||||||
|
out.6 += batch.fragments.len();
|
||||||
|
out.7 += batch.samples.len();
|
||||||
|
if let Some(d) = dump.as_mut() {
|
||||||
|
dump_batch(&batch, d);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let total = inflate_s + index_s;
|
||||||
|
#[allow(static_mut_refs)]
|
||||||
|
let p = unsafe { indexer::PROF };
|
||||||
|
eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
|
||||||
|
println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
|
||||||
|
println!(
|
||||||
|
"{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",
|
||||||
|
out.0,
|
||||||
|
total,
|
||||||
|
out.0 as f64 / total,
|
||||||
|
wall.elapsed().as_secs_f64()
|
||||||
|
);
|
||||||
|
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
|
||||||
|
println!(
|
||||||
|
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
|
||||||
|
out.1,
|
||||||
|
keys.len(),
|
||||||
|
out.2,
|
||||||
|
out.3,
|
||||||
|
out.4 as f64 / 1e6,
|
||||||
|
out.4 as f64 / out.3.max(1) as f64,
|
||||||
|
out.5,
|
||||||
|
out.6,
|
||||||
|
out.7
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,739 @@
|
|||||||
|
//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools.
|
||||||
|
//! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u`
|
||||||
|
//! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS
|
||||||
|
//! `\s` (which includes Unicode spaces) is spelled out as SP.
|
||||||
|
|
||||||
|
use crate::js::{self, len16, sha1_hex};
|
||||||
|
use md5::{Digest, Md5};
|
||||||
|
use regex::bytes::Regex;
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
/// JS `\s` as a byte pattern.
|
||||||
|
const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)";
|
||||||
|
|
||||||
|
fn re(pattern: &str) -> Regex {
|
||||||
|
Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all).
|
||||||
|
fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String {
|
||||||
|
let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) };
|
||||||
|
// The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8.
|
||||||
|
String::from_utf8(out.into_owned()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test(re: &Regex, s: &str) -> bool {
|
||||||
|
re.is_match(s.as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||||
|
vec![
|
||||||
|
(re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""),
|
||||||
|
(re(r"(?i)\.par2$"), ""),
|
||||||
|
(re(r"(?i)\.part[0-9]+\.rar$"), ""),
|
||||||
|
(re(r"(?i)\.r[0-9]{2,3}$"), ""),
|
||||||
|
(re(r"(?i)\.rar$"), ""),
|
||||||
|
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||||||
|
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||||||
|
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||||||
|
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.[0-9]{3}$"), "$1"),
|
||||||
|
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||||||
|
(re(r"(?i)\.tar$"), ""),
|
||||||
|
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||||||
|
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
/// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn file_stem_re(name: &str) -> String {
|
||||||
|
let mut stem = js::trim(name).to_string();
|
||||||
|
let mut changed = true;
|
||||||
|
while changed {
|
||||||
|
changed = false;
|
||||||
|
for (suffix, with) in SUFFIXES.iter() {
|
||||||
|
if !suffix.is_match(stem.as_bytes()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let next = replace(suffix, &stem, with, false);
|
||||||
|
if next != stem && !next.is_empty() {
|
||||||
|
stem = next;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stem
|
||||||
|
}
|
||||||
|
|
||||||
|
static QUOTED: LazyLock<Regex> = LazyLock::new(|| re(r#""[^"]*""#));
|
||||||
|
static COUNTERS: LazyLock<Regex> = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)"));
|
||||||
|
static YENC_END: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$"));
|
||||||
|
static YENC: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b"));
|
||||||
|
static SEPARATORS: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+"));
|
||||||
|
|
||||||
|
/// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn subject_remainder_re(subject: &str) -> String {
|
||||||
|
let s = replace("ED, subject, " ", true);
|
||||||
|
let s = replace(&COUNTERS, &s, " ", true);
|
||||||
|
let s = replace(&YENC_END, &s, " ", false);
|
||||||
|
let s = replace(&YENC, &s, " ", true);
|
||||||
|
let s = replace(&SEPARATORS, &s, " ", true);
|
||||||
|
js::trim(&s).to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn eq_ci(a: &[u8], b: &[u8]) -> bool {
|
||||||
|
a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ends in `suffix` (lowercase), ASCII case-insensitively.
|
||||||
|
fn ends_ci(s: &[u8], suffix: &[u8]) -> bool {
|
||||||
|
s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ASCII digits ending at `end`.
|
||||||
|
fn digits_back(s: &[u8], end: usize) -> usize {
|
||||||
|
s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `.` + one of `exts` at the end: the length without it.
|
||||||
|
fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option<usize> {
|
||||||
|
for ext in exts {
|
||||||
|
let n = ext.len() + 1;
|
||||||
|
if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) {
|
||||||
|
return Some(s.len() - n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `\.\d{3}$`: the length without it.
|
||||||
|
fn strip_three_digits(s: &[u8]) -> Option<usize> {
|
||||||
|
let n = s.len();
|
||||||
|
(n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||||||
|
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||||||
|
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||||||
|
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||||||
|
|
||||||
|
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||||||
|
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||||||
|
let n = s.len();
|
||||||
|
match rule {
|
||||||
|
// \.vol\d+[+-]\d+\.par2$
|
||||||
|
0 => {
|
||||||
|
if !ends_ci(s, b".par2") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let d2 = digits_back(s, n - 5);
|
||||||
|
let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?;
|
||||||
|
if !matches!(s[sign], b'+' | b'-') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let d1 = digits_back(s, sign);
|
||||||
|
let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?;
|
||||||
|
eq_ci(&s[v..v + 4], b".vol").then_some(v)
|
||||||
|
}
|
||||||
|
1 => ends_ci(s, b".par2").then(|| n - 5),
|
||||||
|
// \.part\d+\.rar$
|
||||||
|
2 => {
|
||||||
|
if !ends_ci(s, b".rar") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let d = digits_back(s, n - 4);
|
||||||
|
let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?;
|
||||||
|
eq_ci(&s[p..p + 5], b".part").then_some(p)
|
||||||
|
}
|
||||||
|
// \.r\d{2,3}$
|
||||||
|
3 => {
|
||||||
|
let d = digits_back(s, n);
|
||||||
|
let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?;
|
||||||
|
eq_ci(&s[p..p + 2], b".r").then_some(p)
|
||||||
|
}
|
||||||
|
4 => ends_ci(s, b".rar").then(|| n - 4),
|
||||||
|
// \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$
|
||||||
|
5 | 6 => {
|
||||||
|
let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" };
|
||||||
|
if let Some(m) = strip_three_digits(s) {
|
||||||
|
if ends_ci(&s[..m], ext) {
|
||||||
|
return Some(m - ext.len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ends_ci(s, ext).then(|| n - ext.len())
|
||||||
|
}
|
||||||
|
// (?<=\.(7z|…))\.\d{3}$: only the volume number goes.
|
||||||
|
7 => {
|
||||||
|
let m = strip_three_digits(s)?;
|
||||||
|
strip_ext(&s[..m], &SPLIT_EXTS).map(|_| m)
|
||||||
|
}
|
||||||
|
8 => strip_ext(s, &SIDE_EXTS),
|
||||||
|
9 => strip_ext(s, &[b"tar"]),
|
||||||
|
10 => strip_ext(s, &PACK_EXTS),
|
||||||
|
11 => strip_ext(s, &MEDIA_EXTS),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a
|
||||||
|
/// file's description was most of the indexing time.
|
||||||
|
pub fn file_stem(name: &str) -> String {
|
||||||
|
let stem = js::trim(name);
|
||||||
|
let b = stem.as_bytes();
|
||||||
|
let mut len = b.len();
|
||||||
|
let mut changed = true;
|
||||||
|
while changed {
|
||||||
|
changed = false;
|
||||||
|
for rule in 0..12 {
|
||||||
|
if let Some(next) = strip_suffix_rule(&b[..len], rule) {
|
||||||
|
if next > 0 && next < len {
|
||||||
|
len = next;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The suffixes are ASCII: `len` is a character boundary.
|
||||||
|
stem[..len].to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_word(c: u8) -> bool {
|
||||||
|
c.is_ascii_alphanumeric() || c == b'_'
|
||||||
|
}
|
||||||
|
|
||||||
|
/// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides.
|
||||||
|
fn yenc_at(b: &[u8], i: usize) -> bool {
|
||||||
|
i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4]))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(\s+\d+)?\s*$`
|
||||||
|
fn yenc_tail(rest: &str) -> bool {
|
||||||
|
let after_spaces = rest.trim_start_matches(js::is_space);
|
||||||
|
if after_spaces.chars().all(js::is_space) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if after_spaces.len() == rest.len() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit());
|
||||||
|
after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// subjectRemainder by hand: the same five replacements and trim.
|
||||||
|
pub fn subject_remainder(subject: &str) -> String {
|
||||||
|
let b = subject.as_bytes();
|
||||||
|
// "[^"]*" → ' '
|
||||||
|
let mut s1: Vec<u8> = Vec::with_capacity(b.len());
|
||||||
|
let mut i = 0;
|
||||||
|
while i < b.len() {
|
||||||
|
if b[i] == b'"' {
|
||||||
|
if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') {
|
||||||
|
s1.push(b' ');
|
||||||
|
i += j + 2;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s1.push(b[i]);
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
// \[\d+/\d+\] and \(\d+/\d+\) → ' '
|
||||||
|
let mut s2: Vec<u8> = Vec::with_capacity(s1.len());
|
||||||
|
let mut i = 0;
|
||||||
|
while i < s1.len() {
|
||||||
|
let close = match s1[i] {
|
||||||
|
b'[' => Some(b']'),
|
||||||
|
b'(' => Some(b')'),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
if let Some(close) = close {
|
||||||
|
let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
let slash = i + 1 + a;
|
||||||
|
if a > 0 && s1.get(slash) == Some(&b'/') {
|
||||||
|
let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
if d > 0 && s1.get(slash + 1 + d) == Some(&close) {
|
||||||
|
s2.push(b' ');
|
||||||
|
i = slash + 2 + d;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s2.push(s1[i]);
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
// Only ASCII was replaced, between whole characters.
|
||||||
|
let mut s3 = String::from_utf8(s2).unwrap();
|
||||||
|
// \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits)
|
||||||
|
let b3 = s3.as_bytes();
|
||||||
|
if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) {
|
||||||
|
s3.truncate(at);
|
||||||
|
s3.push(' ');
|
||||||
|
}
|
||||||
|
// \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' '
|
||||||
|
let b3 = s3.as_bytes();
|
||||||
|
let mut s4 = String::with_capacity(s3.len());
|
||||||
|
let mut separator = false;
|
||||||
|
let mut i = 0;
|
||||||
|
while i < b3.len() {
|
||||||
|
if yenc_at(b3, i) {
|
||||||
|
// Its " " starts or joins a separator run.
|
||||||
|
if !separator {
|
||||||
|
s4.push(' ');
|
||||||
|
separator = true;
|
||||||
|
}
|
||||||
|
i += 4;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let c = s3[i..].chars().next().unwrap();
|
||||||
|
if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') {
|
||||||
|
if !separator {
|
||||||
|
s4.push(' ');
|
||||||
|
separator = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s4.push(c);
|
||||||
|
separator = false;
|
||||||
|
}
|
||||||
|
i += c.len_utf8();
|
||||||
|
}
|
||||||
|
js::trim(&s4).to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_random_token(token: &str) -> bool {
|
||||||
|
let t = token.strip_prefix('[').unwrap_or(token);
|
||||||
|
let t = t.strip_suffix(']').unwrap_or(t);
|
||||||
|
if len16(t) < 6 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let digit = t.bytes().any(|b| b.is_ascii_digit());
|
||||||
|
if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-'))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_post_token(remainder: &str) -> bool {
|
||||||
|
let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect();
|
||||||
|
let n = len16(&compact);
|
||||||
|
(6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
static NAME_LIKE: LazyLock<Regex> = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}"));
|
||||||
|
|
||||||
|
pub fn names_release(remainder: &str) -> bool {
|
||||||
|
let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect();
|
||||||
|
name_like(words.join(" ").as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct Release {
|
||||||
|
pub key: String,
|
||||||
|
pub name: String,
|
||||||
|
pub source: &'static str,
|
||||||
|
/// The [i/M] total as parsed (0 is falsy in the TypeScript).
|
||||||
|
pub file_total: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Release {
|
||||||
|
pub fn file_total_truthy(&self) -> Option<i64> {
|
||||||
|
self.file_total.filter(|&n| n != 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
static FILE_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]"));
|
||||||
|
|
||||||
|
fn hash(parts: &[&str]) -> String {
|
||||||
|
sha1_hex(&parts.join("\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// releaseIdentity for a named (not weak) file: never null.
|
||||||
|
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||||
|
let file_total = file_counter_total(subject);
|
||||||
|
let stem = file_stem(name);
|
||||||
|
let remainder = subject_remainder(subject);
|
||||||
|
if names_release(&remainder) {
|
||||||
|
return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total };
|
||||||
|
}
|
||||||
|
if let Some(total) = file_total.filter(|&n| n != 0) {
|
||||||
|
if is_post_token(&remainder) {
|
||||||
|
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total }
|
||||||
|
}
|
||||||
|
|
||||||
|
static GENERIC: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
|
re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$")
|
||||||
|
});
|
||||||
|
|
||||||
|
fn is_generic_name(name: &str) -> bool {
|
||||||
|
test(&GENERIC, js::trim(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// looksRandom.
|
||||||
|
pub fn looks_random(name: &str) -> bool {
|
||||||
|
// name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||||
|
let name = match name.rfind('.') {
|
||||||
|
Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
|
||||||
|
_ => name,
|
||||||
|
};
|
||||||
|
if is_generic_name(name) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let b = name.as_bytes();
|
||||||
|
let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit());
|
||||||
|
let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit());
|
||||||
|
if b.len() >= 16 && hex(b) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if let Some(dot) = name.find('.') {
|
||||||
|
let (h, d) = (&b[..dot], &b[dot + 1..]);
|
||||||
|
if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let core = name.strip_suffix(" n").unwrap_or(name).as_bytes();
|
||||||
|
if core.len() >= 6 && digits(core) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count();
|
||||||
|
let n_digit = count(u8::is_ascii_digit);
|
||||||
|
let n_lower = count(u8::is_ascii_lowercase);
|
||||||
|
let n_upper = count(u8::is_ascii_uppercase);
|
||||||
|
let alnum = b.iter().all(|c| c.is_ascii_alphanumeric());
|
||||||
|
if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
n_digit > 0 || n_upper >= 6
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_random_file(name: &str) -> bool {
|
||||||
|
looks_random(&file_stem(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
|
||||||
|
static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||||
|
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||||
|
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||||||
|
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||||||
|
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||||||
|
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||||||
|
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||||||
|
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||||||
|
|
||||||
|
pub fn is_par2(name: &str) -> bool {
|
||||||
|
test(&PAR2, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn seven_zip_volume(name: &str) -> Option<i64> {
|
||||||
|
if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) {
|
||||||
|
return js::parse_int(std::str::from_utf8(&c[1]).unwrap());
|
||||||
|
}
|
||||||
|
if test(&SEVEN, name) {
|
||||||
|
Some(0)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_archive_start(name: &str) -> bool {
|
||||||
|
if test(&BIN, name) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if matches!(seven_zip_volume(name), Some(0) | Some(1)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
test(&PART1, name) || (test(&RAR, name) && !test(&PART, name))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_later_volume(name: &str) -> bool {
|
||||||
|
if let Some(c) = PART.captures(name.as_bytes()) {
|
||||||
|
return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1;
|
||||||
|
}
|
||||||
|
test(&OLD_RAR, name) || test(&PAR2_VOL, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
static TOOLS: LazyLock<Vec<(&'static str, Regex)>> = LazyLock::new(|| {
|
||||||
|
vec![
|
||||||
|
("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")),
|
||||||
|
("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")),
|
||||||
|
("ngPost", re(r"^[0-9a-f]{32}@ngPost$")),
|
||||||
|
("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")),
|
||||||
|
("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")),
|
||||||
|
("JBinUp", re(r"@JBin(?:Up|Down)\.local$")),
|
||||||
|
("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")),
|
||||||
|
("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")),
|
||||||
|
("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")),
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
/// uploadTool with the regexes, as in the TypeScript (the reference for upload_tool's test).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn upload_tool_re(message_id: &str) -> &'static str {
|
||||||
|
if message_id.is_empty() {
|
||||||
|
return "other";
|
||||||
|
}
|
||||||
|
for (tool, pattern) in TOOLS.iter() {
|
||||||
|
if test(pattern, message_id) {
|
||||||
|
return tool;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"other"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn all(s: &[u8], f: impl Fn(u8) -> bool) -> bool {
|
||||||
|
s.iter().all(|&c| f(c))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lower_hex(c: u8) -> bool {
|
||||||
|
c.is_ascii_digit() || (b'a'..=b'f').contains(&c)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(?:[A-Z][a-z]){12}-\d{13}@`: nyuu's prefix (38 bytes, then '@').
|
||||||
|
fn nyuu_prefix(b: &[u8]) -> bool {
|
||||||
|
b.len() > 38 && (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[24] == b'-' && all(&b[25..38], |c| c.is_ascii_digit()) && b[38] == b'@'
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The posting tool of a Message-ID (TOOLS, in order, by hand: they ran for every per-article file).
|
||||||
|
pub fn upload_tool(message_id: &str) -> &'static str {
|
||||||
|
let b = message_id.as_bytes();
|
||||||
|
if b.is_empty() {
|
||||||
|
return "other";
|
||||||
|
}
|
||||||
|
if nyuu_prefix(b) && b.len() > 39 {
|
||||||
|
// nyuu: "@nyuu" exactly; nyuu-host: any host (`.+`: no line terminators).
|
||||||
|
if &b[39..] == b"nyuu" {
|
||||||
|
return "nyuu";
|
||||||
|
}
|
||||||
|
if !b[39..].iter().any(|&c| c == b'\n' || c == b'\r') && !message_id[39..].contains(['\u{2028}', '\u{2029}']) {
|
||||||
|
return "nyuu-host";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let hex32 = b.len() > 33 && all(&b[..32], lower_hex) && b[32] == b'@';
|
||||||
|
if hex32 {
|
||||||
|
let host = &b[33..];
|
||||||
|
if host == b"ngPost" {
|
||||||
|
return "ngPost";
|
||||||
|
}
|
||||||
|
if host.len() == 37 && all(&host[..32], lower_hex) && &host[32..] == b".yuun" {
|
||||||
|
return "yuun";
|
||||||
|
}
|
||||||
|
if all(host, |c| c.is_ascii_alphanumeric() || c == b'.' || c == b'-') {
|
||||||
|
return "hex32";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if message_id.ends_with("@JBinUp.local") || message_id.ends_with("@JBinDown.local") {
|
||||||
|
return "JBinUp";
|
||||||
|
}
|
||||||
|
// (?i)^part\d+of\d+\.
|
||||||
|
if b.len() > 4 && eq_ci(&b[..4], b"part") {
|
||||||
|
let d1 = b[4..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
let of = 4 + d1;
|
||||||
|
if d1 > 0 && b.len() > of + 2 && eq_ci(&b[of..of + 2], b"of") {
|
||||||
|
let d2 = b[of + 2..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
if d2 > 0 && b.get(of + 2 + d2) == Some(&b'.') {
|
||||||
|
return "partNofM";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// ^\d{13}\$\d+\$\d+@localhost$
|
||||||
|
if let Some(rest) = message_id.strip_suffix("@localhost") {
|
||||||
|
let r = rest.as_bytes();
|
||||||
|
let parts: Vec<&[u8]> = r.split(|&c| c == b'$').collect();
|
||||||
|
if parts.len() == 3 && parts[0].len() == 13 && parts.iter().all(|p| !p.is_empty() && all(p, |c| c.is_ascii_digit())) {
|
||||||
|
return "ts-dollar";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// ^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$
|
||||||
|
if let Some(at) = b.iter().position(|&c| c == b'@') {
|
||||||
|
let (local, host) = (&b[..at], &b[at + 1..]);
|
||||||
|
if (10..=40).contains(&local.len()) && all(local, |c| c.is_ascii_alphanumeric()) {
|
||||||
|
if let Some(dot) = host.iter().position(|&c| c == b'.') {
|
||||||
|
let (name, tld) = (&host[..dot], &host[dot + 1..]);
|
||||||
|
if (3..=12).contains(&name.len()) && (2..=6).contains(&tld.len()) && all(name, |c| c.is_ascii_alphanumeric()) && all(tld, |c| c.is_ascii_alphanumeric()) {
|
||||||
|
return "random";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"other"
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,} anywhere.
|
||||||
|
fn name_like(b: &[u8]) -> bool {
|
||||||
|
let sep = |c: u8| matches!(c, b'.' | b'_' | b' ' | b'-');
|
||||||
|
(2..b.len()).any(|k| {
|
||||||
|
if !(sep(b[k]) && b[k - 1].is_ascii_alphabetic() && b[k - 2].is_ascii_alphabetic()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let m = k + b[k..].iter().take_while(|&&c| sep(c)).count();
|
||||||
|
m + 2 <= b.len() && b[m].is_ascii_alphanumeric() && b[m + 1].is_ascii_alphanumeric()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The second number of the first \[(\d+)/(\d+)\].
|
||||||
|
fn file_counter_total(s: &str) -> Option<i64> {
|
||||||
|
let b = s.as_bytes();
|
||||||
|
for (i, &c) in b.iter().enumerate() {
|
||||||
|
if c != b'[' {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let a = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
let slash = i + 1 + a;
|
||||||
|
if a == 0 || b.get(slash) != Some(&b'/') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
|
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
||||||
|
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// explored: a release in the sampled share of a rule's drops.
|
||||||
|
pub fn explored(key: &str, percent: f64) -> bool {
|
||||||
|
if percent <= 0.0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let d = Md5::digest(key.as_bytes());
|
||||||
|
let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]);
|
||||||
|
((n % 10_000) as f64) < percent * 100.0
|
||||||
|
}
|
||||||
|
|
||||||
|
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||||||
|
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||||
|
|
||||||
|
pub struct JudgedFile<'a> {
|
||||||
|
pub name: &'a str,
|
||||||
|
pub total_parts: u32,
|
||||||
|
pub groups: &'a [String],
|
||||||
|
pub first_message_id: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum Verdict {
|
||||||
|
Keep,
|
||||||
|
Drop { rule: &'static str, key: String, sample: bool },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// RuleSet.judge with DEFAULT_RULES (no dropped signatures).
|
||||||
|
pub fn judge(release_key: &str, files: &[JudgedFile]) -> Verdict {
|
||||||
|
const DROPPED_TOOLS: [&str; 5] = ["ngPost", "hex32", "yuun", "JBinUp", "random"];
|
||||||
|
let sample = explored(release_key, 1.0);
|
||||||
|
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||||||
|
if !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||||||
|
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||||||
|
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||||||
|
return Verdict::Drop { rule: "executable", key, sample };
|
||||||
|
}
|
||||||
|
if files.iter().any(|f| !is_random_file(f.name)) {
|
||||||
|
return Verdict::Keep;
|
||||||
|
}
|
||||||
|
let first = &files[0];
|
||||||
|
let tool = upload_tool(first.first_message_id);
|
||||||
|
let group = first.groups.first().map(String::as_str).unwrap_or("");
|
||||||
|
if DROPPED_TOOLS.contains(&tool) {
|
||||||
|
return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample };
|
||||||
|
}
|
||||||
|
if files.iter().all(|f| f.total_parts <= 3) {
|
||||||
|
return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample };
|
||||||
|
}
|
||||||
|
Verdict::Keep
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stems() {
|
||||||
|
assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x");
|
||||||
|
assert_eq!(file_stem("x.part07.rar"), "x");
|
||||||
|
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||||||
|
assert_eq!(file_stem("a.7z.002"), "a");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remainders() {
|
||||||
|
assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release");
|
||||||
|
assert!(names_release("Some Release"));
|
||||||
|
assert!(!names_release("elys de81bcabef5faf6a"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn by_hand_as_the_regexes() {
|
||||||
|
let names = [
|
||||||
|
"x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234",
|
||||||
|
"d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar",
|
||||||
|
"k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar",
|
||||||
|
];
|
||||||
|
for n in names {
|
||||||
|
assert_eq!(file_stem(n), file_stem_re(n), "{n}");
|
||||||
|
}
|
||||||
|
let subjects = [
|
||||||
|
r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#,
|
||||||
|
r#"[1/3] - "a.par2" yEnc (1/1) 123 "#,
|
||||||
|
r#"yEnc "x" yenc yEncx (1/2)"#,
|
||||||
|
r#"Show – S01E01 | 1080p : x "a" "b"#,
|
||||||
|
r#"name (12/3x) [4/] "q" YENC 12 34"#,
|
||||||
|
" lead\u{a0}nbsp yEnc\u{3000}99 ",
|
||||||
|
r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#,
|
||||||
|
"a_yEnc b yEnc_c yEnc",
|
||||||
|
"yEnc yEnc 5",
|
||||||
|
"x yEnc 12 yEnc",
|
||||||
|
"",
|
||||||
|
];
|
||||||
|
for s in subjects {
|
||||||
|
assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tools_by_hand_as_the_regexes() {
|
||||||
|
let ids = [
|
||||||
|
"FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@news.example",
|
||||||
|
"b94268bc702040f7b811d5318f727e4f@ngPost", "b94268bc702040f7b811d5318f727e4f@b94268bc702040f7b811d5318f727e4f.yuun", "b94268bc702040f7b811d5318f727e4f@host-1.example",
|
||||||
|
"B94268bc702040f7b811d5318f727e4f@ngPost", "x@JBinUp.local", "y@JBinDown.local", "part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local", "PART12OF3.x",
|
||||||
|
"1790923856877$1729905757859$1@localhost", "179092385687$1$1@localhost", "t3Y4mV92YCp0McxDc5@zLZlsw.RYqes", "t3Y4mV92YC@zL.RYqes", "t3Y4mV92YCp0McxDc5@zLZlsw.RY.es",
|
||||||
|
"", "a@b", "b94268bc702040f7b811d5318f727e4f@", "b94268bc702040f7b811d5318f727e4f@host_x",
|
||||||
|
];
|
||||||
|
for id in ids {
|
||||||
|
assert_eq!(upload_tool(id), upload_tool_re(id), "{id}");
|
||||||
|
}
|
||||||
|
for t in ["Some Release", "a b", "ab-cd", "ab - c1", "ab.c", "x ab_12", "12 ab", "ab "] {
|
||||||
|
assert_eq!(name_like(t.as_bytes()), test(&NAME_LIKE, t), "{t}");
|
||||||
|
}
|
||||||
|
for t in ["[01/45] x", "x [1/] [2/30]", "[a/1] [3/4]", "no counter"] {
|
||||||
|
let re = FILE_COUNTER.captures(t.as_bytes()).and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap()));
|
||||||
|
assert_eq!(file_counter_total(t), re, "{t}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn random_names() {
|
||||||
|
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||||||
|
assert!(looks_random("RmISakytFwXSCEzjVEpaKk"));
|
||||||
|
assert!(!looks_random("Some.Show.S01E01.1080p"));
|
||||||
|
assert!(looks_random("VIDEO_TS"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tools() {
|
||||||
|
assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu");
|
||||||
|
assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost");
|
||||||
|
assert_eq!(upload_tool("x@y"), "other");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user