The Rust worker is the product: edge/, built and checked in CI
User: Rust is the product (option a). tools/edge-bench-rs moves to edge/
(crate edge; binaries edge-node, the worker, and edge-bench, the replay).
CI's new job edge runs cargo test and edge/check-fixtures.sh: both
RangeIndexers on three committed captures (edge/fixtures, 25k overview
lines each of a.b.models, a.b.ath and a.b.boneless in 5 chunks, 3.5 MB),
compared field by field with check.ts; deploy waits for it. The
TypeScript RangeIndexer stays as check.ts's reference. README: what it
is now; the benchmark findings as history.
Random names: looksRandom missed long tokens of capitals and digits
("51294P1CUKR2R35X", release 74539068, readable and a Newznab title).
12+ capitals and digits switching 6+ times between them are random
(COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice); both
implementations, identical on the fixtures.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
48f4e982f9
commit
5109481d67
38 files changed
+138
-54
No files matched your search
@@ -0,0 +1 @@
|
||||
target/
|
||||
Generated
+959
@@ -0,0 +1,959 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "edge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"flate2",
|
||||
"libdeflater",
|
||||
"md-5",
|
||||
"memchr",
|
||||
"mimalloc",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1",
|
||||
"signal-hook",
|
||||
"ureq",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.190"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||
|
||||
[[package]]
|
||||
name = "libdeflate-sys"
|
||||
version = "1.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libdeflater"
|
||||
version = "1.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f"
|
||||
dependencies = [
|
||||
"libdeflate-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"signal-hook-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"flate2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8-zero"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.1.0+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "edge"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)"
|
||||
|
||||
[[bin]]
|
||||
name = "edge-bench"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "edge-node"
|
||||
path = "src/bin/edge-node.rs"
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.22"
|
||||
chrono = "0.4"
|
||||
flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
|
||||
libdeflater = "1.26.1"
|
||||
md-5 = "0.10"
|
||||
memchr = "2.8.3"
|
||||
mimalloc = "0.1.52"
|
||||
regex = "1"
|
||||
rustc-hash = "2"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
serde_json = "1.0.151"
|
||||
sha1 = "0.10"
|
||||
signal-hook = "0.4.5"
|
||||
ureq = "3.4.2"
|
||||
zstd = "0.13"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
# edge: voyager 2's edge worker (Rust)
|
||||
|
||||
The product: `edge-node` (`src/bin/edge-node.rs`) is the worker that runs
|
||||
on homelab (compose service `edge`, image target `edge`). It leases ranges
|
||||
from the node API, reads their prefetched XOVER chunks, finds releases and
|
||||
posts batches. `edge-bench` (`src/main.rs`) replays a captured range through
|
||||
the same indexing (`src/lib.rs`) for timing and for comparing.
|
||||
|
||||
The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the
|
||||
reference: `check.ts` runs both on a capture and compares every output field
|
||||
by field. An identity or rule change goes into both and has to come out
|
||||
identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test`
|
||||
and `check-fixtures.sh`, which compares both on the committed captures in
|
||||
`fixtures/` (25k overview lines each of a.b.models, a.b.ath and
|
||||
a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they
|
||||
match.
|
||||
|
||||
```sh
|
||||
cargo test --release --manifest-path edge/Cargo.toml
|
||||
edge/check-fixtures.sh # from the repository root
|
||||
```
|
||||
|
||||
In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores
|
||||
and 565 MB, next to the TypeScript workers' 24M with 0.61 cores and 11.3 GB
|
||||
(8 threads vs 8 processes): about 8× less CPU per header and 20× less
|
||||
memory. See "The worker" below.
|
||||
|
||||
It started as a measurement (moved from `tools/edge-bench-rs` on
|
||||
2026-10-11): whether a Rust worker would be worth a second implementation.
|
||||
The findings below are from then.
|
||||
|
||||
## Findings (2026-10-09)
|
||||
|
||||
Same machine (the workstation, otherwise idle), same captures, 2M headers
|
||||
each:
|
||||
|
||||
| Capture | TypeScript (`edge:bench`) | Rust (`edge-bench`) | Less CPU |
|
||||
|---|---|---|---|
|
||||
| a.b.boneless 2026 (160950000000–160951999999) | 11.1 CPU-s (~215k headers/s), 0.70 GB peak | 2.9 CPU-s (~690k/s), 0.24 GB | 3.8× |
|
||||
| a.b.boneless 2026 (160952000000–160953999999) | 11.8 CPU-s (~205k/s), 0.70 GB | 3.1 CPU-s (~640k/s), 0.24 GB | 3.8× |
|
||||
| a.b.ath 2013 (2549992990–2551992989) | 7.5 CPU-s (~320k/s), 0.74 GB | 2.15 CPU-s (~930k/s), 0.20 GB | 3.5× |
|
||||
| a.b.models (2017215–4017214; image sets, re-uploads) | 16.7 CPU-s (~139k/s) | 5.0 CPU-s (~400k/s) | 3.3× |
|
||||
|
||||
**Comparing the numbers:**
|
||||
- **CPU seconds are the fair comparison.** `edge:bench`'s headers/s leaves
|
||||
out the time Node's GC threads use; the Rust binary is single-threaded, so
|
||||
its wall time is its CPU time.
|
||||
- **Homelab's cores** (Xeon E5-1650 v4) run the TypeScript ~10% slower than
|
||||
the workstation's.
|
||||
|
||||
**The Rust optimizations,** measured on boneless 2026 (all checked identical
|
||||
after each step):
|
||||
|
||||
| Step | Headers/s |
|
||||
|---|---|
|
||||
| Straight port: regex crate for the patterns, chrono for dates, zlib-rs, std allocator | 360k |
|
||||
| File stems and subject remainders by hand: their 12 + 5 regexes were most of a new file's description (333k per boneless range, many of them per-article random subjects) | |
|
||||
| mimalloc for the many small strings | 495k |
|
||||
| Upload tools, release-name checks and the common Date formats by hand | 500k |
|
||||
| libdeflate: inflates a chunk whole in half of zlib's time (1.09 → 0.51 s per range) | |
|
||||
| memchr (SIMD) for line ends and tabs | ~700k |
|
||||
| Fewer allocations in `describe` (hashing the parts without joining them): no measurable gain, reverted | |
|
||||
|
||||
Rows without a figure weren't measured on their own; each figure covers the
|
||||
rows above it.
|
||||
|
||||
**What's left** is spread out: scanning and building segments, two sha1s per
|
||||
new file, aggregation, cut-off and chunk encoding. `perf` isn't available on
|
||||
the workstation (`perf_event_paranoid` = 3), so these come from coarse
|
||||
timers. The binary prints them as "of which aggregate …, evict+cutoff …,
|
||||
judge+encode …" on stderr.
|
||||
|
||||
**What it means for the full target** (~1.25M articles/s with 50 NNTP
|
||||
connections):
|
||||
- **Worker cores:** ~1.5–2 instead of ~6–7. Spreading workers over other
|
||||
machines (batcave, poweredge-kantoor) becomes mostly unnecessary.
|
||||
- **Total speed:** unchanged. The account's NNTP connections set it (~30k
|
||||
articles/s each), not the worker.
|
||||
- **The cost:** a second implementation of `lib/headers`, `lib/release`,
|
||||
`lib/rules`, `lib/files` (aggregation, volumes) and the chunk format.
|
||||
Every rule change has to land in both and be checked identical (below)
|
||||
for as long as both exist.
|
||||
|
||||
## Running it
|
||||
|
||||
You need a captured range. `edge:bench --capture` saves a range's compressed
|
||||
XOVER chunks once (it needs `NNTP_*` in the environment and one NNTP
|
||||
connection):
|
||||
|
||||
```sh
|
||||
node ace edge:bench --capture --dir /path/to/capture --group alt.binaries.boneless --start 160950000000
|
||||
# 2M articles in chunks of 50k (--articles, --chunk), plus range.json
|
||||
```
|
||||
|
||||
Benchmark both on it:
|
||||
|
||||
```sh
|
||||
node ace edge:bench --dir /path/to/capture # TypeScript: headers/s, CPU, inflate/index, peak RSS
|
||||
cd edge && cargo build --release
|
||||
./target/release/edge-bench /path/to/capture # Rust: the same summary
|
||||
```
|
||||
|
||||
Both take results every 10 chunks (`--batch-chunks`) and give a file up after
|
||||
5 idle chunks (`--idle`). Keep these equal on both sides when comparing.
|
||||
The rules are `DEFAULT_RULES` with only the range's group indexed and no
|
||||
dropped signatures, as in `edge:bench`.
|
||||
|
||||
The two "kept: …" summary lines should already agree: release chunks,
|
||||
files, segments, payload size, candidates, fragments and samples. The
|
||||
summary can hide compensating differences, so compare the outputs
|
||||
themselves.
|
||||
|
||||
## Comparing the output
|
||||
|
||||
```sh
|
||||
./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl
|
||||
cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl
|
||||
```
|
||||
|
||||
`--dump` writes every output of every batch as JSON lines:
|
||||
|
||||
| Line `t` | Contents |
|
||||
|---|---|
|
||||
| `b` | batch boundaries: from, next, headers |
|
||||
| `r` | releases: every ReleaseChunk field, the payload in base64 |
|
||||
| `s` | samples: dropped releases kept in full, with their rule |
|
||||
| `c` | probe candidates |
|
||||
| `n` | rule counts |
|
||||
| `f` | fragments |
|
||||
|
||||
`check.ts` runs the TypeScript `RangeIndexer` (from `lib/edge`, not a copy)
|
||||
on the same capture with the same batching. It matches items by batch and
|
||||
key: release key, candidate (release, kind, message-id), rule and key,
|
||||
fragment file key. Then it compares them field by field:
|
||||
|
||||
```
|
||||
same batches: 4 TypeScript, 4 Rust
|
||||
same releases: 1055 TypeScript, 1055 Rust
|
||||
same samples: 2 TypeScript, 2 Rust
|
||||
same candidates: 1090 TypeScript, 1090 Rust
|
||||
same rule counts: 34 TypeScript, 34 Rust
|
||||
same fragments: 130 TypeScript, 130 Rust
|
||||
payload bytes: 13728351 TypeScript, 13727247 Rust
|
||||
```
|
||||
|
||||
It exits 0 only when everything matches. Otherwise each kind says how many
|
||||
items are only on one side or differ, and shows the first differing fields.
|
||||
|
||||
- **Payloads are compared decoded.** Both sides go through `decodeChunk`
|
||||
from `lib/chunks`, and their files are compared, ordered by name and
|
||||
subject. Node's built-in zstd and the zstd crate frame the same data a
|
||||
little differently, so the compressed sizes differ by about 0.1%. The
|
||||
checker prints them, but they don't count as a difference. Decoding the
|
||||
Rust payloads with the TypeScript decoder also checks the Rust encoder.
|
||||
- **Order doesn't count.** Order within a batch isn't compared (the two
|
||||
hash maps iterate differently). Grouping into batches, the order of a
|
||||
release's files and the order of a fragment's segments are compared,
|
||||
because they're part of the values.
|
||||
|
||||
Run it after any change on either side. All four captures above are
|
||||
identical as of cfe85ab. The captures are kept on homelab in
|
||||
`/mnt/bulk/usenet-rebuild/bench/` (boneless-2026, boneless-2026b,
|
||||
ath-2013, models-a). models-a is a bottom-500 group (rank 501–1000 by
|
||||
span) with what the other three lack: image sets (38,630 release chunks
|
||||
for 412k files), documents, and re-uploads (521 chunks of a second or
|
||||
later upload of a key in a batch). Its chunk 26 is the plain fetch of a
|
||||
range whose compressed reply Eweka cuts short (LOG.md 2026-10-09 (11)).
|
||||
|
||||
## JavaScript semantics the port keeps (`src/js.rs`, `src/names.rs`)
|
||||
|
||||
Differences found with `check.ts` or the tests, kept as JavaScript has
|
||||
them:
|
||||
|
||||
- **Regexes without the `u` flag:** `\d` is [0-9], `\b` an ASCII word
|
||||
boundary, `/i` ASCII-only. They run as `(?-u)` byte regexes, or by hand.
|
||||
The hand-written versions (file stems, subject remainders, upload tools,
|
||||
release-name and file-counter checks) are tested against the regex
|
||||
versions, which are kept for that (`*_re`).
|
||||
- **JS `\s` and `trim()`** include Unicode spaces (NBSP, U+2000–200A,
|
||||
U+3000, BOM…); `split(/\s+/)` keeps leading and trailing empty words.
|
||||
- **`lastIndexOf(c, position)`** finds a match at or before the position
|
||||
(the owner rule, the counter and filename scans). **`.length`** is in
|
||||
UTF-16 units (`len16`). **`parseInt`** reads leading digits.
|
||||
- **`Date.parse`:** chrono follows RFC 5322 and rejects "UTC", which
|
||||
`Date.parse` takes. It ends about a quarter of boneless's Date headers, so
|
||||
it's mapped to GMT; check.ts found this. The common formats are parsed by
|
||||
hand:
|
||||
|
||||
```
|
||||
[Day, ]D[D] Mon YY[YY] HH:MM:SS (GMT|UTC|UT|±hhmm)
|
||||
```
|
||||
|
||||
They ignore the weekday as `Date.parse` does (chrono rejects a weekday
|
||||
that doesn't fit). Anything else goes to chrono. Rare forms `Date.parse`
|
||||
takes but chrono doesn't would show up in check.ts as a `postedAt`
|
||||
difference.
|
||||
- **Invalid UTF-8** becomes U+FFFD on both sides (`toString('utf8')`,
|
||||
`from_utf8_lossy`).
|
||||
|
||||
## Files
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `src/main.rs` | arguments, the replay loop, timing, `--dump` |
|
||||
| `src/indexer.rs` | scan.ts (SegmentScanner, scanOverviewText), aggregator.ts, range_indexer.ts |
|
||||
| `src/names.rs` | lib/release (identity, random), lib/files/volumes, lib/rules (upload tools, judge) |
|
||||
| `src/codec.rs` | lib/chunks encodeChunk, format v2 |
|
||||
| `src/js.rs` | JavaScript semantics, sha1 |
|
||||
| `check.ts` | the comparison |
|
||||
|
||||
## The worker
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
|
||||
```
|
||||
|
||||
- **One range per thread** (default one per core): one thread reads the
|
||||
coordinator's stream and inflates a frame ahead, another posts the
|
||||
batches in order (one may wait), the indexer goes on meanwhile.
|
||||
- **As `commands/edge_node.ts`:** retries for network trouble and 5xx;
|
||||
409 means the lease is lost (the range stops, isn't given back); a
|
||||
batch once 500k articles or 100k files wait (a picture group's batches
|
||||
went over the server's 256 MB limit); a chunk that doesn't inflate is
|
||||
reported (the coordinator fetches it plain next time); SIGTERM/SIGINT
|
||||
give each range back after its current chunk; `--max-mbit` is the
|
||||
node's download cap, shared by its threads. Plus a heartbeat when no
|
||||
batch went out for a minute.
|
||||
- **Wire format:** `src/wire.rs` writes a batch as `toWire` does.
|
||||
`edge-bench --wire` and `check.ts --wire` compare it with the
|
||||
TypeScript's, field by field.
|
||||
- **End to end:** both workers indexed the a.b.models capture (spooled
|
||||
as a range) through the web server; what the server stored was the
|
||||
same (releases with every column, chunks, candidates, rule counts,
|
||||
fragments with their JSON).
|
||||
- **On homelab:** compose service `edge` (deploy/compose.yaml), the
|
||||
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8);
|
||||
it talks to `node-api`.
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k
|
||||
# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared
|
||||
# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from
|
||||
# the repository root after any change to either side.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
|
||||
out=$(mktemp -d)
|
||||
trap 'rm -rf "$out"' EXIT
|
||||
status=0
|
||||
for capture in edge/fixtures/*/; do
|
||||
name=$(basename "$capture")
|
||||
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1
|
||||
echo "== $name"
|
||||
node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1
|
||||
done
|
||||
exit $status
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Compares edge-bench's output (--dump) with the TypeScript RangeIndexer
|
||||
* on the same captured range, item by item: releases (every field, and the
|
||||
* payload decoded with lib/chunks), samples, probe candidates, rule counts,
|
||||
* fragments and batch boundaries.
|
||||
*
|
||||
* node --import=@poppinss/ts-exec edge/check.ts <dir> <dump.jsonl> [--wire batches.jsonl] [--batch-chunks 10] [--idle 5]
|
||||
*
|
||||
* With --wire (edge-bench --wire): also each batch as the Rust worker
|
||||
* posts it, against toWire of the TypeScript batch, field by field.
|
||||
*/
|
||||
import { readFile, readdir } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { RangeIndexer, toWire, type EdgeBatch } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
|
||||
const [dir, dumpPath] = process.argv.slice(2)
|
||||
const flag = (name: string, fallback: number) => {
|
||||
const i = process.argv.indexOf(name)
|
||||
return i === -1 ? fallback : Number(process.argv[i + 1])
|
||||
}
|
||||
const batchChunks = flag('--batch-chunks', 10)
|
||||
const idle = flag('--idle', 5)
|
||||
|
||||
const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
|
||||
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
|
||||
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
|
||||
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
|
||||
const batches: EdgeBatch[] = []
|
||||
for (const [i, file] of files.entries()) {
|
||||
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
|
||||
indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
|
||||
const last = i === files.length - 1
|
||||
if (last) indexer.finish()
|
||||
if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
|
||||
}
|
||||
|
||||
/** Canonical lines per kind, in a stable order, from both sides. */
|
||||
type Item = Record<string, unknown>
|
||||
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
|
||||
const payloadBytes = { ts: 0, rs: 0 }
|
||||
let side: 'ts' | 'rs' = 'ts'
|
||||
const release = (r: Item, b: number) => {
|
||||
const { payload, ...rest } = r as Item & { payload: Buffer }
|
||||
// The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
|
||||
// differently); what the payload decodes to is.
|
||||
payloadBytes[side] += payload.length
|
||||
return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
|
||||
}
|
||||
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
batches.forEach((batch, b) => {
|
||||
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
|
||||
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
||||
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
||||
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
|
||||
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
|
||||
})
|
||||
|
||||
const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
side = 'rs'
|
||||
let b = -1
|
||||
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
|
||||
if (!line) continue
|
||||
const { t, ...item } = JSON.parse(line)
|
||||
if (t === 'b') b++
|
||||
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
|
||||
else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
|
||||
else rs[t].push({ b, ...item })
|
||||
}
|
||||
|
||||
const keyOf: Record<string, (i: Item) => string> = {
|
||||
b: (i) => String(i.b),
|
||||
r: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
s: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
|
||||
n: (i) => `${i.b} ${i.rule} ${i.key}`,
|
||||
f: (i) => `${i.b} ${i.fileKey}`,
|
||||
}
|
||||
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
|
||||
let failed = false
|
||||
for (const kind of Object.keys(keyOf)) {
|
||||
const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const missing = [...a.keys()].filter((k) => !z.has(k))
|
||||
const extra = [...z.keys()].filter((k) => !a.has(k))
|
||||
const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
|
||||
const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
|
||||
failed ||= !ok
|
||||
console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
|
||||
for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`)
|
||||
for (const k of differ.slice(0, 2)) {
|
||||
const x = a.get(k)!
|
||||
const y = z.get(k)!
|
||||
const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
|
||||
console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
|
||||
}
|
||||
}
|
||||
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
|
||||
|
||||
const wireIndex = process.argv.indexOf('--wire')
|
||||
if (wireIndex !== -1) {
|
||||
// Order within a batch doesn't count; payloads by what they decode to.
|
||||
const by = (f: (x: any) => string) => (a: any, b: any) => (f(a) < f(b) ? -1 : f(a) > f(b) ? 1 : 0)
|
||||
const normal = (w: any) => ({
|
||||
...w,
|
||||
releases: w.releases.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
samples: w.samples.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
candidates: [...w.candidates].sort(by((c) => JSON.stringify(c))),
|
||||
counts: [...w.counts].sort(by((c) => `${c.rule} ${c.key}`)),
|
||||
fragments: [...w.fragments].sort(by((f) => `${f.fileKey} ${f.segments[0]?.[1]}`)),
|
||||
})
|
||||
const rust = (await readFile(process.argv[wireIndex + 1], 'utf8')).split('\n').filter(Boolean).map((l) => normal(JSON.parse(l)))
|
||||
const typescript = batches.map((b) => normal(JSON.parse(JSON.stringify(toWire(b)))))
|
||||
let wireSame = rust.length === typescript.length
|
||||
const firstDiff = (x: any, y: any, path: string): string | null => {
|
||||
if (isDeepStrictEqual(x, y)) return null
|
||||
if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null) return `${path}: ${JSON.stringify(x)?.slice(0, 120)} vs ${JSON.stringify(y)?.slice(0, 120)}`
|
||||
for (const k of new Set([...Object.keys(x), ...Object.keys(y)])) {
|
||||
const d = firstDiff(x[k], y[k], `${path}.${k}`)
|
||||
if (d) return d
|
||||
}
|
||||
return path
|
||||
}
|
||||
for (let i = 0; i < Math.max(rust.length, typescript.length); i++) {
|
||||
const d = firstDiff(typescript[i], rust[i], `batch ${i}`)
|
||||
if (d) {
|
||||
wireSame = false
|
||||
console.log(`DIFF wire: ${d}`)
|
||||
break
|
||||
}
|
||||
}
|
||||
if (wireSame) console.log(`same wire batches: ${typescript.length} TypeScript, ${rust.length} Rust`)
|
||||
failed ||= !wireSame
|
||||
}
|
||||
process.exit(failed ? 1 : 0)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"group":"alt.binaries.ath","start":2549992990,"end":2550017989}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"group":"alt.binaries.boneless","start":160950000000,"end":160950024999}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"group":"alt.binaries.models","start":2017215,"end":2042214}
|
||||
@@ -0,0 +1,562 @@
|
||||
//! The edge worker in Rust (REBUILD.md, part 2; commands/edge_node.ts): leases
|
||||
//! ranges from the coordinator, reads their XOVER chunks from its stream,
|
||||
//! indexes them (the same RangeIndexer as edge-bench, kept identical to
|
||||
//! the TypeScript by check.ts) and posts the batches. One range per thread.
|
||||
//!
|
||||
//! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
|
||||
//!
|
||||
//! SIGTERM/SIGINT: each thread gives its range back after the current chunk
|
||||
//! (the next worker resumes at the last stored batch).
|
||||
|
||||
use edge::indexer::{Batch, RangeIndexer};
|
||||
use edge::names::Rules;
|
||||
use edge::wire;
|
||||
use flate2::write::GzEncoder;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
|
||||
use std::sync::mpsc::{sync_channel, Receiver};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
/// A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles (commands/edge_node.ts).
|
||||
const MAX_PENDING_FILES: usize = 100_000;
|
||||
const RETRIES: u32 = 8;
|
||||
const IDLE: Duration = Duration::from_secs(30);
|
||||
/// Without a post for this long, the range's lease is kept alive by a heartbeat.
|
||||
const HEARTBEAT: Duration = Duration::from_secs(60);
|
||||
|
||||
fn log(level: &str, message: &str) {
|
||||
println!("[ {level} ] {message}");
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LeaseJson {
|
||||
range: RangeJson,
|
||||
rules: RulesJson,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RangeJson {
|
||||
id: i64,
|
||||
group: String,
|
||||
start: i64,
|
||||
end: i64,
|
||||
next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RulesJson {
|
||||
version: i64,
|
||||
priorities: HashMap<String, u32>,
|
||||
#[serde(default)]
|
||||
dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
per_article_max_parts: Option<u32>,
|
||||
#[serde(default)]
|
||||
sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
drop_executables: Option<bool>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
fn rules(self) -> Rules {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
if let Some(n) = self.per_article_max_parts {
|
||||
rules.per_article_max_parts = n;
|
||||
}
|
||||
if let Some(p) = self.sample_percent {
|
||||
rules.sample_percent = p;
|
||||
}
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StoredJson {
|
||||
releases: u64,
|
||||
candidates: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ApiError {
|
||||
/// 409: the range is no longer this node's (or the batch was stale): stop it, don't give it back.
|
||||
LeaseLost(String),
|
||||
/// 413: the batch was over the server's body limit; the range is tried again in smaller batches.
|
||||
TooLarge(String),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ApiError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
ApiError::LeaseLost(m) | ApiError::TooLarge(m) | ApiError::Other(m) => f.write_str(m),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The coordinator's node API, with retries for network trouble and 5xx (as the TypeScript Api).
|
||||
struct Api {
|
||||
base: String,
|
||||
token: String,
|
||||
agent: ureq::Agent,
|
||||
}
|
||||
|
||||
impl Api {
|
||||
fn new(base: &str, token: &str) -> Self {
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.http_status_as_error(false)
|
||||
.timeout_connect(Some(Duration::from_secs(30)))
|
||||
.timeout_recv_response(Some(Duration::from_secs(600)))
|
||||
.build()
|
||||
.into();
|
||||
Api { base: base.trim_end_matches('/').to_string(), token: token.to_string(), agent }
|
||||
}
|
||||
|
||||
/// A request, retried; `body`: JSON bytes (gzipped when `gzip`).
|
||||
fn request(&self, method: &str, path: &str, body: Option<&[u8]>, gzip: bool, accept: &str) -> Result<ureq::http::Response<ureq::Body>, ApiError> {
|
||||
let url = format!("{}/api/v1/node/{path}", self.base);
|
||||
let auth = format!("Bearer {}", self.token);
|
||||
for attempt in 1..=RETRIES {
|
||||
let result = match method {
|
||||
"GET" => self.agent.get(&url).header("Authorization", &auth).header("Accept", accept).call(),
|
||||
_ => {
|
||||
let mut request = self.agent.post(&url).header("Authorization", &auth).header("Accept", accept).header("Content-Type", "application/json");
|
||||
if gzip {
|
||||
request = request.header("Content-Encoding", "gzip");
|
||||
}
|
||||
request.send(body.unwrap_or(b"{}"))
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(mut response) => {
|
||||
let status = response.status().as_u16();
|
||||
if status == 409 {
|
||||
let text = response.body_mut().read_to_string().unwrap_or_default();
|
||||
let message = serde_json::from_str::<serde_json::Value>(&text).ok().and_then(|v| v["error"].as_str().map(str::to_string)).unwrap_or_else(|| "lease lost".into());
|
||||
return Err(ApiError::LeaseLost(message));
|
||||
}
|
||||
if status < 500 {
|
||||
return Ok(response);
|
||||
}
|
||||
if attempt == RETRIES {
|
||||
return Err(ApiError::Other(format!("{path}: HTTP {status}")));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
if attempt == RETRIES {
|
||||
return Err(ApiError::Other(format!("{path}: {error}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(60_000.min(1000 * 2u64.pow(attempt))));
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn lease(&self) -> Result<Option<LeaseJson>, ApiError> {
|
||||
let mut response = self.request("POST", "lease", None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
204 => Ok(None),
|
||||
200 => {
|
||||
let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("lease: {e}")))?;
|
||||
serde_json::from_str(&text).map(Some).map_err(|e| ApiError::Other(format!("lease: {e}")))
|
||||
}
|
||||
status => Err(ApiError::Other(format!("lease: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn stream(&self, range: i64) -> Result<Box<dyn Read + Send>, ApiError> {
|
||||
let response = self.request("GET", &format!("ranges/{range}/stream"), None, false, "application/octet-stream")?;
|
||||
if response.status().as_u16() != 200 {
|
||||
return Err(ApiError::Other(format!("stream: HTTP {}", response.status())));
|
||||
}
|
||||
Ok(Box::new(response.into_body().into_reader()))
|
||||
}
|
||||
|
||||
/// Posts a batch (gzipped JSON): the releases and candidates stored, and the bytes posted.
|
||||
fn batch(&self, range: i64, batch: &Batch) -> Result<(u64, u64, usize), ApiError> {
|
||||
let json = wire::to_json(batch);
|
||||
let mut gz = GzEncoder::new(Vec::with_capacity(json.len() / 3), flate2::Compression::new(1));
|
||||
gz.write_all(&json).unwrap();
|
||||
let body = gz.finish().unwrap();
|
||||
let mut response = self.request("POST", &format!("ranges/{range}/batches"), Some(&body), true, "application/json")?;
|
||||
if response.status().as_u16() == 413 {
|
||||
return Err(ApiError::TooLarge(format!("batch: HTTP {} ({} MB gzipped, {} MB of JSON)", response.status(), body.len() / 1_000_000, json.len() / 1_000_000)));
|
||||
}
|
||||
if response.status().as_u16() != 200 {
|
||||
return Err(ApiError::Other(format!("batch: HTTP {}", response.status())));
|
||||
}
|
||||
let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("batch: {e}")))?;
|
||||
let stored: StoredJson = serde_json::from_str(&text).map_err(|e| ApiError::Other(format!("batch: {e}")))?;
|
||||
Ok((stored.releases, stored.candidates, body.len()))
|
||||
}
|
||||
|
||||
fn call(&self, range: i64, action: &str) -> Result<(), ApiError> {
|
||||
let response = self.request("POST", &format!("ranges/{range}/{action}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
200..=299 => Ok(()),
|
||||
status => Err(ApiError::Other(format!("{action}: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn corrupt(&self, range: i64, from: i64) -> Result<(), ApiError> {
|
||||
let body = format!("{{\"from\":{from}}}");
|
||||
let response = self.request("POST", &format!("ranges/{range}/corrupt"), Some(body.as_bytes()), false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
200..=299 => Ok(()),
|
||||
status => Err(ApiError::Other(format!("corrupt: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One frame of the range's stream: a chunk inflated (or why it didn't), or a keep-alive.
|
||||
enum Frame {
|
||||
Chunk { from: i64, to: i64, text: Result<Vec<u8>, String> },
|
||||
KeepAlive,
|
||||
End,
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
/// Reads frames (8 B from, 8 B to, 4 B length, data) and inflates them, one frame ahead of the indexer.
|
||||
fn read_frames(mut stream: Box<dyn Read + Send>, bytes_per_second: Option<f64>) -> Receiver<Frame> {
|
||||
let (tx, rx) = sync_channel::<Frame>(2);
|
||||
std::thread::spawn(move || {
|
||||
let mut decompressor = libdeflater::Decompressor::new();
|
||||
let started = Instant::now();
|
||||
let mut read_bytes = 0f64;
|
||||
loop {
|
||||
let mut head = [0u8; 20];
|
||||
match read_full(&mut stream, &mut head) {
|
||||
Ok(0) => {
|
||||
let _ = tx.send(Frame::End);
|
||||
return;
|
||||
}
|
||||
Ok(20) => {}
|
||||
Ok(n) => {
|
||||
let _ = tx.send(Frame::Failed(format!("stream ended inside a frame ({n} bytes)")));
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx.send(Frame::Failed(format!("stream: {e}")));
|
||||
return;
|
||||
}
|
||||
}
|
||||
let from = u64::from_be_bytes(head[0..8].try_into().unwrap()) as i64;
|
||||
let to = u64::from_be_bytes(head[8..16].try_into().unwrap()) as i64;
|
||||
let length = u32::from_be_bytes(head[16..20].try_into().unwrap()) as usize;
|
||||
let mut data = vec![0u8; length];
|
||||
match read_full(&mut stream, &mut data) {
|
||||
Ok(n) if n == length => {}
|
||||
Ok(n) => {
|
||||
let _ = tx.send(Frame::Failed(format!("stream ended inside a frame ({} of {length} bytes)", n)));
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx.send(Frame::Failed(format!("stream: {e}")));
|
||||
return;
|
||||
}
|
||||
}
|
||||
// --max-mbit: read no faster than the cap (TCP holds the coordinator back).
|
||||
if let Some(rate) = bytes_per_second {
|
||||
read_bytes += (20 + length) as f64;
|
||||
let due = Duration::from_secs_f64(((read_bytes - rate) / rate).max(0.0));
|
||||
let elapsed = started.elapsed();
|
||||
if due > elapsed {
|
||||
std::thread::sleep(due - elapsed);
|
||||
}
|
||||
}
|
||||
let frame = if length == 0 && from == 0 {
|
||||
Frame::KeepAlive
|
||||
} else {
|
||||
Frame::Chunk { from, to, text: inflate(&data, &mut decompressor) }
|
||||
};
|
||||
if tx.send(frame).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
rx
|
||||
}
|
||||
|
||||
/// Reads until `buf` is full or the stream ends: the bytes read.
|
||||
fn read_full(stream: &mut dyn Read, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let mut at = 0;
|
||||
while at < buf.len() {
|
||||
match stream.read(&mut buf[at..]) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => at += n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(at)
|
||||
}
|
||||
|
||||
/// A chunk as the news server sent it (zlib; gzip too), whole.
|
||||
fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Result<Vec<u8>, String> {
|
||||
let mut out = vec![0u8; raw.len() * 4 + 4096];
|
||||
loop {
|
||||
let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) };
|
||||
match result {
|
||||
Ok(n) => {
|
||||
out.truncate(n);
|
||||
return Ok(out);
|
||||
}
|
||||
Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0),
|
||||
Err(e) => return Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Config {
|
||||
batch_articles: i64,
|
||||
bytes_per_second: Option<f64>,
|
||||
/// Ranges whose batch was too large (413), and how much smaller their batches are now (halved each time).
|
||||
smaller: Mutex<HashMap<i64, i64>>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Articles and pending files a batch of this range is posted at.
|
||||
fn limits(&self, range: i64) -> (i64, usize) {
|
||||
let divisor = self.smaller.lock().unwrap().get(&range).copied().unwrap_or(1);
|
||||
(self.batch_articles / divisor, MAX_PENDING_FILES / divisor as usize)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Totals {
|
||||
headers: u64,
|
||||
releases: u64,
|
||||
candidates: u64,
|
||||
bytes: usize,
|
||||
post_ms: u128,
|
||||
}
|
||||
|
||||
/// One leased range, start to finish (commands/edge_node.ts runRange).
|
||||
fn run_range(api: &Arc<Api>, lease: LeaseJson, config: &Config, stopping: &AtomicBool) {
|
||||
let range = lease.range;
|
||||
let started = Instant::now();
|
||||
let mut indexer = RangeIndexer::new(&range.group, range.start, range.end, range.next, lease.rules.rules(), 5);
|
||||
let totals = Arc::new(Mutex::new(Totals::default()));
|
||||
let post_error: Arc<Mutex<Option<ApiError>>> = Arc::new(Mutex::new(None));
|
||||
let last_contact = Arc::new(AtomicI64::new(0));
|
||||
let since = Instant::now();
|
||||
// Posts in order on their own thread; the indexer goes on meanwhile (one batch may wait).
|
||||
let (post_tx, post_rx) = sync_channel::<Batch>(1);
|
||||
let poster = {
|
||||
let (api, totals, post_error, last_contact) = (api.clone(), totals.clone(), post_error.clone(), last_contact.clone());
|
||||
std::thread::spawn(move || {
|
||||
for batch in post_rx {
|
||||
if post_error.lock().unwrap().is_some() {
|
||||
continue;
|
||||
}
|
||||
let at = Instant::now();
|
||||
match api.batch(range.id, &batch) {
|
||||
Ok((releases, candidates, bytes)) => {
|
||||
let mut t = totals.lock().unwrap();
|
||||
t.post_ms += at.elapsed().as_millis();
|
||||
t.bytes += bytes;
|
||||
t.headers += batch.headers;
|
||||
t.releases += releases;
|
||||
t.candidates += candidates;
|
||||
last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
|
||||
}
|
||||
Err(e) => *post_error.lock().unwrap() = Some(e),
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
let frames = match api.stream(range.id) {
|
||||
Ok(stream) => read_frames(stream, config.bytes_per_second),
|
||||
Err(e) => {
|
||||
log("error", &format!("Range {} failed: {e}; giving it back.", range.id));
|
||||
let _ = api.call(range.id, "release");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let (batch_articles, max_pending) = config.limits(range.id);
|
||||
let mut batch_from = range.next;
|
||||
let mut reached_end = false;
|
||||
let mut corrupt: Option<(i64, i64, String)> = None;
|
||||
let mut failure: Option<String> = None;
|
||||
for frame in frames.iter() {
|
||||
if stopping.load(Ordering::Relaxed) || post_error.lock().unwrap().is_some() {
|
||||
break;
|
||||
}
|
||||
// Keep the lease while no batch goes out (slow ranges wait for the spool).
|
||||
if since.elapsed().as_millis() as i64 - last_contact.load(Ordering::Relaxed) > HEARTBEAT.as_millis() as i64 {
|
||||
last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
|
||||
let _ = api.call(range.id, "heartbeat");
|
||||
}
|
||||
match frame {
|
||||
Frame::KeepAlive => continue,
|
||||
Frame::End => break,
|
||||
Frame::Failed(e) => {
|
||||
failure = Some(e);
|
||||
break;
|
||||
}
|
||||
Frame::Chunk { from, to, text } => {
|
||||
let data = match text {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
corrupt = Some((from, to, e));
|
||||
break;
|
||||
}
|
||||
};
|
||||
indexer.add_text(&String::from_utf8_lossy(&data), to + 1);
|
||||
reached_end = to >= range.end;
|
||||
if reached_end {
|
||||
indexer.finish();
|
||||
}
|
||||
if reached_end || to + 1 - batch_from >= batch_articles || indexer.pending() >= max_pending {
|
||||
if post_tx.send(indexer.take()).is_err() {
|
||||
break;
|
||||
}
|
||||
batch_from = to + 1;
|
||||
}
|
||||
if reached_end {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(post_tx);
|
||||
let _ = poster.join();
|
||||
let post_error = post_error.lock().unwrap().take();
|
||||
if let Some((from, to, e)) = corrupt {
|
||||
// The server's compressed reply cut short: reported, fetched plain next time.
|
||||
log("warn", &format!("Range {}: chunk {from}–{to} doesn't inflate ({e}); reported, the range goes back", range.id));
|
||||
let _ = api.corrupt(range.id, from);
|
||||
return;
|
||||
}
|
||||
match post_error {
|
||||
Some(ApiError::LeaseLost(m)) => {
|
||||
log("warn", &format!("Range {}: {m}", range.id));
|
||||
return;
|
||||
}
|
||||
Some(ApiError::TooLarge(m)) => {
|
||||
// Batches already stored stay; from the resume point on, half the size (a batch is at least a chunk).
|
||||
let divisor = {
|
||||
let mut smaller = config.smaller.lock().unwrap();
|
||||
let d = smaller.entry(range.id).or_insert(1);
|
||||
*d = (*d * 2).min(64);
|
||||
*d
|
||||
};
|
||||
log("warn", &format!("Range {}: {m}; giving it back, its batches 1/{divisor} the size from now on.", range.id));
|
||||
let _ = api.call(range.id, "release");
|
||||
return;
|
||||
}
|
||||
Some(ApiError::Other(m)) => {
|
||||
log("error", &format!("Range {} failed: {m}; giving it back.", range.id));
|
||||
let _ = api.call(range.id, "release");
|
||||
return;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if let Some(e) = failure {
|
||||
log("error", &format!("Range {} failed: {e}; giving it back.", range.id));
|
||||
let _ = api.call(range.id, "release");
|
||||
return;
|
||||
}
|
||||
if !reached_end {
|
||||
log("info", &format!("Range {}: giving it back.", range.id));
|
||||
let _ = api.call(range.id, "release");
|
||||
return;
|
||||
}
|
||||
match api.call(range.id, "finish") {
|
||||
Ok(()) => {
|
||||
let t = totals.lock().unwrap();
|
||||
let seconds = started.elapsed().as_secs_f64();
|
||||
log(
|
||||
"info",
|
||||
&format!(
|
||||
"{} {}–{}: {} headers in {:.0} s ({:.0}/s), {} release chunks, {} probe candidates; posted {:.1} MB in {:.1} s.",
|
||||
range.group,
|
||||
range.start,
|
||||
range.end,
|
||||
t.headers,
|
||||
seconds,
|
||||
t.headers as f64 / seconds,
|
||||
t.releases,
|
||||
t.candidates,
|
||||
t.bytes as f64 / 1e6,
|
||||
t.post_ms as f64 / 1000.0
|
||||
),
|
||||
);
|
||||
}
|
||||
Err(e) => log("warn", &format!("Range {}: {e}", range.id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
let url = std::env::var("COORDINATOR_URL").expect("COORDINATOR_URL is needed");
|
||||
let token = std::env::var("NODE_TOKEN").expect("NODE_TOKEN is needed");
|
||||
let threads: usize = flag("--threads")
|
||||
.or_else(|| std::env::var("EDGE_PROCESSES").ok())
|
||||
.map(|v| v.parse().unwrap())
|
||||
.filter(|&n| n > 0)
|
||||
.unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1));
|
||||
let ranges: Option<usize> = flag("--ranges").map(|v| v.parse().unwrap());
|
||||
let max_mbit: Option<f64> = flag("--max-mbit").or_else(|| std::env::var("EDGE_MAX_MBIT").ok()).map(|v| v.parse().unwrap()).filter(|&m: &f64| m > 0.0);
|
||||
let config = Arc::new(Config {
|
||||
batch_articles: flag("--batch-articles").map(|v| v.parse().unwrap()).unwrap_or(500_000),
|
||||
// The node's cap, shared by its threads.
|
||||
bytes_per_second: max_mbit.map(|m| m * 1e6 / 8.0 / threads as f64),
|
||||
smaller: Mutex::new(HashMap::new()),
|
||||
});
|
||||
let stopping = Arc::new(AtomicBool::new(false));
|
||||
signal_hook::flag::register(signal_hook::consts::SIGTERM, stopping.clone()).unwrap();
|
||||
signal_hook::flag::register(signal_hook::consts::SIGINT, stopping.clone()).unwrap();
|
||||
let api = Arc::new(Api::new(&url, &token));
|
||||
log("info", &format!("Starting {threads} worker threads{}.", max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
|
||||
let workers: Vec<_> = (0..threads)
|
||||
.map(|_| {
|
||||
let (api, config, stopping) = (api.clone(), config.clone(), stopping.clone());
|
||||
std::thread::spawn(move || {
|
||||
let mut done = 0;
|
||||
while !stopping.load(Ordering::Relaxed) && ranges.is_none_or(|n| done < n) {
|
||||
match api.lease() {
|
||||
Ok(Some(lease)) => {
|
||||
run_range(&api, lease, &config, &stopping);
|
||||
done += 1;
|
||||
}
|
||||
Ok(None) if ranges.is_some() => break,
|
||||
Ok(None) => idle(&stopping),
|
||||
Err(e) => {
|
||||
log("warn", &format!("Lease failed: {e}"));
|
||||
idle(&stopping);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for worker in workers {
|
||||
let _ = worker.join();
|
||||
}
|
||||
}
|
||||
|
||||
fn idle(stopping: &AtomicBool) {
|
||||
let until = Instant::now() + IDLE;
|
||||
while Instant::now() < until && !stopping.load(Ordering::Relaxed) {
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! lib/chunks/codec.ts encodeChunk, format v2: a version byte, the length of
|
||||
//! the compressed part, zstd(JSON head + text ids), then the nyuu and hex ids
|
||||
//! as raw bytes.
|
||||
|
||||
use crate::js::json_str;
|
||||
|
||||
pub struct ChunkFile<'a> {
|
||||
pub name: &'a str,
|
||||
pub poster: &'a str,
|
||||
pub subject: &'a str,
|
||||
pub posted_at: Option<i64>,
|
||||
pub groups: &'a [String],
|
||||
/// [part, message-id, bytes]
|
||||
pub segments: Vec<(u32, &'a str, i64)>,
|
||||
}
|
||||
|
||||
fn is_nyuu(id: &str, suffix: &str) -> bool {
|
||||
let b = id.as_bytes();
|
||||
if b.len() != 38 + suffix.len() || b[24] != b'-' || !id.ends_with(suffix) {
|
||||
return false;
|
||||
}
|
||||
(0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[25..38].iter().all(u8::is_ascii_digit)
|
||||
}
|
||||
|
||||
fn is_hex(id: &str, suffix: &str) -> bool {
|
||||
let b = id.as_bytes();
|
||||
b.len() == 32 + suffix.len() && id.ends_with(suffix) && b[..32].iter().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(c))
|
||||
}
|
||||
|
||||
fn pack_letters(id: &[u8], out: &mut Vec<u8>) {
|
||||
let mut acc: u32 = 0;
|
||||
let mut bits = 0;
|
||||
for i in (0..24).step_by(2) {
|
||||
acc = (acc << 10) | ((id[i] - b'A') as u32 * 26 + (id[i + 1] - b'a') as u32);
|
||||
bits += 10;
|
||||
while bits >= 8 {
|
||||
bits -= 8;
|
||||
out.push(((acc >> bits) & 0xff) as u8);
|
||||
}
|
||||
acc &= (1 << bits) - 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_val(c: u8) -> u8 {
|
||||
if c.is_ascii_digit() {
|
||||
c - b'0'
|
||||
} else {
|
||||
c - b'a' + 10
|
||||
}
|
||||
}
|
||||
|
||||
fn push_num(out: &mut String, n: i64) {
|
||||
use std::fmt::Write;
|
||||
write!(out, "{n}").unwrap();
|
||||
}
|
||||
|
||||
pub fn encode_chunk(files: &[ChunkFile]) -> Vec<u8> {
|
||||
let mut head = String::with_capacity(256 * files.len());
|
||||
let mut text: Vec<&str> = Vec::new();
|
||||
let mut raw: Vec<u8> = Vec::new();
|
||||
head.push('[');
|
||||
for (fi, f) in files.iter().enumerate() {
|
||||
let ids = &f.segments;
|
||||
let code = match ids.first() {
|
||||
None => 't',
|
||||
Some(&(_, first, _)) => {
|
||||
let nyuu = first.get(38..).filter(|s| s.starts_with('@'));
|
||||
let hex = first.get(32..).filter(|s| s.starts_with('@'));
|
||||
if nyuu.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_nyuu(id, suffix))) {
|
||||
'n'
|
||||
} else if hex.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_hex(id, suffix))) {
|
||||
'h'
|
||||
} else {
|
||||
't'
|
||||
}
|
||||
}
|
||||
};
|
||||
if fi > 0 {
|
||||
head.push(',');
|
||||
}
|
||||
head.push_str("{\"n\":");
|
||||
json_str(f.name, &mut head);
|
||||
head.push_str(",\"p\":");
|
||||
json_str(f.poster, &mut head);
|
||||
head.push_str(",\"j\":");
|
||||
json_str(f.subject, &mut head);
|
||||
head.push_str(",\"d\":");
|
||||
match f.posted_at {
|
||||
Some(t) => push_num(&mut head, t),
|
||||
None => head.push_str("null"),
|
||||
}
|
||||
head.push_str(",\"g\":[");
|
||||
for (i, g) in f.groups.iter().enumerate() {
|
||||
if i > 0 {
|
||||
head.push(',');
|
||||
}
|
||||
json_str(g, &mut head);
|
||||
}
|
||||
head.push_str("],\"s\":[");
|
||||
for (i, &(_, _, b)) in ids.iter().enumerate() {
|
||||
if i > 0 {
|
||||
head.push(',');
|
||||
}
|
||||
push_num(&mut head, b);
|
||||
}
|
||||
head.push(']');
|
||||
if !ids.iter().enumerate().all(|(k, &(n, _, _))| n as usize == k + 1) {
|
||||
head.push_str(",\"k\":[");
|
||||
for (i, &(n, _, _)) in ids.iter().enumerate() {
|
||||
if i > 0 {
|
||||
head.push(',');
|
||||
}
|
||||
push_num(&mut head, n as i64);
|
||||
}
|
||||
head.push(']');
|
||||
}
|
||||
match code {
|
||||
'n' => {
|
||||
head.push_str(",\"c\":\"n\",\"x\":");
|
||||
json_str(&ids[0].1[38..], &mut head);
|
||||
head.push_str(",\"t\":[");
|
||||
let mut previous = 0i64;
|
||||
for (i, &(_, id, _)) in ids.iter().enumerate() {
|
||||
pack_letters(id.as_bytes(), &mut raw);
|
||||
let t: i64 = id[25..38].parse().unwrap();
|
||||
if i > 0 {
|
||||
head.push(',');
|
||||
}
|
||||
push_num(&mut head, t - previous);
|
||||
previous = t;
|
||||
}
|
||||
head.push(']');
|
||||
}
|
||||
'h' => {
|
||||
head.push_str(",\"c\":\"h\",\"x\":");
|
||||
json_str(&ids[0].1[32..], &mut head);
|
||||
for &(_, id, _) in ids {
|
||||
let b = id.as_bytes();
|
||||
for i in (0..32).step_by(2) {
|
||||
raw.push(hex_val(b[i]) << 4 | hex_val(b[i + 1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => text.extend(ids.iter().map(|&(_, id, _)| id)),
|
||||
}
|
||||
head.push('}');
|
||||
}
|
||||
head.push(']');
|
||||
let mut plain = head.into_bytes();
|
||||
for id in text {
|
||||
plain.push(b'\n');
|
||||
plain.extend_from_slice(id.as_bytes());
|
||||
}
|
||||
let packed = zstd::bulk::compress(&plain, 3).unwrap();
|
||||
let mut out = Vec::with_capacity(5 + packed.len() + raw.len());
|
||||
out.push(2);
|
||||
out.extend_from_slice(&(packed.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&packed);
|
||||
out.extend_from_slice(&raw);
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,797 @@
|
||||
//! lib/edge/scan.ts (SegmentScanner, scanOverviewText), lib/files/aggregator.ts
|
||||
//! and lib/edge/range_indexer.ts (RangeIndexer, chunkOf, candidatesOf), in the
|
||||
//! same order of operations, so the outputs can be compared one to one.
|
||||
|
||||
use crate::codec::{encode_chunk, ChunkFile};
|
||||
use crate::js::{self, sha1_hex};
|
||||
use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use std::rc::Rc;
|
||||
|
||||
pub static mut PROF: [u64; 8] = [0; 8];
|
||||
|
||||
pub struct FileMeta {
|
||||
pub file_key: String,
|
||||
pub name: String,
|
||||
pub poster: String,
|
||||
pub subject: String,
|
||||
pub total_parts: u32,
|
||||
pub groups: Vec<String>,
|
||||
pub release: Release,
|
||||
}
|
||||
|
||||
pub struct Seg {
|
||||
pub file: Rc<FileMeta>,
|
||||
pub number: u32,
|
||||
pub message_id: String,
|
||||
pub bytes: i64,
|
||||
pub article: i64,
|
||||
pub posted_at: Option<i64>,
|
||||
}
|
||||
|
||||
pub struct ReleaseChunk {
|
||||
pub key: String,
|
||||
pub upload: usize,
|
||||
pub name: String,
|
||||
pub source: &'static str,
|
||||
pub file_total: Option<i64>,
|
||||
pub poster: String,
|
||||
pub groups: Vec<String>,
|
||||
pub posted_at: Option<i64>,
|
||||
pub posted_last: Option<i64>,
|
||||
pub files: usize,
|
||||
pub segments: usize,
|
||||
pub bytes: i64,
|
||||
pub first_article: i64,
|
||||
pub last_article: i64,
|
||||
pub readable: bool,
|
||||
pub has_par2: bool,
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
pub struct Candidate {
|
||||
pub release_key: String,
|
||||
pub upload: usize,
|
||||
pub kind: &'static str,
|
||||
pub message_id: String,
|
||||
pub file: String,
|
||||
}
|
||||
|
||||
pub struct Fragment {
|
||||
pub file_key: String,
|
||||
pub name: String,
|
||||
pub poster: String,
|
||||
pub subject: String,
|
||||
pub total_parts: u32,
|
||||
pub groups: Vec<String>,
|
||||
pub release: Release,
|
||||
pub posted_at: Option<i64>,
|
||||
pub segments: Vec<(u32, String, i64, i64)>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Batch {
|
||||
pub from: i64,
|
||||
pub next: i64,
|
||||
pub headers: u64,
|
||||
pub releases: Vec<ReleaseChunk>,
|
||||
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
||||
pub candidates: Vec<Candidate>,
|
||||
/// (rule, key) → (articles, files)
|
||||
pub counts: Vec<(&'static str, String, u64, u64)>,
|
||||
pub fragments: Vec<Fragment>,
|
||||
}
|
||||
|
||||
struct Counter {
|
||||
index: usize,
|
||||
length: usize,
|
||||
segment: i64,
|
||||
total: i64,
|
||||
}
|
||||
|
||||
/// The last "(k/N)" after `after`, scanning back from the end (findCounter).
|
||||
fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
|
||||
let mut open = subject.len();
|
||||
loop {
|
||||
// lastIndexOf('(', open - 1): at most open - 1.
|
||||
if open == 0 {
|
||||
return None;
|
||||
}
|
||||
open = subject[..open].iter().rposition(|&c| c == b'(')?;
|
||||
if open <= after {
|
||||
return None;
|
||||
}
|
||||
let mut i = open + 1;
|
||||
let mut segment: i64 = 0;
|
||||
let mut digits = 0;
|
||||
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||||
segment = segment * 10 + (c - b'0') as i64;
|
||||
digits += 1;
|
||||
i += 1;
|
||||
}
|
||||
if digits == 0 || subject.get(i) != Some(&b'/') {
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
let mut total: i64 = 0;
|
||||
digits = 0;
|
||||
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||||
total = total * 10 + (c - b'0') as i64;
|
||||
digits += 1;
|
||||
i += 1;
|
||||
}
|
||||
if digits == 0 || subject.get(i) != Some(&b')') {
|
||||
continue;
|
||||
}
|
||||
return Some(Counter { index: open, length: i + 1 - open, segment, total });
|
||||
}
|
||||
}
|
||||
|
||||
fn xref_groups(xref: &str, group: &str) -> Vec<String> {
|
||||
let mut groups = Vec::new();
|
||||
for token in js::split_spaces(js::trim(xref)).into_iter().skip(1) {
|
||||
if let Some(colon) = token.rfind(':') {
|
||||
if colon > 0 {
|
||||
groups.push(token[..colon].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if groups.is_empty() {
|
||||
groups.push(group.to_string());
|
||||
}
|
||||
groups
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Scanner {
|
||||
files: FxHashMap<String, Option<Rc<FileMeta>>>,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
fn owner<'a>(&self, xref: &'a str, group: &'a str, priorities: &FxHashMap<String, u32>) -> &'a str {
|
||||
let b = xref.as_bytes();
|
||||
let Some(mut at) = xref.find(' ') else { return group };
|
||||
let mut owner: Option<&str> = None;
|
||||
let mut best = u32::MAX;
|
||||
loop {
|
||||
while b.get(at) == Some(&b' ') {
|
||||
at += 1;
|
||||
}
|
||||
if at >= b.len() {
|
||||
break;
|
||||
}
|
||||
let end = xref[at..].find(' ').map(|i| i + at);
|
||||
// lastIndexOf(':', end ?? length): a colon at or before `end`.
|
||||
let limit = end.unwrap_or(b.len());
|
||||
let colon = xref[..limit.min(b.len())].rfind(':');
|
||||
if let Some(colon) = colon.filter(|&c| c > at) {
|
||||
let name = &xref[at..colon];
|
||||
if let Some(&p) = priorities.get(name) {
|
||||
if p < best {
|
||||
best = p;
|
||||
owner = Some(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
match end {
|
||||
Some(e) => at = e,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
owner.unwrap_or(group)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn segment(&mut self, article: i64, subject: &str, from: &str, date: &str, message_id: &str, bytes: &str, xref: &str, group: &str) -> Option<Seg> {
|
||||
if subject.is_empty() || message_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let sb = subject.as_bytes();
|
||||
let name_end = subject.rfind('"')?;
|
||||
if name_end == 0 {
|
||||
return None;
|
||||
}
|
||||
let counter = find_counter(sb, name_end)?;
|
||||
self.key.clear();
|
||||
self.key.push_str(from);
|
||||
self.key.push('\u{1}');
|
||||
self.key.push_str(&subject[..counter.index]);
|
||||
self.key.push('\u{0}');
|
||||
self.key.push_str(&subject[counter.index + counter.length..]);
|
||||
let file = match self.files.get(self.key.as_str()) {
|
||||
Some(f) => f.clone(),
|
||||
None => {
|
||||
let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new);
|
||||
self.files.insert(self.key.clone(), f.clone());
|
||||
f
|
||||
}
|
||||
};
|
||||
let file = file?;
|
||||
if counter.segment < 1 || counter.segment > file.total_parts as i64 {
|
||||
return None;
|
||||
}
|
||||
let id = if message_id.starts_with('<') { message_id.get(1..message_id.len().saturating_sub(1)).unwrap_or("") } else { message_id };
|
||||
Some(Seg {
|
||||
number: counter.segment as u32,
|
||||
message_id: id.to_string(),
|
||||
bytes: js::parse_int(bytes).unwrap_or(0),
|
||||
article,
|
||||
posted_at: js::parse_date(date),
|
||||
file,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
||||
let name_start = subject[..name_end].rfind('"')?;
|
||||
let name = &subject[name_start + 1..name_end];
|
||||
if name.is_empty() || counter.total < 1 {
|
||||
return None;
|
||||
}
|
||||
let joined = format!("{}{}", &subject[..counter.index], &subject[counter.index + counter.length..]);
|
||||
let base = js::trim(&js::collapse_spaces(&joined)).to_string();
|
||||
let file_key = sha1_hex(&format!("{from}\n{base}\n{}", counter.total));
|
||||
let release = names::release_identity(from, &base, name);
|
||||
Some(FileMeta {
|
||||
file_key,
|
||||
name: name.to_string(),
|
||||
poster: from.to_string(),
|
||||
subject: base,
|
||||
total_parts: counter.total as u32,
|
||||
groups: xref_groups(xref, group),
|
||||
release,
|
||||
})
|
||||
}
|
||||
|
||||
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
|
||||
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
|
||||
let b = text.as_bytes();
|
||||
let mut start = 0;
|
||||
while start < b.len() {
|
||||
let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
|
||||
let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end };
|
||||
if stop > start {
|
||||
let l = &text[start..stop];
|
||||
let lb = l.as_bytes();
|
||||
let mut tabs = [usize::MAX; 9];
|
||||
let mut n = 0;
|
||||
for i in memchr::memchr_iter(b'\t', lb) {
|
||||
tabs[n] = i;
|
||||
n += 1;
|
||||
if n == 9 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if n >= 5 {
|
||||
let field = |a: usize, b: usize| -> &str {
|
||||
if a == usize::MAX {
|
||||
""
|
||||
} else {
|
||||
&l[a + 1..if b == usize::MAX { l.len() } else { b }]
|
||||
}
|
||||
};
|
||||
let mut xref = if n >= 8 { field(tabs[7], tabs[8]) } else { "" };
|
||||
if let Some(rest) = xref.strip_prefix("Xref:") {
|
||||
xref = rest.strip_prefix(' ').unwrap_or(rest);
|
||||
}
|
||||
line(js::parse_int(&l[..tabs[0]]), &l[tabs[0] + 1..tabs[1]], field(tabs[1], tabs[2]), field(tabs[2], tabs[3]), field(tabs[3], tabs[4]), field(tabs[5], tabs[6]), xref);
|
||||
}
|
||||
}
|
||||
start = end + 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// A re-upload's copy of a file is kept apart (FileAggregator, uploadGapMs).
|
||||
pub const UPLOAD_GAP_MS: i64 = 3 * 60 * 60 * 1000;
|
||||
|
||||
struct OpenFile {
|
||||
parts: Vec<Seg>,
|
||||
seen: FxHashSet<u32>,
|
||||
total: u32,
|
||||
touched: u64,
|
||||
/// Posting times seen.
|
||||
earliest: Option<i64>,
|
||||
latest: Option<i64>,
|
||||
}
|
||||
|
||||
impl OpenFile {
|
||||
fn distance(&self, t: Option<i64>) -> i64 {
|
||||
match (t, self.earliest, self.latest) {
|
||||
(Some(t), Some(e), Some(l)) => 0.max(e - t).max(t - l),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Aggregator {
|
||||
/// Per file key its open copies, oldest first.
|
||||
open: FxHashMap<String, Vec<OpenFile>>,
|
||||
batch: u64,
|
||||
}
|
||||
|
||||
impl Aggregator {
|
||||
fn add(&mut self, segments: Vec<Seg>) -> Vec<Vec<Seg>> {
|
||||
self.batch += 1;
|
||||
let batch = self.batch;
|
||||
let mut touched: Vec<String> = Vec::new();
|
||||
let mut touched_set: FxHashSet<String> = FxHashSet::default();
|
||||
for s in segments {
|
||||
let key = s.file.file_key.clone();
|
||||
if !touched_set.contains(key.as_str()) {
|
||||
touched_set.insert(key.clone());
|
||||
touched.push(key.clone());
|
||||
}
|
||||
let copies = self.open.entry(key).or_default();
|
||||
let t = s.posted_at;
|
||||
// A copy lacking the part, the nearest in time.
|
||||
let mut chosen: Option<usize> = None;
|
||||
let mut best = i64::MAX;
|
||||
for (i, copy) in copies.iter().enumerate() {
|
||||
if copy.seen.contains(&s.number) {
|
||||
continue;
|
||||
}
|
||||
let d = copy.distance(t);
|
||||
if d < best {
|
||||
best = d;
|
||||
chosen = Some(i);
|
||||
}
|
||||
}
|
||||
let index = match chosen {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
if let Some(near) = copies.iter_mut().find(|c| c.distance(t) <= UPLOAD_GAP_MS) {
|
||||
near.touched = batch;
|
||||
continue;
|
||||
}
|
||||
copies.push(OpenFile { parts: Vec::new(), seen: FxHashSet::default(), total: s.file.total_parts, touched: batch, earliest: None, latest: None });
|
||||
copies.len() - 1
|
||||
}
|
||||
};
|
||||
let file = &mut copies[index];
|
||||
file.seen.insert(s.number);
|
||||
if let Some(t) = t {
|
||||
if file.earliest.is_none_or(|e| t < e) {
|
||||
file.earliest = Some(t);
|
||||
}
|
||||
if file.latest.is_none_or(|l| t > l) {
|
||||
file.latest = Some(t);
|
||||
}
|
||||
}
|
||||
file.touched = batch;
|
||||
file.parts.push(s);
|
||||
}
|
||||
let mut complete = Vec::new();
|
||||
for key in touched {
|
||||
let copies = self.open.get_mut(&key).unwrap();
|
||||
let mut i = 0;
|
||||
while i < copies.len() {
|
||||
if (copies[i].parts.len() as u32) < copies[i].total {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let mut f = copies.remove(i);
|
||||
f.parts.sort_by_key(|s| s.number);
|
||||
complete.push(f.parts);
|
||||
}
|
||||
if copies.is_empty() {
|
||||
self.open.remove(&key);
|
||||
}
|
||||
}
|
||||
complete
|
||||
}
|
||||
|
||||
/// Files idle for `idle` batches (all when 0): their segments, per file key (its copies one after another).
|
||||
fn evict(&mut self, idle: u64) -> Vec<Vec<Seg>> {
|
||||
let batch = self.batch;
|
||||
let mut out = Vec::new();
|
||||
let keys: Vec<String> = self.open.keys().cloned().collect();
|
||||
for key in keys {
|
||||
let copies = self.open.get_mut(&key).unwrap();
|
||||
let mut gone: Vec<Seg> = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < copies.len() {
|
||||
if idle > 0 && batch - copies[i].touched < idle {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
gone.extend(copies.remove(i).parts);
|
||||
}
|
||||
if copies.is_empty() {
|
||||
self.open.remove(&key);
|
||||
}
|
||||
if !gone.is_empty() {
|
||||
out.push(gone);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// uploadsOf: a key's files split where posting pauses for more than UPLOAD_GAP_MS after the latest article so far.
|
||||
fn uploads_of(files: Vec<Vec<Seg>>) -> Vec<Vec<Vec<Seg>>> {
|
||||
if files.len() < 2 {
|
||||
return vec![files];
|
||||
}
|
||||
let first: Vec<Option<i64>> = files.iter().map(|f| posted_of(f)).collect();
|
||||
let last: Vec<Option<i64>> = files.iter().map(|f| f.iter().filter_map(|s| s.posted_at).max()).collect();
|
||||
let mut dated: Vec<usize> = (0..files.len()).filter(|&i| first[i].is_some()).collect();
|
||||
if dated.len() < 2 {
|
||||
return vec![files];
|
||||
}
|
||||
dated.sort_by_key(|&i| (first[i].unwrap(), i));
|
||||
let mut upload = vec![0usize; files.len()];
|
||||
let mut current = 0;
|
||||
let mut end = last[dated[0]].unwrap();
|
||||
for &i in &dated[1..] {
|
||||
if first[i].unwrap() - end > UPLOAD_GAP_MS {
|
||||
current += 1;
|
||||
end = last[i].unwrap();
|
||||
} else if last[i].unwrap() > end {
|
||||
end = last[i].unwrap();
|
||||
}
|
||||
upload[i] = current;
|
||||
}
|
||||
if current == 0 {
|
||||
return vec![files];
|
||||
}
|
||||
let mut out: Vec<Vec<Vec<Seg>>> = (0..=current).map(|_| Vec::new()).collect();
|
||||
for (i, f) in files.into_iter().enumerate() {
|
||||
out[upload[i]].push(f);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub struct RangeIndexer {
|
||||
group: String,
|
||||
rules: Rules,
|
||||
start: i64,
|
||||
end: i64,
|
||||
priorities: FxHashMap<String, u32>,
|
||||
idle: u64,
|
||||
chunk_articles: i64,
|
||||
scanner: Scanner,
|
||||
aggregator: Aggregator,
|
||||
batch: Batch,
|
||||
counts: FxHashMap<(&'static str, String), (u64, u64)>,
|
||||
count_order: Vec<(&'static str, String)>,
|
||||
completed: FxHashMap<String, Vec<Vec<Seg>>>,
|
||||
completed_order: Vec<String>,
|
||||
}
|
||||
|
||||
impl RangeIndexer {
|
||||
/// `resume_at`: where this worker starts (a range taken over midway); its first batch starts there.
|
||||
pub fn new(group: &str, start: i64, end: i64, resume_at: i64, rules: Rules, idle: u64) -> Self {
|
||||
RangeIndexer {
|
||||
group: group.to_string(),
|
||||
start,
|
||||
end,
|
||||
priorities: rules.priorities.clone(),
|
||||
rules,
|
||||
idle,
|
||||
chunk_articles: 0,
|
||||
scanner: Scanner::default(),
|
||||
aggregator: Aggregator::default(),
|
||||
batch: Batch { from: resume_at, next: resume_at, ..Default::default() },
|
||||
counts: FxHashMap::default(),
|
||||
count_order: Vec::new(),
|
||||
completed: FxHashMap::default(),
|
||||
completed_order: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_text(&mut self, text: &str, next: i64) {
|
||||
let mut segments: Vec<Seg> = Vec::new();
|
||||
let mut crossposts: Vec<String> = Vec::new();
|
||||
let mut obfuscated = 0u64;
|
||||
let mut headers = 0u64;
|
||||
{
|
||||
let group = self.group.clone();
|
||||
let scanner = &mut self.scanner;
|
||||
let priorities = &self.priorities;
|
||||
scan_overview_text(text, |n, subject, from, date, message_id, bytes, xref| {
|
||||
let Some(n) = n else { return };
|
||||
if message_id.is_empty() {
|
||||
return;
|
||||
}
|
||||
headers += 1;
|
||||
let owner = if xref.is_empty() { group.as_str() } else { scanner.owner(xref, &group, priorities) };
|
||||
if owner != group {
|
||||
crossposts.push(owner.to_string());
|
||||
return;
|
||||
}
|
||||
match scanner.segment(n, subject, from, date, message_id, bytes, xref, &group) {
|
||||
Some(s) => segments.push(s),
|
||||
None => obfuscated += 1,
|
||||
}
|
||||
});
|
||||
}
|
||||
self.batch.headers += headers;
|
||||
for owner in crossposts {
|
||||
self.count("crosspost", owner, 1, 0);
|
||||
}
|
||||
if obfuscated > 0 {
|
||||
self.count("obfuscated", String::new(), obfuscated, 0);
|
||||
}
|
||||
self.end_chunk(segments, next);
|
||||
}
|
||||
|
||||
fn end_chunk(&mut self, segments: Vec<Seg>, next: i64) {
|
||||
let t = std::time::Instant::now();
|
||||
let done = self.aggregator.add(segments);
|
||||
unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
|
||||
self.files(done);
|
||||
if self.chunk_articles == 0 {
|
||||
self.chunk_articles = next - self.batch.from;
|
||||
}
|
||||
let t = std::time::Instant::now();
|
||||
let gone = self.aggregator.evict(self.idle);
|
||||
self.cut_off(gone, false);
|
||||
unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
|
||||
self.batch.next = next;
|
||||
}
|
||||
|
||||
pub fn finish(&mut self) {
|
||||
let gone = self.aggregator.evict(0);
|
||||
self.cut_off(gone, true);
|
||||
self.batch.next = self.end + 1;
|
||||
}
|
||||
|
||||
/// Files completed and fragments cut since the last take (RangeIndexer.pending).
|
||||
pub fn pending(&self) -> usize {
|
||||
self.batch.fragments.len() + self.completed.values().map(|l| l.len()).sum::<usize>()
|
||||
}
|
||||
|
||||
pub fn take(&mut self) -> Batch {
|
||||
let t = std::time::Instant::now();
|
||||
self.judge_releases();
|
||||
unsafe { PROF[5] += t.elapsed().as_nanos() as u64; }
|
||||
self.scanner.files.clear();
|
||||
let next = self.batch.next;
|
||||
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
||||
for id in self.count_order.drain(..) {
|
||||
let (articles, files) = self.counts.remove(&id).unwrap();
|
||||
out.counts.push((id.0, id.1, articles, files));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn files(&mut self, files: Vec<Vec<Seg>>) {
|
||||
for file in files {
|
||||
let key = release_key_of(&file[0]);
|
||||
match self.completed.get_mut(&key) {
|
||||
Some(list) => list.push(file),
|
||||
None => {
|
||||
self.completed_order.push(key.clone());
|
||||
self.completed.insert(key, vec![file]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn judge_releases(&mut self) {
|
||||
let order = std::mem::take(&mut self.completed_order);
|
||||
let mut by_release = std::mem::take(&mut self.completed);
|
||||
for key in order {
|
||||
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
||||
let judged: Vec<JudgedFile> = list
|
||||
.iter()
|
||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
||||
.collect();
|
||||
match names::judge(&key, &judged, &self.rules) {
|
||||
Verdict::Keep => {
|
||||
let candidates = candidates_of(&key, &list, upload);
|
||||
self.batch.releases.push(chunk_of(&key, &list, upload));
|
||||
self.batch.candidates.extend(candidates);
|
||||
}
|
||||
Verdict::Drop { rule, key: rule_key, sample } => {
|
||||
let segments: usize = list.iter().map(|f| f.len()).sum();
|
||||
self.count(rule, rule_key.clone(), segments as u64, list.len() as u64);
|
||||
if sample {
|
||||
self.batch.samples.push((chunk_of(&key, &list, upload), rule, rule_key));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cut_off(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
|
||||
let window = self.idle as i64 * self.chunk_articles.max(1);
|
||||
for list in files {
|
||||
if list.len() == 1 && is_random_file(&list[0].file.name) {
|
||||
let tool = names::upload_tool(&list[0].message_id);
|
||||
self.count("per-article", tool.to_string(), 1, 0);
|
||||
continue;
|
||||
}
|
||||
let min_part = list.iter().map(|s| s.number).min().unwrap();
|
||||
let max_part = list.iter().map(|s| s.number).max().unwrap();
|
||||
let first_article = list.iter().map(|s| s.article).min().unwrap();
|
||||
let cut_at_start = min_part > 1 && first_article < self.start + window;
|
||||
let cut_at_end = at_end && max_part < list[0].file.total_parts;
|
||||
if !cut_at_start && !cut_at_end {
|
||||
self.count("incomplete", String::new(), list.len() as u64, 1);
|
||||
continue;
|
||||
}
|
||||
let f = &list[0].file;
|
||||
let mut fragment = Fragment {
|
||||
file_key: f.file_key.clone(),
|
||||
name: f.name.clone(),
|
||||
poster: f.poster.clone(),
|
||||
subject: f.subject.clone(),
|
||||
total_parts: f.total_parts,
|
||||
groups: f.groups.clone(),
|
||||
release: f.release.clone(),
|
||||
posted_at: None,
|
||||
segments: Vec::with_capacity(list.len()),
|
||||
};
|
||||
for s in &list {
|
||||
fragment.segments.push((s.number, s.message_id.clone(), s.bytes, s.article));
|
||||
if let Some(t) = s.posted_at {
|
||||
if fragment.posted_at.is_none_or(|p| t < p) {
|
||||
fragment.posted_at = Some(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.batch.fragments.push(fragment);
|
||||
}
|
||||
}
|
||||
|
||||
fn count(&mut self, rule: &'static str, key: String, articles: u64, files: u64) {
|
||||
match self.counts.get_mut(&(rule, key.clone())) {
|
||||
Some(c) => {
|
||||
c.0 += articles;
|
||||
c.1 += files;
|
||||
}
|
||||
None => {
|
||||
self.count_order.push((rule, key.clone()));
|
||||
self.counts.insert((rule, key), (articles, files));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn release_key_of(s: &Seg) -> String {
|
||||
// A named file always has a release (releaseIdentity isn't null unless weak).
|
||||
s.file.release.key.clone()
|
||||
}
|
||||
|
||||
fn posted_of(segments: &[Seg]) -> Option<i64> {
|
||||
segments.iter().filter_map(|s| s.posted_at).min()
|
||||
}
|
||||
|
||||
fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
||||
let first = &files[0][0].file;
|
||||
let chunk_files: Vec<ChunkFile> = files
|
||||
.iter()
|
||||
.map(|f| {
|
||||
let m = &f[0].file;
|
||||
ChunkFile { name: &m.name, poster: &m.poster, subject: &m.subject, posted_at: posted_of(f), groups: &m.groups, segments: f.iter().map(|s| (s.number, s.message_id.as_str(), s.bytes)).collect() }
|
||||
})
|
||||
.collect();
|
||||
let mut segments = 0;
|
||||
let mut bytes = 0;
|
||||
let mut first_article = i64::MAX;
|
||||
let mut last_article = 0;
|
||||
let mut posted_at: Option<i64> = None;
|
||||
let mut posted_last: Option<i64> = None;
|
||||
for f in files {
|
||||
for s in f {
|
||||
if let Some(t) = s.posted_at {
|
||||
if posted_last.is_none_or(|p| t > p) {
|
||||
posted_last = Some(t);
|
||||
}
|
||||
}
|
||||
segments += 1;
|
||||
bytes += s.bytes;
|
||||
first_article = first_article.min(s.article);
|
||||
last_article = last_article.max(s.article);
|
||||
}
|
||||
if let Some(t) = posted_of(f) {
|
||||
if posted_at.is_none_or(|p| t < p) {
|
||||
posted_at = Some(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
ReleaseChunk {
|
||||
key: key.to_string(),
|
||||
upload,
|
||||
name: first.release.name.clone(),
|
||||
source: first.release.source,
|
||||
file_total: files.iter().find_map(|f| f[0].file.release.file_total_truthy()),
|
||||
poster: first.poster.clone(),
|
||||
groups: first.groups.clone(),
|
||||
posted_at,
|
||||
posted_last,
|
||||
files: files.len(),
|
||||
segments,
|
||||
bytes,
|
||||
first_article,
|
||||
last_article,
|
||||
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
||||
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
||||
payload: encode_chunk(&chunk_files),
|
||||
}
|
||||
}
|
||||
|
||||
fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate> {
|
||||
let mut out = Vec::new();
|
||||
let mut tail: Option<(i64, &Vec<Seg>)> = None;
|
||||
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
||||
for f in files {
|
||||
let name = &f[0].file.name;
|
||||
if let Some(volume) = names::seven_zip_volume(name).filter(|&v| v > 1) {
|
||||
if tail.is_none_or(|(v, _)| volume > v) {
|
||||
tail = Some((volume, f));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if names::is_archive_start(name) {
|
||||
out.push(c("archive", &f[0]));
|
||||
} else if names::is_par2(name) && !names::is_later_volume(name) {
|
||||
out.push(c("par2", &f[0]));
|
||||
}
|
||||
}
|
||||
if let Some((_, f)) = tail {
|
||||
out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
|
||||
}
|
||||
if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
|
||||
out.push(c("lone", &files[0][0]));
|
||||
}
|
||||
out.truncate(4);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const GROUP: &str = "alt.binaries.test";
|
||||
|
||||
fn line(n: i64, subject: &str, minutes: i64) -> String {
|
||||
let date = chrono::DateTime::from_timestamp_millis(1_791_194_400_000 + minutes * 60_000).unwrap().format("%a, %d %b %Y %H:%M:%S +0000");
|
||||
format!("{n}\t{subject}\tp <p@x>\t{date}\t<u{n}@test>\t\t1000\t10\tXref: news.example {GROUP}:{n}")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploads_split_as_in_the_typescript() {
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(GROUP.to_string(), 1);
|
||||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||||
let mut lines: Vec<String> = (0..5).map(|i| line(100 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), i * 60)).collect();
|
||||
lines.push(line(200, r#"Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)"#, 245));
|
||||
lines.extend((0..5).map(|i| line(300 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), 1440 + i)));
|
||||
ix.add_text(&lines.join("\r\n"), 10_001);
|
||||
ix.finish();
|
||||
let batch = ix.take();
|
||||
let got: Vec<(usize, usize)> = batch.releases.iter().map(|r| (r.upload, r.files)).collect();
|
||||
assert_eq!(got, vec![(0, 2), (1, 1)]);
|
||||
assert_eq!(batch.releases[0].posted_last.unwrap() - batch.releases[0].posted_at.unwrap(), 245 * 60_000);
|
||||
assert_eq!(batch.releases[0].key, batch.releases[1].key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copies_and_fills_as_in_the_typescript() {
|
||||
let hour = 3_600_000i64;
|
||||
let meta = Rc::new(FileMeta {
|
||||
file_key: "a".into(),
|
||||
name: "a.bin".into(),
|
||||
poster: "p".into(),
|
||||
subject: "s".into(),
|
||||
total_parts: 3,
|
||||
groups: vec![],
|
||||
release: names::release_identity("p", "s", "a.bin"),
|
||||
});
|
||||
let at = |number: u32, h: f64, id: &str| Seg { file: meta.clone(), number, message_id: id.into(), bytes: 100, article: 0, posted_at: Some((h * hour as f64) as i64) };
|
||||
let mut agg = Aggregator::default();
|
||||
assert!(agg.add(vec![at(1, 10.0, "a1"), at(2, 10.0, "a2"), at(1, 10.1, "resend")]).is_empty());
|
||||
assert!(agg.add(vec![at(1, 34.0, "b1"), at(2, 34.0, "b2")]).is_empty());
|
||||
assert_eq!(agg.open.get("a").unwrap().len(), 2);
|
||||
let second = agg.add(vec![at(3, 34.2, "b3")]);
|
||||
assert_eq!(second[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["b1", "b2", "b3"]);
|
||||
let first = agg.add(vec![at(3, 58.0, "a3")]);
|
||||
assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["a1", "a2", "a3"]);
|
||||
assert!(agg.open.is_empty());
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
//! JavaScript semantics the TypeScript worker relies on: `\s` and `trim()`,
|
||||
//! UTF-16 lengths, `parseInt`, `Date.parse` (for Usenet dates) and sha1 hex.
|
||||
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
/// JS `\s` / `trim()` whitespace (WhiteSpace and LineTerminator).
|
||||
pub fn is_space(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'\t' | '\n' | '\u{0B}' | '\u{0C}' | '\r' | ' ' | '\u{A0}' | '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{2028}' | '\u{2029}' | '\u{202F}' | '\u{205F}' | '\u{3000}' | '\u{FEFF}'
|
||||
)
|
||||
}
|
||||
|
||||
pub fn trim(s: &str) -> &str {
|
||||
s.trim_matches(is_space)
|
||||
}
|
||||
|
||||
/// `s.replace(/\s+/g, ' ')`.
|
||||
pub fn collapse_spaces(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut in_space = false;
|
||||
for c in s.chars() {
|
||||
if is_space(c) {
|
||||
if !in_space {
|
||||
out.push(' ');
|
||||
in_space = true;
|
||||
}
|
||||
} else {
|
||||
out.push(c);
|
||||
in_space = false;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `s.split(/\s+/)`: like JS, a leading or trailing run gives an empty word.
|
||||
pub fn split_spaces(s: &str) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut in_space = false;
|
||||
let mut run_start = 0;
|
||||
for (i, c) in s.char_indices() {
|
||||
if is_space(c) {
|
||||
if !in_space {
|
||||
run_start = i;
|
||||
in_space = true;
|
||||
}
|
||||
} else if in_space {
|
||||
out.push(&s[start..run_start]);
|
||||
start = i;
|
||||
in_space = false;
|
||||
}
|
||||
}
|
||||
if in_space {
|
||||
out.push(&s[start..run_start]);
|
||||
out.push("");
|
||||
} else {
|
||||
out.push(&s[start..]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A string's `.length` in JavaScript (UTF-16 code units).
|
||||
pub fn len16(s: &str) -> usize {
|
||||
if s.is_ascii() {
|
||||
s.len()
|
||||
} else {
|
||||
s.encode_utf16().count()
|
||||
}
|
||||
}
|
||||
|
||||
/// `Number.parseInt(s, 10)`, None for NaN.
|
||||
pub fn parse_int(s: &str) -> Option<i64> {
|
||||
let s = s.trim_start_matches(is_space);
|
||||
let (neg, digits) = match s.as_bytes().first() {
|
||||
Some(b'-') => (true, &s[1..]),
|
||||
Some(b'+') => (false, &s[1..]),
|
||||
_ => (false, s),
|
||||
};
|
||||
let mut n: i64 = 0;
|
||||
let mut any = false;
|
||||
for b in digits.bytes() {
|
||||
if !b.is_ascii_digit() {
|
||||
break;
|
||||
}
|
||||
n = n.wrapping_mul(10).wrapping_add((b - b'0') as i64);
|
||||
any = true;
|
||||
}
|
||||
if any {
|
||||
Some(if neg { -n } else { n })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Days since 1970-01-01 of a proleptic Gregorian date (month 1..=12; a day past the month's end runs on, as in JS).
|
||||
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
||||
let y = if m <= 2 { y - 1 } else { y };
|
||||
let era = y.div_euclid(400);
|
||||
let yoe = y - era * 400;
|
||||
let mp = (m + 9) % 12;
|
||||
let doy = (153 * mp + 2) / 5 + d - 1;
|
||||
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
||||
era * 146_097 + doe - 719_468
|
||||
}
|
||||
|
||||
/// The usual Date headers: [Day, ]D[D] Mon YY[YY] HH:MM:SS (GMT|UTC|UT|±hhmm). None for anything else.
|
||||
fn fast_date(s: &[u8]) -> Option<i64> {
|
||||
let mut i = 0;
|
||||
if s.len() > 4 && s[3] == b',' && s[..3].iter().all(u8::is_ascii_alphabetic) {
|
||||
i = 4;
|
||||
while s.get(i) == Some(&b' ') {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
let num = |i: &mut usize, min: usize, max: usize| -> Option<i64> {
|
||||
let start = *i;
|
||||
while *i < s.len() && s[*i].is_ascii_digit() && *i - start < max {
|
||||
*i += 1;
|
||||
}
|
||||
if *i - start < min || s.get(*i).is_some_and(u8::is_ascii_digit) {
|
||||
return None;
|
||||
}
|
||||
Some(std::str::from_utf8(&s[start..*i]).ok()?.parse().ok()?)
|
||||
};
|
||||
let expect = |i: &mut usize, c: u8| -> Option<()> {
|
||||
(s.get(*i) == Some(&c)).then(|| *i += 1)
|
||||
};
|
||||
let day = num(&mut i, 1, 2)?;
|
||||
expect(&mut i, b' ')?;
|
||||
const MONTHS: [&[u8]; 12] = [b"jan", b"feb", b"mar", b"apr", b"may", b"jun", b"jul", b"aug", b"sep", b"oct", b"nov", b"dec"];
|
||||
let mon = s.get(i..i + 3)?;
|
||||
let month = MONTHS.iter().position(|m| mon.iter().zip(m.iter()).all(|(a, b)| a.to_ascii_lowercase() == *b))? as i64 + 1;
|
||||
i += 3;
|
||||
expect(&mut i, b' ')?;
|
||||
let year_start = i;
|
||||
let mut year = num(&mut i, 2, 4)?;
|
||||
match i - year_start {
|
||||
2 => year += if year < 50 { 2000 } else { 1900 },
|
||||
4 => {}
|
||||
_ => return None,
|
||||
}
|
||||
expect(&mut i, b' ')?;
|
||||
let hour = num(&mut i, 2, 2)?;
|
||||
expect(&mut i, b':')?;
|
||||
let minute = num(&mut i, 2, 2)?;
|
||||
expect(&mut i, b':')?;
|
||||
let second = num(&mut i, 2, 2)?;
|
||||
expect(&mut i, b' ')?;
|
||||
if day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return None;
|
||||
}
|
||||
let offset_minutes = match &s[i..] {
|
||||
b"GMT" | b"UTC" | b"UT" => 0,
|
||||
[sign @ (b'+' | b'-'), h1, h2, m1, m2] if [h1, h2, m1, m2].iter().all(|c| c.is_ascii_digit()) => {
|
||||
let minutes = ((h1 - b'0') as i64 * 10 + (h2 - b'0') as i64) * 60 + (m1 - b'0') as i64 * 10 + (m2 - b'0') as i64;
|
||||
if *sign == b'+' { minutes } else { -minutes }
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
let days = days_from_civil(year, month, day);
|
||||
Some((((days * 24 + hour) * 60 + minute - offset_minutes) * 60 + second) * 1000)
|
||||
}
|
||||
|
||||
/// `Date.parse` for the Date header (RFC 5322 and the obsolete forms); None for NaN.
|
||||
pub fn parse_date(s: &str) -> Option<i64> {
|
||||
let s = trim(s);
|
||||
if let Some(t) = fast_date(s.as_bytes()) {
|
||||
return Some(t);
|
||||
}
|
||||
parse_date_chrono(s)
|
||||
}
|
||||
|
||||
fn parse_date_chrono(s: &str) -> Option<i64> {
|
||||
// "UTC" isn't an RFC 5322 zone (UT and GMT are), but Date.parse takes it (~a quarter of a.b.boneless).
|
||||
let owned;
|
||||
let s = match s.strip_suffix(" UTC") {
|
||||
Some(rest) => {
|
||||
owned = format!("{rest} GMT");
|
||||
owned.as_str()
|
||||
}
|
||||
None => s,
|
||||
};
|
||||
if let Ok(d) = chrono::DateTime::parse_from_rfc2822(s) {
|
||||
return Some(d.timestamp_millis());
|
||||
}
|
||||
// "Mon, 5 Oct 2026 10:00:00 +0000 (UTC)" and other trailing comments.
|
||||
if let Some(i) = s.find(" (") {
|
||||
if let Ok(d) = chrono::DateTime::parse_from_rfc2822(&s[..i]) {
|
||||
return Some(d.timestamp_millis());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// sha1 of the UTF-8 text, as lowercase hex (node's `hash('sha1', text)`).
|
||||
pub fn sha1_hex(text: &str) -> String {
|
||||
let digest = Sha1::digest(text.as_bytes());
|
||||
let mut out = String::with_capacity(40);
|
||||
for b in digest {
|
||||
out.push(char::from_digit((b >> 4) as u32, 16).unwrap());
|
||||
out.push(char::from_digit((b & 15) as u32, 16).unwrap());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// JSON.stringify of a string.
|
||||
pub fn json_str(s: &str, out: &mut String) {
|
||||
out.push('"');
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\u{08}' => out.push_str("\\b"),
|
||||
'\u{0C}' => out.push_str("\\f"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
|
||||
c => out.push(c),
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn split_like_js() {
|
||||
assert_eq!(split_spaces("a b"), vec!["a", "b"]);
|
||||
assert_eq!(split_spaces(" a b "), vec!["", "a", "b", ""]);
|
||||
assert_eq!(split_spaces(""), vec![""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_int_like_js() {
|
||||
assert_eq!(parse_int("123abc"), Some(123));
|
||||
assert_eq!(parse_int(" -5"), Some(-5));
|
||||
assert_eq!(parse_int("x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dates() {
|
||||
assert_eq!(parse_date("Mon, 05 Oct 2026 10:00:00 +0000"), Some(1_791_194_400_000));
|
||||
assert_eq!(parse_date("5 Oct 2026 10:00:00 GMT"), Some(1_791_194_400_000));
|
||||
assert_eq!(parse_date("Mon, 05 Oct 26 10:00:00 UTC"), Some(1_791_194_400_000));
|
||||
for d in [
|
||||
"Mon, 05 Oct 2026 10:00:00 +0000", "Mon, 5 Oct 2026 12:00:00 +0200", "05 Oct 2026 10:00:00 GMT", "Thu, 01 Jan 1970 00:00:00 GMT",
|
||||
"Sat, 29 Feb 2020 23:59:59 -0130", "Fri, 31 Dec 99 00:00:00 UTC", "Mon, 05 Oct 2026 10:00:00 UT",
|
||||
] {
|
||||
assert_eq!(fast_date(d.as_bytes()), parse_date_chrono(d), "{d}");
|
||||
}
|
||||
assert_eq!(fast_date(b"Mon, 05 Oct 2026 10:00:00 EST"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! The edge worker's indexing (lib/edge RangeIndexer and what it uses) in
|
||||
//! Rust, kept identical to the TypeScript (check.ts): the edge-bench
|
||||
//! benchmark and the edge-node worker share it.
|
||||
|
||||
pub mod codec;
|
||||
pub mod indexer;
|
||||
pub mod js;
|
||||
pub mod names;
|
||||
pub mod wire;
|
||||
@@ -0,0 +1,243 @@
|
||||
//! The edge worker's indexing in Rust, on a range captured by `edge:bench
|
||||
//! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as
|
||||
//! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off
|
||||
//! idle ones, and every N chunks judge and encode the releases. `--dump`
|
||||
//! writes every output as JSON lines for edge/check.ts.
|
||||
//!
|
||||
//! edge-bench <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl]
|
||||
|
||||
use base64::Engine;
|
||||
use edge::indexer::{self, Batch, RangeIndexer};
|
||||
use edge::js::json_str;
|
||||
use edge::{names, wire};
|
||||
use rustc_hash::FxHashMap;
|
||||
use std::io::Write;
|
||||
use std::time::Instant;
|
||||
|
||||
// Many small strings (message-ids, names): the system allocator was a measurable share.
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
/// A chunk as the news server sent it (zlib; gzip also taken), whole: libdeflate wants the output
|
||||
/// size, so it grows until the text fits (~3.6 times the compressed size on boneless).
|
||||
fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Vec<u8> {
|
||||
let mut out = vec![0u8; raw.len() * 4 + 4096];
|
||||
loop {
|
||||
let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) };
|
||||
match result {
|
||||
Ok(n) => {
|
||||
out.truncate(n);
|
||||
return out;
|
||||
}
|
||||
Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0),
|
||||
Err(e) => panic!("inflate: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn json_field(json: &str, name: &str) -> String {
|
||||
let at = json.find(&format!("\"{name}\":")).unwrap() + name.len() + 3;
|
||||
let rest = json[at..].trim_start();
|
||||
if let Some(s) = rest.strip_prefix('"') {
|
||||
s[..s.find('"').unwrap()].to_string()
|
||||
} else {
|
||||
rest[..rest.find([',', '}']).unwrap()].trim().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn peak_rss_mb() -> u64 {
|
||||
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
|
||||
status.lines().find(|l| l.starts_with("VmHWM:")).and_then(|l| l.split_whitespace().nth(1)).and_then(|k| k.parse::<u64>().ok()).unwrap_or(0) / 1024
|
||||
}
|
||||
|
||||
fn opt_num(n: Option<i64>, out: &mut String) {
|
||||
match n {
|
||||
Some(n) => out.push_str(&n.to_string()),
|
||||
None => out.push_str("null"),
|
||||
}
|
||||
}
|
||||
|
||||
fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
let b64 = base64::engine::general_purpose::STANDARD;
|
||||
let mut line = String::new();
|
||||
let mut emit = |line: &mut String| {
|
||||
line.push('\n');
|
||||
out.write_all(line.as_bytes()).unwrap();
|
||||
line.clear();
|
||||
};
|
||||
line.push_str(&format!("{{\"t\":\"b\",\"from\":{},\"next\":{},\"headers\":{}}}", batch.from, batch.next, batch.headers));
|
||||
emit(&mut line);
|
||||
let release = |r: &indexer::ReleaseChunk, kind: &str, line: &mut String| {
|
||||
line.push_str(&format!("{{\"t\":\"{kind}\",\"upload\":{},\"key\":", r.upload));
|
||||
json_str(&r.key, line);
|
||||
line.push_str(",\"name\":");
|
||||
json_str(&r.name, line);
|
||||
line.push_str(&format!(",\"source\":\"{}\",\"fileTotal\":", r.source));
|
||||
opt_num(r.file_total, line);
|
||||
line.push_str(",\"poster\":");
|
||||
json_str(&r.poster, line);
|
||||
line.push_str(",\"groups\":[");
|
||||
for (i, g) in r.groups.iter().enumerate() {
|
||||
if i > 0 {
|
||||
line.push(',');
|
||||
}
|
||||
json_str(g, line);
|
||||
}
|
||||
line.push_str("],\"postedAt\":");
|
||||
opt_num(r.posted_at, line);
|
||||
line.push_str(",\"postedLast\":");
|
||||
opt_num(r.posted_last, line);
|
||||
line.push_str(&format!(
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
|
||||
r.files,
|
||||
r.segments,
|
||||
r.bytes,
|
||||
r.first_article,
|
||||
r.last_article,
|
||||
r.readable,
|
||||
r.has_par2,
|
||||
b64.encode(&r.payload)
|
||||
));
|
||||
};
|
||||
for r in &batch.releases {
|
||||
release(r, "r", &mut line);
|
||||
line.push('}');
|
||||
emit(&mut line);
|
||||
}
|
||||
for (r, rule, key) in &batch.samples {
|
||||
release(r, "s", &mut line);
|
||||
line.push_str(&format!(",\"rule\":\"{rule}\",\"ruleKey\":"));
|
||||
json_str(key, &mut line);
|
||||
line.push('}');
|
||||
emit(&mut line);
|
||||
}
|
||||
for c in &batch.candidates {
|
||||
line.push_str(&format!("{{\"t\":\"c\",\"upload\":{},\"releaseKey\":", c.upload));
|
||||
json_str(&c.release_key, &mut line);
|
||||
line.push_str(&format!(",\"kind\":\"{}\",\"messageId\":", c.kind));
|
||||
json_str(&c.message_id, &mut line);
|
||||
line.push_str(",\"file\":");
|
||||
json_str(&c.file, &mut line);
|
||||
line.push('}');
|
||||
emit(&mut line);
|
||||
}
|
||||
for (rule, key, articles, files) in &batch.counts {
|
||||
line.push_str(&format!("{{\"t\":\"n\",\"rule\":\"{rule}\",\"key\":"));
|
||||
json_str(key, &mut line);
|
||||
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
|
||||
emit(&mut line);
|
||||
}
|
||||
for f in &batch.fragments {
|
||||
line.push_str("{\"t\":\"f\",\"fileKey\":");
|
||||
json_str(&f.file_key, &mut line);
|
||||
line.push_str(",\"name\":");
|
||||
json_str(&f.name, &mut line);
|
||||
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
|
||||
opt_num(f.posted_at, &mut line);
|
||||
line.push_str(",\"segments\":[");
|
||||
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
|
||||
if i > 0 {
|
||||
line.push(',');
|
||||
}
|
||||
line.push_str(&format!("[{n},"));
|
||||
json_str(id, &mut line);
|
||||
line.push_str(&format!(",{bytes},{article}]"));
|
||||
}
|
||||
line.push_str("]}");
|
||||
emit(&mut line);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let dir = args.get(1).expect("usage: edge-bench <dir> [--batch-chunks N] [--idle N] [--dump file]").clone();
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
|
||||
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
|
||||
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
|
||||
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
|
||||
let group = json_field(&range, "group");
|
||||
let start: i64 = json_field(&range, "start").parse().unwrap();
|
||||
let end: i64 = json_field(&range, "end").parse().unwrap();
|
||||
let mut files: Vec<String> = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||
files.sort();
|
||||
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, names::Rules::defaults(priorities), idle);
|
||||
|
||||
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
let mut decompressor = libdeflater::Decompressor::new();
|
||||
let wall = Instant::now();
|
||||
for (i, file) in files.iter().enumerate() {
|
||||
let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
|
||||
compressed += raw.len();
|
||||
let t = Instant::now();
|
||||
let data = inflate(&raw, &mut decompressor);
|
||||
inflate_s += t.elapsed().as_secs_f64();
|
||||
text_bytes += data.len();
|
||||
let t = Instant::now();
|
||||
let text = String::from_utf8_lossy(&data);
|
||||
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||
indexer.add_text(&text, next);
|
||||
let last = i + 1 == files.len();
|
||||
if last {
|
||||
indexer.finish();
|
||||
}
|
||||
if !last && (i + 1) % batch_chunks != 0 {
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
continue;
|
||||
}
|
||||
let batch = indexer.take();
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
out.0 += batch.headers;
|
||||
out.1 += batch.releases.len();
|
||||
for r in &batch.releases {
|
||||
keys.insert(r.key.clone());
|
||||
out.2 += r.files;
|
||||
out.3 += r.segments;
|
||||
out.4 += r.payload.len();
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.fragments.len();
|
||||
out.7 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
}
|
||||
}
|
||||
let total = inflate_s + index_s;
|
||||
#[allow(static_mut_refs)]
|
||||
let p = unsafe { indexer::PROF };
|
||||
eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
|
||||
println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
|
||||
println!(
|
||||
"{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",
|
||||
out.0,
|
||||
total,
|
||||
out.0 as f64 / total,
|
||||
wall.elapsed().as_secs_f64()
|
||||
);
|
||||
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
|
||||
println!(
|
||||
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
|
||||
out.1,
|
||||
keys.len(),
|
||||
out.2,
|
||||
out.3,
|
||||
out.4 as f64 / 1e6,
|
||||
out.4 as f64 / out.3.max(1) as f64,
|
||||
out.5,
|
||||
out.6,
|
||||
out.7
|
||||
);
|
||||
}
|
||||
+1006
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,144 @@
|
||||
//! A batch as the coordinator takes it (lib/edge/wire.ts toWire): JSON with
|
||||
//! the TypeScript names, chunk payloads in base64.
|
||||
|
||||
use crate::indexer::{Batch, Candidate, Fragment, ReleaseChunk};
|
||||
use base64::Engine;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireRelease<'a> {
|
||||
key: &'a str,
|
||||
upload: usize,
|
||||
name: &'a str,
|
||||
source: &'a str,
|
||||
file_total: Option<i64>,
|
||||
poster: &'a str,
|
||||
groups: &'a [String],
|
||||
posted_at: Option<i64>,
|
||||
posted_last: Option<i64>,
|
||||
files: usize,
|
||||
segments: usize,
|
||||
bytes: i64,
|
||||
first_article: i64,
|
||||
last_article: i64,
|
||||
readable: bool,
|
||||
has_par2: bool,
|
||||
payload: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
rule: Option<&'a str>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
rule_key: Option<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireCandidate<'a> {
|
||||
release_key: &'a str,
|
||||
upload: usize,
|
||||
kind: &'a str,
|
||||
message_id: &'a str,
|
||||
file: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct WireCount<'a> {
|
||||
rule: &'a str,
|
||||
key: &'a str,
|
||||
articles: u64,
|
||||
files: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragmentRelease<'a> {
|
||||
key: &'a str,
|
||||
name: &'a str,
|
||||
source: &'a str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
file_total: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragment<'a> {
|
||||
file_key: &'a str,
|
||||
name: &'a str,
|
||||
poster: &'a str,
|
||||
subject: &'a str,
|
||||
total_parts: u32,
|
||||
groups: &'a [String],
|
||||
release: Option<WireFragmentRelease<'a>>,
|
||||
posted_at: Option<i64>,
|
||||
segments: Vec<(u32, &'a str, i64, i64)>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct WireBatch<'a> {
|
||||
from: i64,
|
||||
next: i64,
|
||||
headers: u64,
|
||||
releases: Vec<WireRelease<'a>>,
|
||||
samples: Vec<WireRelease<'a>>,
|
||||
candidates: Vec<WireCandidate<'a>>,
|
||||
counts: Vec<WireCount<'a>>,
|
||||
fragments: Vec<WireFragment<'a>>,
|
||||
}
|
||||
|
||||
fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRelease<'a> {
|
||||
WireRelease {
|
||||
key: &r.key,
|
||||
upload: r.upload,
|
||||
name: &r.name,
|
||||
source: r.source,
|
||||
file_total: r.file_total,
|
||||
poster: &r.poster,
|
||||
groups: &r.groups,
|
||||
posted_at: r.posted_at,
|
||||
posted_last: r.posted_last,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
bytes: r.bytes,
|
||||
first_article: r.first_article,
|
||||
last_article: r.last_article,
|
||||
readable: r.readable,
|
||||
has_par2: r.has_par2,
|
||||
payload: base64::engine::general_purpose::STANDARD.encode(&r.payload),
|
||||
rule: rule.map(|(rule, _)| rule),
|
||||
rule_key: rule.map(|(_, key)| key),
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate(c: &Candidate) -> WireCandidate<'_> {
|
||||
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
|
||||
}
|
||||
|
||||
fn fragment(f: &Fragment) -> WireFragment<'_> {
|
||||
WireFragment {
|
||||
file_key: &f.file_key,
|
||||
name: &f.name,
|
||||
poster: &f.poster,
|
||||
subject: &f.subject,
|
||||
total_parts: f.total_parts,
|
||||
groups: &f.groups,
|
||||
// A named file always has a release; fileTotal only when truthy, as fragmentOf.
|
||||
release: Some(WireFragmentRelease { key: &f.release.key, name: &f.release.name, source: f.release.source, file_total: f.release.file_total_truthy() }),
|
||||
posted_at: f.posted_at,
|
||||
segments: f.segments.iter().map(|(n, id, bytes, article)| (*n, id.as_str(), *bytes, *article)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// toWire(batch) as JSON bytes.
|
||||
pub fn to_json(batch: &Batch) -> Vec<u8> {
|
||||
let wire = WireBatch {
|
||||
from: batch.from,
|
||||
next: batch.next,
|
||||
headers: batch.headers,
|
||||
releases: batch.releases.iter().map(|r| release(r, None)).collect(),
|
||||
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
||||
candidates: batch.candidates.iter().map(candidate).collect(),
|
||||
counts: batch.counts.iter().map(|(rule, key, articles, files)| WireCount { rule, key, articles: *articles, files: *files }).collect(),
|
||||
fragments: batch.fragments.iter().map(fragment).collect(),
|
||||
};
|
||||
serde_json::to_vec(&wire).unwrap()
|
||||
}
|
||||
Reference in new issue
Block a user