Compare commits
34
Commits
29e0cc8f6e
...
rebuild
No files matched your search
@@ -59,35 +59,28 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Rust toolchain
|
||||
run: |
|
||||
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
# The edge worker (edge/, Rust): built first, since the e2e suite runs edge/target/release/edge-node against
|
||||
# the node API; then its own tests, and the Rust and TypeScript RangeIndexers compared on the committed
|
||||
# captures (edge/fixtures).
|
||||
- name: Build the edge worker
|
||||
run: cargo build --release --manifest-path edge/Cargo.toml --bin edge-node
|
||||
- name: Type check
|
||||
run: npx tsc --noEmit
|
||||
- name: Migrate
|
||||
run: node ace migration:run --force
|
||||
- name: Test
|
||||
run: node ace test
|
||||
|
||||
# The edge worker (edge/, Rust): its tests, and the Rust and TypeScript
|
||||
# RangeIndexers compared on the committed captures (edge/fixtures).
|
||||
edge:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Rust toolchain
|
||||
run: |
|
||||
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
- name: cargo test
|
||||
run: cargo test --release --manifest-path edge/Cargo.toml
|
||||
- name: The Rust indexer over the fixtures
|
||||
run: edge/bench-fixtures.sh
|
||||
|
||||
deploy:
|
||||
needs: [test, edge]
|
||||
needs: [test]
|
||||
if: github.ref == 'refs/heads/rebuild'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+174
-17
@@ -60,9 +60,21 @@ most workers on batcave, a few on poweredge-kantoor with
|
||||
`edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
|
||||
the cap is shared by the node's processes).
|
||||
|
||||
Every group is tiled from article 1 (2026-10-28): one "ancient" range,
|
||||
status `ancient`, covers 1 … low watermark − 1, what had expired on the
|
||||
server before the group was planned. Never leased, fetched or filled —
|
||||
nothing selects it — and the status panel reports it apart ("Ancient: …
|
||||
articles below the low watermark"); migration 6700 added it for the
|
||||
groups planned before, `Planner.historyRows` for groups planned since.
|
||||
|
||||
Also built (2026-10-09): the live tail (new articles planned every 2
|
||||
minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
|
||||
fetched before history); the catalog API, Newznab (`/api`) and the web
|
||||
fetched before history; since 2026-10-27 a round grows the group's open
|
||||
tail range, a planned one at the tail's priority nothing spooled or
|
||||
took, up to RANGE_ARTICLES before it adds rows, as a tail nothing leased
|
||||
had become a sliver a group a round, 127k rows of a median 14k articles;
|
||||
`ranges:merge-tail` packs what such rounds left); the catalog API,
|
||||
Newznab (`/api`) and the web
|
||||
interface, from voyager 1 (the current build), with categories from the
|
||||
name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
|
||||
`database/data/voyager1_signatures.tsv`).
|
||||
@@ -139,7 +151,9 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
indexed groups in the article's Xref, the one with the lowest priority
|
||||
number; see the group list) → group articles into files → files
|
||||
into releases (`lib/release` rules) → drop garbage (random name +
|
||||
dropped signature or tool) → set aside per-article random subjects and
|
||||
dropped signature or tool), bare executables, dropped posters, and the
|
||||
single-archive fakes (stub archives and nonsense pieces, below) → set
|
||||
aside per-article random subjects and
|
||||
obfuscated subjects (counted only; phase 2). Returns per batch: release
|
||||
chunks, edge fragments, probe candidates, counters. Holds no state beyond
|
||||
its lease.
|
||||
@@ -327,8 +341,13 @@ or music name whose content is a program: archive entries all
|
||||
executables, or the file itself one by name or by its first bytes, sniffed
|
||||
`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
|
||||
*locked* (passworded, no password from a spot, its NZB or the title),
|
||||
*unusable* (random names, no inner name, no par2 or inner-name merge, no
|
||||
spot, a day after settling). Verdicts (`release_verdicts`, with the rule
|
||||
*tiny* (one file of one article, 2–100 kB, under a readable release-like
|
||||
name outside the music, e-book and comic groups, without an e-book,
|
||||
subtitle, crack or sidecar word: the catalog's side of the edge's
|
||||
tiny-archive rule, 1.5M rows of the 2015–17 flood that the edge indexed
|
||||
before the rule; LOG.md (131), (135)), *unusable* (random names, no inner
|
||||
name, no par2 or inner-name merge, no spot, a day after settling).
|
||||
Verdicts (`release_verdicts`, with the rule
|
||||
set's version) hide releases from search (`judged` shows them); a verdict
|
||||
whose rule no longer matches is taken back. Nothing is dropped yet.
|
||||
|
||||
@@ -459,6 +478,13 @@ releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
||||
merger: the carry does the joining. The settler goes: `settled_at` is
|
||||
the finish. The **tail** needs no path of its own: its lease ends leave
|
||||
such provisional releases, the next lease's continuation finishes them.
|
||||
One exception (2026-10-27): a provisional release that is complete
|
||||
(every file its counter announced found) and quiet for six hours is
|
||||
settled by the coordinator (`Settler.settleComplete`, every five
|
||||
minutes), as the finish would: no counted file can still come, and the
|
||||
continuation may be hours away or never (the first run's range leases
|
||||
left 109k provisional rows at their blocks' end edges, 606 of them
|
||||
complete, shown as still collecting; the user, release 10697).
|
||||
|
||||
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
||||
only by a number sequence, or share a directory-like prefix, posted
|
||||
@@ -544,6 +570,57 @@ releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
||||
(a live reply cut short; the spool doesn't check) is reported after
|
||||
the batch before it went out and the range asked for again from that
|
||||
chunk: the window stays (the first run lost it, and the open batch).
|
||||
The lease bounds alone don't select a window: the history planner's
|
||||
oldest buckets are priority 0–2 too, so the first run also took 2026
|
||||
history at 0 (25k ranges in 40 groups, fetched live) while 1,739 of
|
||||
the 2016 corpus ranges sat at voyager 2's leftover priorities (done
|
||||
when the window was marked, planned again by the reset): hence
|
||||
`LEASE_CAPTURE_ONLY` (the lease, next and the prefetcher's run-ahead
|
||||
take capture ranges only) and the one-off repair of the priorities.
|
||||
The stream polls the corpus as well as the spool while it waits for a
|
||||
chunk: the corpus fill claims chunks of the same range ahead of the
|
||||
leased run (the prefetcher's claims are per range and chunk), stores
|
||||
them in the corpus only, and the run leaves them there; a stream that
|
||||
looked at the corpus once, seconds before that store, waited its whole
|
||||
hour on the spool and the worker lost the group and its open batch
|
||||
(three groups on the first morning). A failed stream now posts the
|
||||
open batch before the group goes back, like a stop does, so the
|
||||
window's snapshot survives it; only a batch the coordinator refused
|
||||
(lease lost, too large) still loses the window.
|
||||
**Two lanes a group (2026-10-27; migration 6600, LOG.md (119)).** A
|
||||
group's ranges in article order through one window meant a thread
|
||||
that leased a group for its tail range first waded through the
|
||||
group's history (older articles come first), and a held history
|
||||
group's tail waited behind days of its ranges; and the bounds that ran
|
||||
the 2016 window alone (0..0, capture only) shut the tail out entirely.
|
||||
Now `group_leases` has a row per group and *lane*: the `history` lane
|
||||
works the ranges the bounds hand out (the windows, the history), the
|
||||
`tail` lane the tail planner's (priority at or below the tail's,
|
||||
whatever the bounds), each with its own holder, window, snapshot
|
||||
(`<group>-tail.gz`) and takeover redo (the lane's ranges only, and
|
||||
only those the window did after the snapshot: leased, done since it,
|
||||
or read past it). The edge puts `--tail-threads` of its threads on the
|
||||
tail lane (EDGE_TAIL_THREADS; 1 of 2 or more by default, which is
|
||||
what homelab's 16 run with), the rest on history; the prefetcher runs ahead of every held
|
||||
cursor. The seam a group is the same one as before, once: where the
|
||||
history lane reaches the first range the tail lane did, it is caught
|
||||
up. **The lease order** within a lane and priority is a mix (the
|
||||
user, 2026-10-27): one lease starts from the group with the most
|
||||
planned articles, the next from the longest unheld one (never held
|
||||
first), alternating on a sequence (`group_lease_turns`), so a big
|
||||
group snapshotted at a stop doesn't wait behind hundreds of small
|
||||
never-held groups and a quiet group's tail is visited all the same.
|
||||
**`coordinator:status`** (`app/services/status.ts`, `--json`) prints
|
||||
the catalog's room against `CATALOG_MAX_NVME_GB`, the ranges by band
|
||||
(tail, raised, each capture window, the history by thousand), the
|
||||
last hour's indexed and stored articles, the spool, the tail backlog,
|
||||
the held groups with their snapshots' age and size, the snapshot
|
||||
files, and the phase-2 backlogs (provisional releases, probe
|
||||
candidates, the pending ladder, the judge's dirty marks). **End to
|
||||
end** (`tests/e2e`, suite `e2e`, CI builds the binary first): the Rust
|
||||
worker against the real node API, database, spool and corpus — the
|
||||
corpus-fill race, a failed stream and the restart that restores the
|
||||
snapshot, a restart whose snapshot file is gone, a takeover.
|
||||
|
||||
6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
|
||||
(30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
|
||||
@@ -640,10 +717,13 @@ releases; the inspector matches them again), `nntp_shares`, `nodes`,
|
||||
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
|
||||
`settled_at IS NOT NULL`), the settler, the fragment merger,
|
||||
`releases_category_newest_idx`.
|
||||
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
|
||||
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
|
||||
rule read all of `release_inspections` every round, 17 min on the HDD;
|
||||
the judge must read it by release or keep what it needs on the NVMe.
|
||||
- `release_inspections` and `probe_candidates` were on `CHUNK_TABLESPACE`
|
||||
(the HDD) with their indexes; both are back on the NVMe (migrations 4800
|
||||
and 7300). Note LOG.md 2026-10-19 (81): the judge's fake rule read all of
|
||||
`release_inspections` every round, 17 min on the HDD; and 2026-11-03
|
||||
(136): the prober's catalog claim updates `probe_candidates` from random
|
||||
points every minute, 2–3 min on the HDD beside the pending ladder's sweep.
|
||||
What a worker reads or updates by random key every minute stays on the NVMe.
|
||||
|
||||
### Batch store (built 2026-10-25)
|
||||
|
||||
@@ -704,7 +784,49 @@ is before frontier − `WINDOW_MS` are judged and written `finished`, as an
|
||||
it, else `finished: false, edge: true`. A file short of parts stays in
|
||||
its release and sets `partial` (the store: `complete = false`). A lone
|
||||
article of a multi-part random file is still counted `per-article`, not
|
||||
written. Gone: fragments and the cut-off, `uploads_of` (online now),
|
||||
written; before the block's end a scrambled name counts as random too
|
||||
(`looks_scrambled`: a random subject and poster per article, 160k of
|
||||
alt.binaries.storage's releases; LOG.md (122)).
|
||||
|
||||
**Single-archive rules (built 2026-10-31; LOG.md (123)-(127)).** The judge
|
||||
(`names::judge`, `RuleSet.judge`) gets each file's articles seen and their
|
||||
bytes, and whether the release is *settled* (the window closed on it: a
|
||||
par2 can still join an open one; the start edge doesn't count, since a par2
|
||||
follows its rar and 7% of the catalog's releases are start edges). A settled
|
||||
release of one unsplit zip, rar or 7z and nothing else is dropped as a
|
||||
**stub archive** (`stub-archive`, key: the legs tripped) when posted to
|
||||
2+ groups, under 100 MB (the fakes stop at ~90 MB; hash-named uploads,
|
||||
real albums and passworded videos above it don't), and (A) its articles
|
||||
are ≤ 256 KiB with the release ≥ 1 MB (Power-Post's 250 kB default; a
|
||||
one-article release always has small articles, which exempts e-books and
|
||||
sidecars), or (X) posted to 4+ groups, or (W) by a webmail poster. A
|
||||
ROT13 volume name (".cnegN.rar" is ".partN.rar": a.b.erotica's passworded
|
||||
videos, LOG.md (128)) is a volume, not a single archive. A release
|
||||
touching a music, e-book, comics, tabs or fonts group is exempt: those
|
||||
communities post real single small archives the same way (karaoke zips to
|
||||
a.b.sounds.karaoke + a.b.test with Power-Post's articles; 16 of 32,308
|
||||
known fakes lost, LOG.md (127)). These are the
|
||||
exe-stub and recycled-installer floods under film and software names:
|
||||
55% of the catalog on 2026-10-30, 99.3% of the known fakes, 0 known
|
||||
content in the 217k sample, verified on 1,500 decoded articles. A settled
|
||||
single archive of one article and 100 kB–1 MB under a release-like,
|
||||
unscrambled name without spaces and without an e-book format in it is
|
||||
dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool):
|
||||
the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of
|
||||
the catalog; under 100 kB are cue/log sidecars and adverts, and the names
|
||||
with spaces are books and albums. Below that, a settled single archive of
|
||||
one article and 2–100 kB under a release-like name (spaces allowed), outside
|
||||
the exempt groups and without an e-book, subtitle, crack/keygen/trainer,
|
||||
cover or sidecar word, is dropped as a **tiny archive** (`tiny-archive`,
|
||||
key: the upload tool; LOG.md (131), (133)): the 2015–17 flood's RARs
|
||||
holding "<title>.exe" at 2–60 kB, 1.66M catalog rows from a dozen posters
|
||||
(12 of 12 fetched fake); the word exemptions keep the real tiny archives
|
||||
(subtitles, cracks, trainers) at the price of ~50k fakes named "Incl
|
||||
Keygen" that the prober judges later. The rules are switched by the
|
||||
rule set (`dropStubArchives`, `dropNonsensePieces`, `dropTinyArchives`),
|
||||
sampled like every drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
|
||||
judges as each batch completes files, so it passes settled without the
|
||||
window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now),
|
||||
`check.ts` and the fixture comparison (the parity with the TypeScript
|
||||
indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
|
||||
(incomplete releases are kept). The wire carries `finished`, `edge` and
|
||||
@@ -773,7 +895,11 @@ e. Headers through the coordinator's relay, with prefetch (part 1).
|
||||
f. **Nothing enters the catalog without a usable name** (amended
|
||||
2026-10-26, the user's direction; before: random-named releases keep
|
||||
no chunk but their rows). A release the window closed on whose files
|
||||
all have random names is no release yet: it goes to `pending_releases`
|
||||
all have random names (`isUnreadableName`, `names::is_unreadable_name`:
|
||||
a name read token by token, so separators, spaces and random
|
||||
"extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
|
||||
when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
|
||||
and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
|
||||
(migration 6300, the HDD) with its keys, counts, article span, upload
|
||||
tool and probe targets, no row in `releases`, `release_texts` or
|
||||
`posters`, no chunk. The pending ladder works them **from least to most
|
||||
@@ -787,13 +913,29 @@ f. **Nothing enters the catalog without a usable name** (amended
|
||||
yields none counts against its signature, and a signature with
|
||||
≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
|
||||
2026-10-08 (5)) is **discarded** from then on, a sample of it still
|
||||
probed to notice a change. The reason, in the 2016 replay: one poster's
|
||||
probed to notice a change. **By class** (2026-11-02, LOG.md (130)): a
|
||||
pending release's class is its tool, group, size bucket, name shape and
|
||||
partial flag (`lib/release/classes.ts`, `names::pending_class`, the SQL
|
||||
`pending_class()`; `pending_releases.klass`), and `pending_classes` keeps
|
||||
each class's climbs and names; the claim takes the classes with the best
|
||||
record first (`classScore`, a prior for the unexplored, at most a quarter
|
||||
of a claim from one class), the cheapest target article first within one
|
||||
(`target_bytes`), and the oldest of the rest; a class with the learner's
|
||||
thresholds is dropped like a signature, and the rule set's
|
||||
`droppedClasses` keep its releases out of the pending store at the edge,
|
||||
the explored sample excepted. A probed file whose first 16 KiB and size
|
||||
match a named catalog release's (`file_hashes`) takes that name without
|
||||
another fetch (`app/services/hash_twins.ts`), on the ladder and for
|
||||
catalog releases without one. The reason, in the 2016 replay: one poster's
|
||||
hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
|
||||
groups) were 76% of the rows and three quarters of the NVMe bytes for
|
||||
11% of the segments and nothing anyone can find; the full history has
|
||||
725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
|
||||
edge sends an unnamed release with an empty payload and its tool, the
|
||||
batch store writes `pending_releases` (LOG.md (114)); the ladder in the
|
||||
batch store writes `pending_releases` (LOG.md (114)); a finished release
|
||||
the edge found no target for (the edge's `candidates_of`: par2, archive
|
||||
start, 7z tail, else the first random-named file as the lone target;
|
||||
LOG.md (140)) is written exhausted at once; the ladder in the
|
||||
prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
|
||||
the oldest waiting releases' next target (par2, archive, lone, 7z-tail
|
||||
in that order), at least PENDING_SHARE (25%) of each claim next to the
|
||||
@@ -804,13 +946,28 @@ f. **Nothing enters the catalog without a usable name** (amended
|
||||
target or status 3 (exhausted); each finished climb counts on
|
||||
`pending_signatures` for (tool, group, poster) and (tool, group), and a
|
||||
signature with PENDING_MIN_PROBES (100) climbs and at most
|
||||
PENDING_MAX_NAMED (0.5%) named is dropped: every five minutes the
|
||||
waiting releases of dropped signatures are discarded (status 2), the
|
||||
rules' `samplePercent` of them left to climb. The namer
|
||||
PENDING_MAX_NAMED (0.5%) named is dropped: a sweep beside the lanes
|
||||
(every five minutes, in its own loop) discards the waiting releases
|
||||
of each dropped signature and class once (`swept_at`; the edge drops
|
||||
their new releases itself), the rules' `samplePercent` of them left to
|
||||
climb, each by an index (`pending_releases_signature_idx` for a
|
||||
poster's rows: tested on the rows, a signature was a scan of the whole
|
||||
table; LOG.md (133), (134)). The batch store, the record, the claim
|
||||
and the sweep lock pending rows in one order, by key and the counters
|
||||
after (`lockOrder` for the multi-row statements; the record and the
|
||||
sweep lock their rows with `SELECT … ORDER BY release_key FOR UPDATE`
|
||||
first, since an UPDATE's join or `= ANY(subquery)` locks in plan
|
||||
order), and a flush that deadlocks anyway keeps its outcomes for the
|
||||
next (LOG.md (134), (137)). The namer
|
||||
(`app/services/pending_namer.ts`, in the prober's process): a named
|
||||
release's article span read again from the corpus (the stored range's
|
||||
chunks) or by XOVER on a connection of its own, through the TypeScript
|
||||
indexer, written as a release under the pending key with `source`
|
||||
chunks) or by XOVER on a connection of its own, a chunk of articles
|
||||
at a time (one string of a wide span topped V8's limit; spans over 3M
|
||||
articles are given up), through the TypeScript
|
||||
indexer with `partialFiles` (a file given up short of parts stays a
|
||||
partial file of its release, as the Rust window writes it; without it
|
||||
the range path's fragment rule left nothing to find: by 2026-10-29
|
||||
about 52k of 210k named releases failed, all partial), written as a release under the pending key with `source`
|
||||
weak-stem, the name as `inner_name`, its chunk and its candidates (the
|
||||
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
|
||||
the password flag). Not built: the spot/NZB match on pending keys (step
|
||||
|
||||
@@ -91,6 +91,12 @@ export default defineConfig({
|
||||
name: 'functional',
|
||||
timeout: 30000,
|
||||
},
|
||||
// The Rust edge worker against the real node API (tests/e2e): needs edge/target/release/edge-node.
|
||||
{
|
||||
files: ['tests/e2e/**/*.spec.{ts,js}'],
|
||||
name: 'e2e',
|
||||
timeout: 120000,
|
||||
},
|
||||
],
|
||||
forceExit: false,
|
||||
},
|
||||
|
||||
@@ -3,18 +3,13 @@ import { Readable } from 'node:stream'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import Leases, { LeaseLost } from '#services/leases'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import GroupLeases, { LANES } from '#services/group_leases'
|
||||
import BatchStore, { StaleBatch } from '#services/batch_store'
|
||||
import Spool from '#services/spool'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
/**
|
||||
* How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
|
||||
* coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
|
||||
* (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
|
||||
*/
|
||||
const CHUNK_WAIT_MS = 60 * 60 * 1000
|
||||
/** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
|
||||
const KEEPALIVE_MS = 20_000
|
||||
|
||||
@@ -35,16 +30,29 @@ export default class NodeController {
|
||||
|
||||
// Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
|
||||
// ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
|
||||
// `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
|
||||
|
||||
/** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
|
||||
private lane(ctx: HttpContext): Lane | null {
|
||||
const lane = ctx.request.qs().lane ?? 'history'
|
||||
if (LANES.includes(lane)) return lane
|
||||
ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
|
||||
return null
|
||||
}
|
||||
|
||||
async leaseGroup(ctx: HttpContext) {
|
||||
const lease = await GroupLeases.lease(this.node(ctx))
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const lease = await GroupLeases.lease(this.node(ctx), lane)
|
||||
if (!lease) return ctx.response.noContent()
|
||||
return lease
|
||||
}
|
||||
|
||||
async nextRange(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx))
|
||||
const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!next) return ctx.response.noContent()
|
||||
return next
|
||||
} catch (error) {
|
||||
@@ -54,11 +62,15 @@ export default class NodeController {
|
||||
}
|
||||
|
||||
async groupHeartbeat(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId))
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
async releaseGroup(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId))
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,16 +78,20 @@ export default class NodeController {
|
||||
* (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
|
||||
*/
|
||||
async putSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const next = Number(ctx.request.qs().next)
|
||||
if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
|
||||
const raw = ctx.request.raw()
|
||||
const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body))
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
|
||||
}
|
||||
|
||||
async getSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx))
|
||||
const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
|
||||
ctx.response.header('Content-Type', 'application/octet-stream')
|
||||
ctx.response.header('Content-Length', String(snapshot.size))
|
||||
@@ -99,19 +115,22 @@ export default class NodeController {
|
||||
const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
|
||||
const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
|
||||
const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
|
||||
// The chunk from the corpus or the spool, whichever has it. Both are polled while waiting: the corpus fill
|
||||
// claims a chunk ahead of the leased run (the prefetcher's claims are per range and chunk), stores it in the
|
||||
// corpus only, and the run then leaves it there — a stream that looked at the corpus once, before that
|
||||
// store, waited the whole hour on the spool (three groups on 2026-10-27, LOG.md (118)).
|
||||
const stored = async (span: ChunkSpan) =>
|
||||
corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
|
||||
async function* frames() {
|
||||
for (const span of spans) {
|
||||
if (corpusRange && (await Corpus.has(corpusRange, span))) {
|
||||
const data = await Corpus.read(corpusRange, span)
|
||||
if (data) {
|
||||
yield encodeFrame({ ...span, data })
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
continue
|
||||
}
|
||||
}
|
||||
const waitUntil = Date.now() + CHUNK_WAIT_MS
|
||||
const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
|
||||
let quietSince = Date.now()
|
||||
while (!(await Spool.has(rangeId, span))) {
|
||||
let data = await stored(span)
|
||||
while (!data) {
|
||||
if (await Spool.has(rangeId, span)) {
|
||||
data = await Spool.read(rangeId, span)
|
||||
break
|
||||
}
|
||||
if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
|
||||
if (Date.now() - quietSince >= KEEPALIVE_MS) {
|
||||
quietSince = Date.now()
|
||||
@@ -119,8 +138,9 @@ export default class NodeController {
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
await sleep(500)
|
||||
data = await stored(span)
|
||||
}
|
||||
yield encodeFrame({ ...span, data: await Spool.read(rangeId, span) })
|
||||
yield encodeFrame({ ...span, data })
|
||||
// Streaming keeps the lease alive.
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
@@ -148,7 +168,7 @@ export default class NodeController {
|
||||
return this.lifecycle(ctx, async (rangeId, nodeId) => {
|
||||
const range = await Leases.leased(rangeId, nodeId)
|
||||
await Leases.finish(rangeId, nodeId)
|
||||
await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers))
|
||||
await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+52
-12
@@ -9,6 +9,7 @@ import { SIDECAR } from '#services/sidecar_folder'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
import { uploadTool } from '#lib/rules/index'
|
||||
import { PENDING, byEffort, lockOrder } from '#services/pending_ladder'
|
||||
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
|
||||
|
||||
/**
|
||||
@@ -179,16 +180,21 @@ class BatchStore {
|
||||
candidates: ProbeCandidate[]
|
||||
): Promise<number> {
|
||||
if (releases.length === 0) return 0
|
||||
const targets = new Map<string, { kind: string; messageId: string; file: string }[]>()
|
||||
const targets = new Map<
|
||||
string,
|
||||
{ kind: string; messageId: string; file: string; bytes?: number }[]
|
||||
>()
|
||||
for (const c of candidates) {
|
||||
const key = slot(c.releaseKey, c.upload ?? 0)
|
||||
const list = targets.get(key) ?? []
|
||||
list.push({ kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
|
||||
targets.set(key, list)
|
||||
}
|
||||
let written = 0
|
||||
for (let i = 0; i < releases.length; i += 1000) {
|
||||
const rows = releases.slice(i, i + 1000).map((r) => {
|
||||
// In the record's lock order (LOG.md (136)): all rows sorted, then all classes sorted, in one transaction. The
|
||||
// upsert locks a finished row too, before its WHERE passes it by; a chunk's classes before the next chunk's rows
|
||||
// crossed the ladder's record(), which holds its rows and then takes its classes.
|
||||
const rows = lockOrder(
|
||||
releases.map((r) => {
|
||||
const probes = targets.get(slot(r.key, r.upload)) ?? []
|
||||
return {
|
||||
release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
|
||||
@@ -207,25 +213,59 @@ class BatchStore {
|
||||
file_total: r.fileTotal,
|
||||
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
|
||||
candidates: probes,
|
||||
// A finished release without a target is through the ladder before it starts: exhausted, not waiting
|
||||
// (1.57M rows waited forever, never claimed, before the edge gave the hash-named pieces one; LOG.md (140)).
|
||||
status: r.finished !== false && probes.length === 0 ? PENDING.exhausted : PENDING.waiting,
|
||||
// The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
|
||||
target_bytes: byEffort(probes)[0]?.bytes ?? null,
|
||||
}
|
||||
})
|
||||
}),
|
||||
(r) => r.release_key
|
||||
)
|
||||
let written = 0
|
||||
const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
|
||||
for (let i = 0; i < rows.length; i += 1000) {
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
|
||||
files, segments, bytes, partial, finished, file_total, tool, candidates)
|
||||
files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes, status)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
|
||||
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates
|
||||
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates,
|
||||
pending_class(v.name, v.bytes, v.partial), v.target_bytes, v.status
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
|
||||
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb)
|
||||
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb, target_bytes bigint, status int)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
|
||||
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
|
||||
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
|
||||
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END
|
||||
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END,
|
||||
klass = pending_class(pending_releases.name, pending_releases.bytes + excluded.bytes, excluded.partial),
|
||||
target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes),
|
||||
status = CASE WHEN pending_releases.status = ? AND excluded.finished
|
||||
AND jsonb_array_length(pending_releases.candidates) = 0 AND jsonb_array_length(excluded.candidates) = 0
|
||||
THEN ? ELSE pending_releases.status END
|
||||
WHERE NOT pending_releases.finished
|
||||
RETURNING release_key`,
|
||||
[groupId, JSON.stringify(rows)]
|
||||
RETURNING release_key, tool, klass, (xmax = 0) AS inserted, status`,
|
||||
[groupId, JSON.stringify(rows.slice(i, i + 1000)), PENDING.waiting, PENDING.exhausted]
|
||||
)
|
||||
written += done.length
|
||||
// Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
|
||||
for (const d of done) {
|
||||
const key = `${d.tool}\t${d.klass}`
|
||||
const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
|
||||
if (d.inserted && Number(d.status) === PENDING.waiting) entry.waiting++
|
||||
classes.set(key, entry)
|
||||
}
|
||||
}
|
||||
if (classes.size > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_classes (tool, group_id, klass, waiting)
|
||||
SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
|
||||
[
|
||||
groupId,
|
||||
JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${groupId}\t${c.klass}`)),
|
||||
]
|
||||
)
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
+183
-67
@@ -8,150 +8,266 @@ import coordinatorConfig from '#config/coordinator'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { LeaseLost } from '#services/leases'
|
||||
import type { GroupLease, NextRange } from '#lib/edge/index'
|
||||
import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
|
||||
|
||||
export const LANES: Lane[] = ['history', 'tail']
|
||||
|
||||
/**
|
||||
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
|
||||
* next range each time it finished one, so a group's ranges are indexed in article order through one carried
|
||||
* window. The coordinator keeps one row a group (group_leases): the holder, its heartbeat, the range it works and
|
||||
* the snapshot its window stands at.
|
||||
* window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
|
||||
* works and the snapshot its window stands at.
|
||||
*
|
||||
* - lease: the free group with the most urgent planned range (the ranges' priority: the tail before history, then
|
||||
* the windows), the longest unheld first; a holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group
|
||||
* resets what the window's snapshot doesn't cover: the ranges from the snapshot's article on are planned again
|
||||
* (the batch store's keys make the redo idempotent); without a snapshot the window is lost and the group
|
||||
* continues at its resume point (one seam).
|
||||
* - next: the group's lowest planned range (or the one this node already has), leased to the node; none when the
|
||||
* group is caught up: the worker snapshots and gives the group back, and gets it again when the tail planned
|
||||
* more.
|
||||
* - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR), given to the next holder.
|
||||
* - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
|
||||
* LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
|
||||
* the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
|
||||
* group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
|
||||
* the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
|
||||
* reaches the tail lane's first range (its ranges are done: the history lane is caught up).
|
||||
* - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
|
||||
* mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
|
||||
* big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
|
||||
* is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
|
||||
* lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
|
||||
* (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
|
||||
* idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
|
||||
* - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
|
||||
* when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
|
||||
* planned more.
|
||||
* - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
|
||||
* `<group>-tail.gz`), given to the next holder of the lane.
|
||||
*/
|
||||
class GroupLeases {
|
||||
get dir(): string {
|
||||
return coordinatorConfig.snapshotDir
|
||||
}
|
||||
|
||||
async lease(nodeId: number): Promise<GroupLease | null> {
|
||||
/**
|
||||
* The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
|
||||
* above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
|
||||
*/
|
||||
private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
|
||||
if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
|
||||
const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
|
||||
return {
|
||||
sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
|
||||
bindings: [tail, min, max],
|
||||
}
|
||||
}
|
||||
|
||||
async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
|
||||
// The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
|
||||
if (lane === 'history' && (await CatalogSpace.full())) return null
|
||||
const rules = await RuleSets.current()
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
|
||||
// The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
|
||||
const history = (await CatalogSpace.full()) ? 'AND r.priority < 0' : ''
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
return db.transaction(async (trx) => {
|
||||
// One lease at a time: the choice reads several tables.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
// The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
|
||||
// last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
|
||||
// most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
|
||||
// the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
|
||||
// win).
|
||||
const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
|
||||
const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT g.id, g.name, g.priority, l.node_id AS holder, l.snapshot_next, todo.priority AS todo_priority
|
||||
FROM groups g
|
||||
JOIN LATERAL (SELECT min(r.priority) AS priority FROM ranges r WHERE r.group_id = g.id AND r.status = 'planned' AND r.next_id <= r.end_id AND r.priority BETWEEN ? AND ? ${history}) todo ON true
|
||||
LEFT JOIN group_leases l ON l.group_id = g.id
|
||||
WHERE g.active AND todo.priority IS NOT NULL
|
||||
AND (l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?))
|
||||
ORDER BY todo.priority, l.heartbeat_at NULLS FIRST, g.priority
|
||||
LIMIT 1`,
|
||||
[min, max, ttl]
|
||||
`WITH todo AS (
|
||||
SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
|
||||
FROM ranges r
|
||||
WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
|
||||
AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
GROUP BY r.group_id
|
||||
), free AS (
|
||||
SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
|
||||
todo.priority, todo.articles
|
||||
FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
|
||||
LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
|
||||
WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
|
||||
), turns AS (
|
||||
SELECT free.*,
|
||||
least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
|
||||
2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
|
||||
FROM free
|
||||
)
|
||||
SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
|
||||
[ttl, ...todo.bindings, lane, ttl, first, second]
|
||||
)
|
||||
const row = rows[0]
|
||||
if (!row) return null
|
||||
const groupId = Number(row.id)
|
||||
// What the snapshot doesn't cover is read again: the ranges from its article on, the one holding it from there.
|
||||
// What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
|
||||
// after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
|
||||
if (row.snapshot_next !== null) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges SET status = 'planned', node_id = NULL, next_id = greatest(start_id, ?::bigint)
|
||||
WHERE group_id = ? AND end_id >= ?::bigint AND (status IN ('leased', 'done') OR next_id > ?::bigint)`,
|
||||
[row.snapshot_next, groupId, row.snapshot_next, row.snapshot_next]
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
|
||||
WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
|
||||
AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
|
||||
[
|
||||
row.snapshot_next,
|
||||
groupId,
|
||||
row.snapshot_next,
|
||||
...all.bindings,
|
||||
row.snapshot_at ?? new Date(0),
|
||||
row.snapshot_next,
|
||||
]
|
||||
)
|
||||
} else {
|
||||
await trx.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL WHERE group_id = ? AND status = 'leased'`, [groupId])
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
|
||||
[groupId, ...all.bindings]
|
||||
)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO group_leases (group_id, node_id, leased_at, heartbeat_at, range_id)
|
||||
VALUES (?, ?, now(), now(), NULL)
|
||||
ON CONFLICT (group_id) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
|
||||
[groupId, nodeId]
|
||||
`INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
|
||||
VALUES (?, ?, ?, now(), now(), NULL)
|
||||
ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
|
||||
[groupId, lane, nodeId]
|
||||
)
|
||||
await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
return {
|
||||
group: { id: groupId, name: row.name, priority: Number(row.priority) },
|
||||
group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
|
||||
lane,
|
||||
rules,
|
||||
snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** The group's lease row, when this node holds it. */
|
||||
async held(groupId: number, nodeId: number) {
|
||||
const lease = await db.from('group_leases').where('group_id', groupId).first()
|
||||
if (!lease || Number(lease.node_id) !== nodeId) throw new LeaseLost(`Group ${groupId} is not held by this node`)
|
||||
/** The group's lease row of the lane, when this node holds it. */
|
||||
async held(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
const lease = await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.first()
|
||||
if (!lease || Number(lease.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
|
||||
return lease
|
||||
}
|
||||
|
||||
/** The group's next range in article order (or the one this node has), leased to the node; null when caught up. */
|
||||
async next(groupId: number, nodeId: number): Promise<NextRange | null> {
|
||||
await this.held(groupId, nodeId)
|
||||
/**
|
||||
* The group's next range of the lane in article order (or the one this node has), leased to the node; null when
|
||||
* the lane is caught up.
|
||||
*/
|
||||
async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const rules = await RuleSets.current()
|
||||
const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
|
||||
const mine = await db.from('ranges').where('group_id', groupId).where('status', 'leased').where('node_id', nodeId).orderBy('start_id').first()
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
const mine = (
|
||||
await db.rawQuery(
|
||||
`SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
const row =
|
||||
mine ??
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
|
||||
WHERE r.id = (
|
||||
SELECT id FROM ranges WHERE group_id = ? AND status = 'planned' AND next_id <= end_id AND priority BETWEEN ? AND ?
|
||||
ORDER BY start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, groupId, min, max]
|
||||
[nodeId, rules.version, groupId, ...todo.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
|
||||
if (!row) return null
|
||||
return { range: { id: Number(row.id), start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) } }
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async heartbeat(groupId: number, nodeId: number) {
|
||||
await this.held(groupId, nodeId)
|
||||
await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: new Date() })
|
||||
async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ heartbeat_at: new Date() })
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
}
|
||||
|
||||
/** A range of the group finished by its holder: counted on the lease (Leases.finish does the range). */
|
||||
async rangeDone(groupId: number, headers: number) {
|
||||
await db.rawQuery(`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ?`, [headers, groupId])
|
||||
/** The lane a range belongs to, by its priority. */
|
||||
laneOf(priority: number): Lane {
|
||||
return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
|
||||
}
|
||||
|
||||
/** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
|
||||
async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
|
||||
await db.rawQuery(
|
||||
`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
|
||||
[headers, groupId, lane]
|
||||
)
|
||||
}
|
||||
|
||||
/** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
|
||||
async release(groupId: number, nodeId: number) {
|
||||
await this.held(groupId, nodeId)
|
||||
await db.from('ranges').where('group_id', groupId).where('status', 'leased').where('node_id', nodeId).update({ status: 'planned', node_id: null })
|
||||
await db.from('group_leases').where('group_id', groupId).update({ node_id: null, range_id: null })
|
||||
async release(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const all = this.laneFilter(lane, false)
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ node_id: null, range_id: null })
|
||||
}
|
||||
|
||||
private snapshotPath(groupId: number) {
|
||||
return join(this.dir, `${groupId}.gz`)
|
||||
snapshotPath(groupId: number, lane: Lane = 'history') {
|
||||
return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
|
||||
}
|
||||
|
||||
/**
|
||||
* The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
|
||||
* gigabytes) and written whole before it replaces the old one.
|
||||
*/
|
||||
async putSnapshot(groupId: number, nodeId: number, next: number, body: Readable) {
|
||||
await this.held(groupId, nodeId)
|
||||
async putSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
next: number,
|
||||
body: Readable,
|
||||
lane: Lane = 'history'
|
||||
) {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await mkdir(this.dir, { recursive: true })
|
||||
const path = this.snapshotPath(groupId)
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
await pipeline(body, createWriteStream(`${path}.part`))
|
||||
await rename(`${path}.part`, path)
|
||||
await db.from('group_leases').where('group_id', groupId).update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
|
||||
}
|
||||
|
||||
/** The snapshot's stream and size, or null when there is none. */
|
||||
async getSnapshot(groupId: number, nodeId: number): Promise<{ stream: Readable; size: number; next: number } | null> {
|
||||
const lease = await this.held(groupId, nodeId)
|
||||
async getSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
lane: Lane = 'history'
|
||||
): Promise<{ stream: Readable; size: number; next: number } | null> {
|
||||
const lease = await this.held(groupId, nodeId, lane)
|
||||
if (lease.snapshot_next === null) return null
|
||||
const path = this.snapshotPath(groupId)
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
try {
|
||||
const { size } = await stat(path)
|
||||
return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* Twins by content (LOG.md (130)): a probed file's first 16 KiB and size (file_hashes, what the par2 merge and the
|
||||
* repeated verdict match) in a catalog release that has a name, readable or found inside. The same film posted
|
||||
* under scrambles by other tools gets that name without another fetch. Releases with a junk verdict don't count.
|
||||
*/
|
||||
/** Twins read per pair: the hottest pairs have 131k. */
|
||||
const TWIN_SCAN = 500
|
||||
|
||||
export async function twinNames(
|
||||
pairs: { hash16k: string; size: number }[],
|
||||
trx: { rawQuery: typeof db.rawQuery } = db
|
||||
): Promise<Map<string, string>> {
|
||||
const out = new Map<string, string>()
|
||||
if (pairs.length === 0) return out
|
||||
// Per pair the first named twin among its lowest TWIN_SCAN release ids. hash16k is typed as the column
|
||||
// (char(32)): as text the join cast the column and scanned all of file_hashes, 4 s a call; and with the index
|
||||
// the hottest pairs (55 with over 500 twins, up to 131k, all readable) made a call walk every twin, 37 s inside
|
||||
// the flush's transaction (LOG.md (138)). One name is all a pair needs.
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT v.hash16k, v.size, t.name
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(hash16k char(32), size bigint)
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT coalesce(r.inner_name, r.name) AS name
|
||||
FROM (SELECT release_id FROM file_hashes h WHERE h.hash16k = v.hash16k AND h.size = v.size ORDER BY release_id LIMIT ?) h
|
||||
JOIN releases r ON r.id = h.release_id AND (r.readable OR r.inner_name IS NOT NULL)
|
||||
WHERE NOT EXISTS (SELECT 1 FROM release_verdicts x WHERE x.release_id = r.id)
|
||||
ORDER BY r.readable DESC, r.id LIMIT 1) t`,
|
||||
[JSON.stringify(pairs), TWIN_SCAN]
|
||||
)
|
||||
for (const r of rows) out.set(`${r.hash16k}\t${r.size}`, r.name)
|
||||
return out
|
||||
}
|
||||
|
||||
export const twinKey = (hash16k: string, size: number) => `${hash16k}\t${size}`
|
||||
+323
-39
@@ -4,6 +4,7 @@ import coordinatorConfig from '#config/coordinator'
|
||||
import { explored } from '#lib/rules/index'
|
||||
import { usableName, type Inspection } from '#lib/probe/index'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { twinKey, twinNames } from '#services/hash_twins'
|
||||
|
||||
/** pending_releases.status (migration 6400). */
|
||||
export const PENDING = {
|
||||
@@ -22,6 +23,8 @@ export interface PendingTarget {
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
/** The target article's bytes (candidates from 2026-11-02 on). */
|
||||
bytes?: number
|
||||
}
|
||||
|
||||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||||
@@ -33,6 +36,8 @@ export interface PendingProbe {
|
||||
file: string
|
||||
tool: string
|
||||
groupId: number
|
||||
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
|
||||
klass: number
|
||||
poster: string
|
||||
/** Targets probed before this one. */
|
||||
tries: number
|
||||
@@ -53,12 +58,82 @@ export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||||
}
|
||||
|
||||
/** A class's record (pending_classes), for the claim's order. */
|
||||
interface ClassRow {
|
||||
tool: string
|
||||
groupId: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
|
||||
/** The record is read again this often. */
|
||||
const CLASSES_MS = 60_000
|
||||
/** A class found short of a claim's share is left alone this long. */
|
||||
const EMPTY_MS = 5 * 60_000
|
||||
/**
|
||||
* Rows in one order for a statement that locks many of them (pending_releases by key, pending_classes and
|
||||
* pending_signatures by theirs): the batch store, the record, the claim and the sweep all lock the same way, so two
|
||||
* of them can't deadlock on each other (LOG.md (134)).
|
||||
*/
|
||||
export function lockOrder<T>(rows: Iterable<T>, key: (row: T) => string): T[] {
|
||||
return [...rows].sort((a, b) => {
|
||||
const ka = key(a)
|
||||
const kb = key(b)
|
||||
return ka < kb ? -1 : ka > kb ? 1 : 0
|
||||
})
|
||||
}
|
||||
|
||||
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
|
||||
const PRIOR_RATE = 0.15
|
||||
const PRIOR_WEIGHT = 10
|
||||
/** One class takes at most this share of a claim: the rest explores the classes after it. */
|
||||
const CLASS_SHARE = 0.25
|
||||
|
||||
/** The expected naming rate of a class: its record shrunk towards the prior. */
|
||||
export function classScore(c: { probed: number; named: number }): number {
|
||||
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
|
||||
}
|
||||
|
||||
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
|
||||
function claimed(rows: any[]): any[] {
|
||||
const bytes = (r: any) =>
|
||||
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
|
||||
return [...rows].sort(
|
||||
(a, b) =>
|
||||
bytes(a) - bytes(b) ||
|
||||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
|
||||
Number(a.first_article) - Number(b.first_article)
|
||||
)
|
||||
}
|
||||
|
||||
function toProbe(r: any): PendingProbe {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass ?? 0),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
|
||||
* signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
|
||||
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
|
||||
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
|
||||
* classes with the best record first (classScore), the cheapest target first within one.
|
||||
*/
|
||||
class PendingLadder {
|
||||
/** A previous prober's claims back in the queue. */
|
||||
@@ -70,34 +145,84 @@ class PendingLadder {
|
||||
return rowCount ?? 0
|
||||
}
|
||||
|
||||
/** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
|
||||
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
|
||||
private classes: ClassRow[] = []
|
||||
private classesAt = 0
|
||||
private empty = new Map<string, number>()
|
||||
|
||||
/** The record read again on the next claim (the tests). */
|
||||
forget() {
|
||||
this.classesAt = 0
|
||||
this.empty.clear()
|
||||
}
|
||||
|
||||
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
|
||||
private async classRows(): Promise<ClassRow[]> {
|
||||
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
|
||||
)
|
||||
this.classes = rows
|
||||
.map((r: any) => ({
|
||||
tool: r.tool as string,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass),
|
||||
probed: Number(r.probed),
|
||||
named: Number(r.named),
|
||||
waiting: Number(r.waiting),
|
||||
}))
|
||||
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
|
||||
this.classesAt = Date.now()
|
||||
return this.classes
|
||||
}
|
||||
|
||||
/**
|
||||
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
|
||||
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
|
||||
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
|
||||
* classes, status 0 of a class without a row).
|
||||
*/
|
||||
async claim(n: number): Promise<PendingProbe[]> {
|
||||
if (n <= 0) return []
|
||||
const out: PendingProbe[] = []
|
||||
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
|
||||
const now = Date.now()
|
||||
for (const c of await this.classRows()) {
|
||||
if (out.length >= n) break
|
||||
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
|
||||
if ((this.empty.get(key) ?? 0) > now) continue
|
||||
const want = Math.min(perClass, n - out.length)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
|
||||
)
|
||||
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
|
||||
if (rows.length > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||||
[rows.length, c.tool, c.groupId, c.klass]
|
||||
)
|
||||
}
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
if (out.length < n) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
|
||||
[PENDING.taken, PENDING.waiting, n]
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, n - out.length]
|
||||
)
|
||||
return rows.map((r: any) => {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -106,6 +231,13 @@ class PendingLadder {
|
||||
*/
|
||||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||||
if (outcomes.length === 0) return
|
||||
// A twin by content names what the probe itself didn't (hash_twins.ts).
|
||||
const twins = await twinNames(
|
||||
outcomes
|
||||
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
|
||||
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||||
trx
|
||||
)
|
||||
const rows = outcomes.map((o) => {
|
||||
const p = o.probe
|
||||
if (o.status === 'waiting') {
|
||||
@@ -118,7 +250,12 @@ class PendingLadder {
|
||||
final: null,
|
||||
}
|
||||
}
|
||||
const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const twin =
|
||||
o.status === 'done' && o.inspection?.hash16k
|
||||
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
|
||||
: undefined
|
||||
const name = probed ?? twin ?? null
|
||||
const tries = p.tries + 1
|
||||
const inspection = {
|
||||
kind: p.kind,
|
||||
@@ -152,13 +289,26 @@ class PendingLadder {
|
||||
final: null,
|
||||
}
|
||||
})
|
||||
// The rows locked in key order first: the UPDATE's join below takes them in whatever order the plan does.
|
||||
await trx.rawQuery(
|
||||
`SELECT 1 FROM pending_releases WHERE release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||||
ORDER BY release_key FOR UPDATE`,
|
||||
[rows.map((r) => r.release_key).sort()]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||||
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
|
||||
[
|
||||
JSON.stringify(
|
||||
lockOrder(
|
||||
rows.map(({ final: _final, ...row }) => row),
|
||||
(r) => r.release_key
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
// The signatures: the poster's own and the tool's and group's together.
|
||||
const counts = new Map<
|
||||
@@ -188,42 +338,176 @@ class PendingLadder {
|
||||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
[JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`))]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
|
||||
swept_at = CASE WHEN s.dropped THEN s.swept_at END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
|
||||
lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`).map(
|
||||
({ tool, group_id, poster }) => ({ tool, group_id, poster })
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
|
||||
const classes = new Map<
|
||||
string,
|
||||
{
|
||||
tool: string
|
||||
group_id: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
|
||||
const entry = classes.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
klass: o.probe.klass,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
waiting: 0,
|
||||
}
|
||||
const final = rows[i].final
|
||||
if (final) entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
if (o.status === 'waiting') entry.waiting++
|
||||
classes.set(key, entry)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
|
||||
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
|
||||
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
|
||||
[JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
|
||||
swept_at = CASE WHEN c.dropped THEN c.swept_at END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`).map(
|
||||
({ tool, group_id, klass }) => ({ tool, group_id, klass })
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
|
||||
async discard(limit = 10_000): Promise<number> {
|
||||
/**
|
||||
* Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
|
||||
* many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
|
||||
* `perSweep` of them at most, each by its index (the claim's for a class, the signature index for a poster's
|
||||
* rows, the ladder's for a tool's and group's) in rounds of `limit`. The prober runs this beside the lanes, not
|
||||
* in the claim (LOG.md (133)); the edge drops new releases of dropped signatures and classes itself, so one
|
||||
* sweep is all a drop needs.
|
||||
*/
|
||||
async discard(limit = 10_000, perSweep = 20): Promise<number> {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
|
||||
WHERE p.status = ? AND EXISTS (
|
||||
SELECT 1 FROM pending_signatures s
|
||||
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
|
||||
LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
let discarded = 0
|
||||
const { rows: classes } = await db.rawQuery(
|
||||
`SELECT tool, group_id, klass FROM pending_classes WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, klass LIMIT ?`,
|
||||
[perSweep]
|
||||
)
|
||||
for (const c of classes) {
|
||||
let rows: any[]
|
||||
do {
|
||||
;({ rows } = await db.rawQuery(
|
||||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? LIMIT ?`,
|
||||
[PENDING.waiting, c.tool, c.group_id, c.klass, limit]
|
||||
))
|
||||
discarded += await this.discardKeys(rows, samplePercent)
|
||||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET swept_at = now() WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||||
[c.tool, c.group_id, c.klass]
|
||||
)
|
||||
}
|
||||
const { rows: signatures } = await db.rawQuery(
|
||||
`SELECT tool, group_id, poster FROM pending_signatures WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, poster LIMIT ?`,
|
||||
[perSweep]
|
||||
)
|
||||
for (const s of signatures) {
|
||||
let rows: any[]
|
||||
do {
|
||||
// The poster's rows by the signature index (on the poster's md5: a poster can be longer than a btree
|
||||
// key), a tool's and group's by the ladder's: never a test on the rows, which was a scan of the whole
|
||||
// table per signature on the HDD (LOG.md (134)).
|
||||
;({ rows } =
|
||||
s.poster === ''
|
||||
? await db.rawQuery(
|
||||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||||
WHERE status = ? AND group_id = ? AND tool = ? LIMIT ?`,
|
||||
[PENDING.waiting, s.group_id, s.tool, limit]
|
||||
)
|
||||
: await db.rawQuery(
|
||||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||||
WHERE status = ? AND group_id = ? AND tool = ? AND md5(poster) = md5(?::text) AND poster = ? LIMIT ?`,
|
||||
[PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
|
||||
))
|
||||
discarded += await this.discardKeys(rows, samplePercent)
|
||||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_signatures SET swept_at = now() WHERE tool = ? AND group_id = ? AND poster = ?`,
|
||||
[s.tool, s.group_id, s.poster]
|
||||
)
|
||||
}
|
||||
return discarded
|
||||
}
|
||||
|
||||
/** The rows discarded (status 2), the explored sample excepted, their classes' waiting counts lowered; how many. */
|
||||
private async discardKeys(rows: any[], samplePercent: number): Promise<number> {
|
||||
const keys = rows
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
.sort()
|
||||
if (keys.length === 0) return 0
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
|
||||
// Locked in key order (the record's lock order, LOG.md (137)): a plain `key = ANY(subquery)` locks in scan
|
||||
// order and deadlocked against a batch upsert; ORDER BY sorts before FOR UPDATE locks.
|
||||
const { rows: gone } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ?
|
||||
FROM (SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||||
ORDER BY release_key FOR UPDATE) k
|
||||
WHERE p.release_key = k.release_key
|
||||
RETURNING p.tool, p.group_id, p.klass`,
|
||||
[PENDING.discarded, PENDING.waiting, keys]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
|
||||
for (const g of gone) {
|
||||
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: g.tool,
|
||||
group_id: Number(g.group_id),
|
||||
klass: Number(g.klass),
|
||||
n: 0,
|
||||
}
|
||||
entry.n++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
if (counts.size > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
|
||||
)
|
||||
}
|
||||
return gone.length
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,13 @@ interface NamedRow {
|
||||
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
|
||||
const MAX_MOVES = 3
|
||||
|
||||
/**
|
||||
* The widest article span the namer reads again (a busy group posts 50M articles a day, a large upload among
|
||||
* them spans a million or two): one wider is a name the edge saw again much later ("v-lgpc.3ds"), and its span
|
||||
* of segments would not fit the indexer's memory (LOG.md (133)).
|
||||
*/
|
||||
const MAX_SPAN = 3_000_000
|
||||
|
||||
/**
|
||||
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
|
||||
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
|
||||
@@ -94,6 +101,10 @@ export class PendingNamer {
|
||||
|
||||
/** One named release into the catalog. */
|
||||
async move(row: NamedRow): Promise<void> {
|
||||
if (row.last_article - row.first_article > MAX_SPAN)
|
||||
throw new Error(
|
||||
`its articles ${row.first_article}–${row.last_article} span more than ${MAX_SPAN}`
|
||||
)
|
||||
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
|
||||
const range = await db
|
||||
.from('ranges')
|
||||
@@ -103,17 +114,20 @@ export class PendingNamer {
|
||||
.orderBy('id')
|
||||
.first()
|
||||
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
|
||||
const text = await this.overviewText(group.name, row)
|
||||
const rules = await RuleSets.current()
|
||||
const ix = new RangeIndexer({
|
||||
group: group.name,
|
||||
start: row.first_article,
|
||||
end: row.last_article,
|
||||
rules: { ...rules, samplePercent: 0 },
|
||||
idleChunks: 0,
|
||||
// No file given up before the end: the span is read chunk by chunk (one text of it topped V8's string).
|
||||
idleChunks: Number.MAX_SAFE_INTEGER,
|
||||
edgeArticles: 0,
|
||||
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
|
||||
partialFiles: true,
|
||||
})
|
||||
ix.addText(text, row.last_article + 1)
|
||||
for await (const piece of this.overviewPieces(group.name, row))
|
||||
ix.addText(piece.text, piece.next)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
|
||||
@@ -158,50 +172,20 @@ export class PendingNamer {
|
||||
})
|
||||
}
|
||||
|
||||
/** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
|
||||
private async overviewText(groupName: string, row: NamedRow): Promise<string> {
|
||||
if (this.corpusDir) {
|
||||
const stored = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.whereNotNull('stored_at')
|
||||
.where('start_id', '<=', row.last_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('start_id')
|
||||
const parts: string[] = []
|
||||
let covered = true
|
||||
for (const r of stored) {
|
||||
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
/**
|
||||
* The overview lines of the release's article span, a chunk of articles at a time with the first article after
|
||||
* it: the stored corpus chunks when its ranges are stored and contiguous, else XOVER in windows of a chunk.
|
||||
*/
|
||||
private async *overviewPieces(
|
||||
groupName: string,
|
||||
row: NamedRow
|
||||
): AsyncGenerator<{ text: string; next: number }> {
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
for (
|
||||
let from =
|
||||
range.start +
|
||||
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
|
||||
from <= Math.min(row.last_article, range.end);
|
||||
from += size
|
||||
) {
|
||||
const raw = await Corpus.read(
|
||||
range,
|
||||
{ from, to: Math.min(from + size - 1, range.end) },
|
||||
this.corpusDir
|
||||
)
|
||||
if (!raw) {
|
||||
covered = false
|
||||
break
|
||||
}
|
||||
parts.push((await inflate(raw)).toString('latin1'))
|
||||
}
|
||||
if (!covered) break
|
||||
}
|
||||
if (
|
||||
covered &&
|
||||
parts.length > 0 &&
|
||||
stored.length > 0 &&
|
||||
Number(stored[0].start_id) <= row.first_article &&
|
||||
Number(stored[stored.length - 1].end_id) >= row.last_article
|
||||
) {
|
||||
return parts.join('')
|
||||
}
|
||||
const stored = await this.storedChunks(groupName, row)
|
||||
if (stored) {
|
||||
for (const { raw, next } of stored)
|
||||
yield { text: (await inflate(raw)).toString('latin1'), next }
|
||||
return
|
||||
}
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
@@ -215,13 +199,55 @@ export class PendingNamer {
|
||||
this.usedAt = Date.now()
|
||||
try {
|
||||
await this.client.group(groupName)
|
||||
for (let from = row.first_article; from <= row.last_article; from += size) {
|
||||
const to = Math.min(from + size - 1, row.last_article)
|
||||
// One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
|
||||
const raw = await this.client.xoverRaw(row.first_article, row.last_article)
|
||||
return (await inflate(raw)).toString('latin1')
|
||||
const raw = await this.client.xoverRaw(from, to)
|
||||
yield { text: (await inflate(raw)).toString('latin1'), next: to + 1 }
|
||||
}
|
||||
} catch (error) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** The span's stored corpus chunks (compressed, with the first article after each), or null when any is missing. */
|
||||
private async storedChunks(
|
||||
groupName: string,
|
||||
row: NamedRow
|
||||
): Promise<{ raw: Buffer; next: number }[] | null> {
|
||||
if (!this.corpusDir) return null
|
||||
const stored = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.whereNotNull('stored_at')
|
||||
.where('start_id', '<=', row.last_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('start_id')
|
||||
if (
|
||||
stored.length === 0 ||
|
||||
Number(stored[0].start_id) > row.first_article ||
|
||||
Number(stored[stored.length - 1].end_id) < row.last_article
|
||||
)
|
||||
return null
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
const chunks: { raw: Buffer; next: number }[] = []
|
||||
for (const r of stored) {
|
||||
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
for (
|
||||
let from =
|
||||
range.start +
|
||||
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
|
||||
from <= Math.min(row.last_article, range.end);
|
||||
from += size
|
||||
) {
|
||||
const to = Math.min(from + size - 1, range.end)
|
||||
const raw = await Corpus.read(range, { from, to }, this.corpusDir)
|
||||
if (!raw) return null
|
||||
chunks.push({ raw, next: to + 1 })
|
||||
}
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
}
|
||||
+177
-31
@@ -35,20 +35,8 @@ class Planner {
|
||||
|
||||
async planGroup(group: { id: number; name: string }): Promise<number> {
|
||||
const info = await NntpPool.use((client) => client.group(group.name))
|
||||
const size = coordinatorConfig.rangeArticles
|
||||
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
|
||||
const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
|
||||
const span = Math.max(1, info.high - info.low)
|
||||
const rows = []
|
||||
for (let start = from; start <= info.high; start += size) {
|
||||
rows.push({
|
||||
group_id: group.id,
|
||||
start_id: start,
|
||||
end_id: Math.min(start + size - 1, info.high),
|
||||
next_id: start,
|
||||
priority: Math.floor((1000 * (start - info.low)) / span),
|
||||
})
|
||||
}
|
||||
const rows = historyRows(group.id, info, Number(last?.end ?? 0), coordinatorConfig.rangeArticles)
|
||||
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
|
||||
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
|
||||
return rows.length
|
||||
@@ -75,32 +63,99 @@ class Planner {
|
||||
* numbers come from LIST ACTIVE, one command per name prefix (GROUP per
|
||||
* group took minutes for 1000 groups).
|
||||
*/
|
||||
async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
|
||||
async planTail(now = new Date()): Promise<TailPlanned> {
|
||||
const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
|
||||
if (groups.length === 0) return { groups: 0, ranges: 0 }
|
||||
const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
|
||||
const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
|
||||
if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
const { rows: ends } = await db.rawQuery(
|
||||
`SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
|
||||
(status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
|
||||
FROM ranges ORDER BY group_id, start_id DESC`,
|
||||
[coordinatorConfig.tail.priority]
|
||||
)
|
||||
const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
|
||||
ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
|
||||
)
|
||||
const highs = new Map<string, number>()
|
||||
await NntpPool.use(async (client) => {
|
||||
for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
|
||||
for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
|
||||
}
|
||||
})
|
||||
const rows = tailRanges(
|
||||
groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
|
||||
const plan = tailRanges(
|
||||
groups.map((g) => {
|
||||
const l = last.get(Number(g.id))
|
||||
return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
|
||||
}),
|
||||
now,
|
||||
{ rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
|
||||
)
|
||||
if (rows.length === 0) return { groups: 0, ranges: 0 }
|
||||
return this.applyTail(plan, now)
|
||||
}
|
||||
|
||||
/** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
|
||||
async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
|
||||
const rows = [...plan.rows]
|
||||
let extended = 0
|
||||
if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('ranges').multiInsert(rows)
|
||||
for (const id of new Set(rows.map((r) => r.group_id))) {
|
||||
const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
|
||||
await trx.from('groups').where('id', id).update({ high, planned_at: now })
|
||||
for (const e of plan.extended) {
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
|
||||
[e.end_id, e.id, e.from - 1]
|
||||
)
|
||||
if (done.length > 0) extended++
|
||||
else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
|
||||
}
|
||||
if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
|
||||
const highs = new Map<number, number>()
|
||||
for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
|
||||
for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
|
||||
})
|
||||
return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
|
||||
return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
|
||||
* RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
|
||||
* (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
|
||||
*/
|
||||
async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT id, group_id, start_id, end_id, priority FROM ranges
|
||||
WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
|
||||
ORDER BY group_id, start_id`
|
||||
)
|
||||
const merges = tailMerges(
|
||||
candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
|
||||
coordinatorConfig.rangeArticles
|
||||
)
|
||||
const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
|
||||
if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
await db.transaction(async (trx) => {
|
||||
// The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
const { rows: gone } = await trx.rawQuery(
|
||||
`DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
|
||||
[merges.flatMap((m) => m.drop)]
|
||||
)
|
||||
if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
|
||||
const { rows: grown } = await trx.rawQuery(
|
||||
`UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
|
||||
WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
|
||||
[merges.map((m) => m.id), merges.map((m) => m.end_id)]
|
||||
)
|
||||
if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
|
||||
})
|
||||
return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
}
|
||||
}
|
||||
|
||||
export interface TailPlanned {
|
||||
groups: number
|
||||
/** Rows added. */
|
||||
ranges: number
|
||||
/** Open ranges grown. */
|
||||
extended: number
|
||||
}
|
||||
|
||||
/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
|
||||
@@ -117,28 +172,119 @@ export interface TailGroup {
|
||||
plannedAt: Date
|
||||
/** The server's newest article (null: not listed). */
|
||||
high: number | null
|
||||
/**
|
||||
* Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
|
||||
* nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
|
||||
*/
|
||||
open?: { id: number; start: number } | null
|
||||
}
|
||||
|
||||
export interface TailRow {
|
||||
group_id: number
|
||||
start_id: number
|
||||
end_id: number
|
||||
next_id: number
|
||||
priority: number
|
||||
}
|
||||
|
||||
export interface TailPlan {
|
||||
/** New ranges. */
|
||||
rows: TailRow[]
|
||||
/** Open ranges grown to end_id (from: the first new article, the old end + 1). */
|
||||
extended: { id: number; group_id: number; from: number; end_id: number }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The tail's new ranges: per group, the articles after its last range, in
|
||||
* ranges of at most `rangeArticles`, once there are `minArticles` of them or
|
||||
* the group's last planning is `maxDelayMinutes` ago.
|
||||
* The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
|
||||
* the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
|
||||
* tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
|
||||
*/
|
||||
export function tailRanges(
|
||||
groups: TailGroup[],
|
||||
now: Date,
|
||||
options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
|
||||
) {
|
||||
const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
|
||||
): TailPlan {
|
||||
const rows: TailRow[] = []
|
||||
const extended: TailPlan['extended'] = []
|
||||
for (const g of groups) {
|
||||
if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
|
||||
const waiting = g.high - g.plannedTo
|
||||
const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
|
||||
if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
|
||||
for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
|
||||
let start = g.plannedTo + 1
|
||||
if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
|
||||
const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
|
||||
extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
|
||||
start = end + 1
|
||||
}
|
||||
for (; start <= g.high; start += options.rangeArticles) {
|
||||
rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
|
||||
}
|
||||
}
|
||||
return { rows, extended }
|
||||
}
|
||||
|
||||
export interface TailMerge {
|
||||
/** The run's first range, grown to end_id. */
|
||||
id: number
|
||||
end_id: number
|
||||
/** The ranges it absorbed. */
|
||||
drop: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
|
||||
* `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
|
||||
*/
|
||||
export function tailMerges(
|
||||
ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
|
||||
rangeArticles: number
|
||||
): TailMerge[] {
|
||||
const merges: TailMerge[] = []
|
||||
let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
|
||||
const flush = () => {
|
||||
if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
|
||||
}
|
||||
for (const r of ranges) {
|
||||
if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
|
||||
cur.end_id = r.end_id
|
||||
cur.drop.push(r.id)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
|
||||
}
|
||||
flush()
|
||||
return merges
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows a group's history is cut into, from `lastEnd + 1` (0: the group has no ranges yet) up to the high
|
||||
* watermark: RANGE_ARTICLES slices at the position priority. A group planned for the first time also gets one
|
||||
* "ancient" range, articles 1 … low − 1: what had expired on the server before it was planned. Status
|
||||
* 'ancient', never leased or fetched; it keeps the tiling complete from article 1 (migration 6700 added it for
|
||||
* the groups planned before 2026-10-28).
|
||||
*/
|
||||
export function historyRows(
|
||||
groupId: number,
|
||||
info: { low: number; high: number },
|
||||
lastEnd: number,
|
||||
size: number
|
||||
): Record<string, unknown>[] {
|
||||
const rows: Record<string, unknown>[] = []
|
||||
if (lastEnd === 0 && info.low > 1)
|
||||
rows.push({ group_id: groupId, start_id: 1, end_id: info.low - 1, next_id: 1, priority: 0, status: 'ancient' })
|
||||
const from = Math.max(info.low, lastEnd + 1)
|
||||
const span = Math.max(1, info.high - info.low)
|
||||
for (let start = from; start <= info.high; start += size) {
|
||||
rows.push({
|
||||
group_id: groupId,
|
||||
start_id: start,
|
||||
end_id: Math.min(start + size - 1, info.high),
|
||||
next_id: start,
|
||||
priority: Math.floor((1000 * (start - info.low)) / span),
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
+40
-12
@@ -425,14 +425,19 @@ export class Prefetcher {
|
||||
)
|
||||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||||
// The groups workers hold (group_leases; REBUILD.md design 1): the prefetcher runs ahead of their cursors.
|
||||
const held: number[] = (
|
||||
// The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
|
||||
// cursors (a group's history and tail lanes are two cursors).
|
||||
const held: { group: number; lane: string }[] = (
|
||||
await db
|
||||
.from('group_leases')
|
||||
.whereNotNull('node_id')
|
||||
.where('heartbeat_at', '>=', new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000))
|
||||
.select('group_id')
|
||||
).map((l) => Number(l.group_id))
|
||||
.where(
|
||||
'heartbeat_at',
|
||||
'>=',
|
||||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||||
)
|
||||
.select('group_id', 'lane')
|
||||
).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
const ahead = Math.max(
|
||||
coordinatorConfig.spool.minAhead,
|
||||
@@ -456,17 +461,37 @@ export class Prefetcher {
|
||||
const want = Math.max(ahead - Number(ready), short)
|
||||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||||
const full = await CatalogSpace.full()
|
||||
// Ahead of each held group's cursor: its next ranges in article order, a range of every held group before any
|
||||
// group's second; without held groups (no window worker running) nothing is fetched ahead for workers.
|
||||
// Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
|
||||
// leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
|
||||
// at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
|
||||
// window worker running) nothing is fetched ahead for workers.
|
||||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||||
const tailPriority = coordinatorConfig.tail.priority
|
||||
const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
|
||||
const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
|
||||
const fresh =
|
||||
room && want > 0 && held.length > 0
|
||||
room && want > 0 && heldHistory.length + heldTail.length > 0
|
||||
? await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', 0)
|
||||
.whereRaw('r.next_id <= r.end_id')
|
||||
.whereIn('r.group_id', held)
|
||||
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
|
||||
.orderByRaw('row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((h: any) =>
|
||||
h
|
||||
.whereIn('r.group_id', heldHistory)
|
||||
.where('r.priority', '>', tailPriority)
|
||||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||||
.where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
|
||||
)
|
||||
.orWhere((t: any) =>
|
||||
t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
|
||||
)
|
||||
)
|
||||
.orderByRaw(
|
||||
'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
|
||||
[tailPriority]
|
||||
)
|
||||
.limit(want)
|
||||
: []
|
||||
const targets: Target[] = []
|
||||
@@ -478,7 +503,8 @@ export class Prefetcher {
|
||||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span))) continue
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||||
continue
|
||||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||||
missing.push(span)
|
||||
}
|
||||
@@ -535,6 +561,8 @@ export class Prefetcher {
|
||||
const groups = new Map<string, number>()
|
||||
const candidates = await rows(db.query())
|
||||
.where('r.capture', true)
|
||||
// A group taken off the list (groups.active) isn't worth the connections.
|
||||
.where('g.active', true)
|
||||
.whereNull('r.stored_at')
|
||||
.where((q: any) =>
|
||||
q
|
||||
|
||||
+71
-11
@@ -9,6 +9,7 @@ import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/i
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
import { twinKey, twinNames } from '#services/hash_twins'
|
||||
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
|
||||
import { PendingNamer } from '#services/pending_namer'
|
||||
|
||||
@@ -40,6 +41,8 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
const FLUSH_MS = 2000
|
||||
/** Between sweeps that discard the waiting releases of dropped signatures. */
|
||||
const SWEEP_MS = 5 * 60_000
|
||||
/** A flush that deadlocks (against the batch store's pending upsert) keeps its outcomes for the next, this often. */
|
||||
const DEADLOCK_RETRIES = 5
|
||||
|
||||
/**
|
||||
* JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
|
||||
@@ -49,6 +52,20 @@ function jsonb(value: unknown): string {
|
||||
return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
|
||||
}
|
||||
|
||||
/** Rounds of every lane's pipeline still queued when the next claim starts. */
|
||||
const LOW_ROUNDS = 8
|
||||
|
||||
/**
|
||||
* The candidate queue's levels: the next claim starts below `low`, LOW_ROUNDS rounds of every lane's pipeline (at
|
||||
* ~0.2 s an article and a pipeline of 8, ~13 s of work), and fills it to `target`, twice that. A claim takes 1.5–4 s
|
||||
* and more (random slices of an 11 M-row table); refilling at half of four rounds left 3–6 s, and at 20–24 lanes the
|
||||
* lanes waited for candidates up to 19% of their time (LOG.md 2026-10-28 (120)). At least `minBatch` either way.
|
||||
*/
|
||||
export function queueLevels(lanes: number, pipeline: number, minBatch: number): { low: number; target: number } {
|
||||
const low = Math.max(minBatch, lanes * pipeline * LOW_ROUNDS)
|
||||
return { low, target: 2 * low }
|
||||
}
|
||||
|
||||
/** Slices a claim takes from random points of the queue. */
|
||||
const SLICES = 20
|
||||
const REPORT_MS = 60_000
|
||||
@@ -67,6 +84,8 @@ export class Prober {
|
||||
private refilling: Promise<void> | null = null
|
||||
private stopped = false
|
||||
private stats = Prober.freshStats()
|
||||
/** Flushes deadlocked in a row, their outcomes kept. */
|
||||
private deadlocks = 0
|
||||
/** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
|
||||
private lanes = new Set<Promise<void>>()
|
||||
private running = 0
|
||||
@@ -82,7 +101,6 @@ export class Prober {
|
||||
}
|
||||
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
|
||||
private namer = new PendingNamer()
|
||||
private sweptAt = 0
|
||||
|
||||
async run() {
|
||||
// A previous run's claims: one prober at a time.
|
||||
@@ -91,6 +109,17 @@ export class Prober {
|
||||
const released = await PendingLadder.release()
|
||||
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
|
||||
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
|
||||
// The ladder's discard sweep beside the lanes: a dropped signature's rows take minutes on the HDD (LOG.md (133)).
|
||||
const sweeper = (async () => {
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
} catch (error: any) {
|
||||
logger.error(`Probing: the discard sweep failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < SWEEP_MS && !this.stopped; waited += 1000) await sleep(1000)
|
||||
}
|
||||
})()
|
||||
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
|
||||
const flusher = (async () => {
|
||||
while (!this.stopped || this.outcomes.length > 0) {
|
||||
@@ -121,6 +150,7 @@ export class Prober {
|
||||
}
|
||||
await Promise.all(this.lanes)
|
||||
await namer
|
||||
await sweeper
|
||||
await this.share.leave().catch(() => {})
|
||||
// What was claimed and not probed goes back.
|
||||
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
@@ -209,13 +239,18 @@ export class Prober {
|
||||
}
|
||||
}
|
||||
|
||||
/** Candidates claimed at a time: a few rounds of every lane's pipeline (a claim takes seconds; 400 lasted ~2 s at 37 lanes). */
|
||||
/** The queue's low-water mark and the level a claim fills it to (queueLevels). */
|
||||
private get levels() {
|
||||
return queueLevels(this.running, coordinatorConfig.probe.pipeline, coordinatorConfig.probe.batch)
|
||||
}
|
||||
|
||||
/** Claims to the queue's target; the claim after starts when it's below its low-water mark (queueLevels). */
|
||||
private get batch() {
|
||||
return Math.max(coordinatorConfig.probe.batch, this.running * coordinatorConfig.probe.pipeline * 4)
|
||||
return this.levels.target
|
||||
}
|
||||
|
||||
private async next(): Promise<Candidate | undefined> {
|
||||
if (this.queue.length < this.batch / 2) {
|
||||
if (this.queue.length < this.levels.low) {
|
||||
this.refilling ??= this.claim().finally(() => (this.refilling = null))
|
||||
if (this.queue.length === 0) await this.refilling
|
||||
}
|
||||
@@ -246,10 +281,6 @@ export class Prober {
|
||||
|
||||
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
|
||||
private async claimPending() {
|
||||
if (Date.now() - this.sweptAt > SWEEP_MS) {
|
||||
this.sweptAt = Date.now()
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
}
|
||||
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
|
||||
const probes = await PendingLadder.claim(want)
|
||||
if (probes.length === 0) return
|
||||
@@ -319,15 +350,34 @@ export class Prober {
|
||||
}
|
||||
}
|
||||
|
||||
/** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
|
||||
/**
|
||||
* The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. A deadlock
|
||||
* (the batch store's pending upsert locks the same rows; LOG.md (134)) keeps the outcomes for the next flush
|
||||
* instead of losing the probes.
|
||||
*/
|
||||
private async flush() {
|
||||
const taken = this.outcomes.splice(0)
|
||||
if (taken.length === 0) return
|
||||
try {
|
||||
await this.store(taken)
|
||||
} catch (error: any) {
|
||||
if (error?.code === '40P01' && this.deadlocks < DEADLOCK_RETRIES) {
|
||||
this.deadlocks++
|
||||
this.outcomes.unshift(...taken)
|
||||
logger.warn(`Probing: storing results deadlocked, kept for the next flush (${this.deadlocks}).`)
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
this.deadlocks = 0
|
||||
for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
this.stats.pending += taken.filter((o) => o.candidate.pending && o.status !== 'waiting').length
|
||||
}
|
||||
|
||||
private async store(taken: Outcome[]) {
|
||||
// The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
|
||||
const pending = taken.filter((o) => o.candidate.pending)
|
||||
const outcomes = taken.filter((o) => !o.candidate.pending)
|
||||
this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
|
||||
await db.transaction(async (trx) => {
|
||||
if (outcomes.length > 0) {
|
||||
await trx.rawQuery(
|
||||
@@ -373,10 +423,20 @@ export class Prober {
|
||||
entry.all.push({ kind: r.kind, result: r.result })
|
||||
byRelease.set(id, entry)
|
||||
}
|
||||
// A twin by content names a release its own inspections didn't (hash_twins.ts; LOG.md (130)).
|
||||
const twins = await twinNames(
|
||||
done.filter((o) => o.inspection!.hash16k && o.inspection!.size).map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||||
trx
|
||||
)
|
||||
const twinOf = new Map<number, string>()
|
||||
for (const o of done) {
|
||||
const twin = o.inspection!.hash16k ? twins.get(twinKey(o.inspection!.hash16k, o.inspection!.size)) : undefined
|
||||
if (twin && twin !== byRelease.get(o.candidate.releaseId)?.name) twinOf.set(o.candidate.releaseId, twin)
|
||||
}
|
||||
const updates = [...byRelease].map(([id, { name, all }]) => ({
|
||||
id,
|
||||
passworded: all.some((i) => i.result.passworded === true),
|
||||
inner_name: innerName(all, name),
|
||||
inner_name: innerName(all, name) ?? twinOf.get(id) ?? null,
|
||||
}))
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import { TINY_MIN_BYTES, NONSENSE_MIN_BYTES } from '#lib/rules/index'
|
||||
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
|
||||
export const JUDGE_VERSION = 1
|
||||
@@ -28,6 +29,14 @@ const MEDIA =
|
||||
const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
|
||||
/** An executable's name. */
|
||||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||||
/**
|
||||
* The tiny rule's name tests, lib/rules/rule_set.ts's RELEASE_LIKE_SPACED, EBOOK, TINY_CONTENT and
|
||||
* STUB_EXEMPT_GROUPS in Postgres's dialect (and no "?", see MEDIA).
|
||||
*/
|
||||
const TINY_RELEASE_LIKE = `(\\y(19|20)[0-9][0-9]\\y|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-{0,1}dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\\.[0-9]+)+)\\y|-[a-z0-9]{2,}$|\\.[a-z0-9]+\\.[a-z0-9]+\\.)`
|
||||
const TINY_EBOOK = `\\y(epub|mobi|azw3{0,1}|pdf|djvu|cbr|cbz|e-{0,1}books{0,1})\\y`
|
||||
const TINY_CONTENT = `\\y(subs{0,1}|subtitles{0,1}|subpack|srt|untertitel|ondertitels{0,1}|ondertiteling|legendas{0,1}|napisy|sottotitoli|cracks{0,1}|cracked|keygen|keymaker|trainer|patch|patched|serials{0,1}|activator|loader|nocd|no-cd|fix|covers{0,1}|artwork|booklet|scans{0,1}|proof|sample|info|readme)\\y`
|
||||
const TINY_EXEMPT_GROUPS = `(sound|mp3|karaoke|music|audio|flac|lossless|e-{0,1}book|epub|comic|guitar|font|magazine)`
|
||||
|
||||
export interface Rule {
|
||||
name: string
|
||||
@@ -100,6 +109,20 @@ export const RULES: Rule[] = [
|
||||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
name: 'tiny',
|
||||
says: 'one file of one article, 2–100 kB, under a release-like name: the exe-stub flood in miniature',
|
||||
// The catalog's side of the edge's tiny-archive rule (lib/rules/rule_set.ts isTinyArchive; LOG.md (131),
|
||||
// (135)): the same name tests in Postgres's regex dialect ("\y" for "\b", "{0,1}" for "?"), the readable flag
|
||||
// for its scrambled and random tests, the group set for its groups. The 2015–17 flood that the edge indexed
|
||||
// before the rule: 1.5M rows, 98% of them with an archive candidate, the rest lone par2 files and scrambled
|
||||
// names, so no join on the candidates. A scan of releases (9 GB) a full round.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||||
WHERE r.files = 1 AND r.segments = 1 AND r.bytes >= ${TINY_MIN_BYTES} AND r.bytes < ${NONSENSE_MIN_BYTES}
|
||||
AND r.readable AND NOT coalesce(r.sidecar_only, false)
|
||||
AND r.name ~* '${TINY_RELEASE_LIKE}' AND r.name !~* '${TINY_EBOOK}' AND r.name !~* '${TINY_CONTENT}'
|
||||
AND NOT EXISTS (SELECT 1 FROM group_sets gs WHERE gs.id = r.group_set_id AND gs.groups::text ~* '${TINY_EXEMPT_GROUPS}')`,
|
||||
},
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||
|
||||
@@ -85,6 +85,19 @@ function contains(word: string): string {
|
||||
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* A search word's condition on release_texts: anywhere in a text (ILIKE), or with a leading "^" at a word's start, the
|
||||
* text's or after a character that's no letter or digit ("^zed" finds "Zed.Dead", "The_Zed", not "Facialized"). Both
|
||||
* use the trigram index.
|
||||
*/
|
||||
function wordMatch(word: string): { sql: string; binding: string } {
|
||||
if (word.startsWith('^') && word.length > 1) {
|
||||
const rest = word.slice(1).replace(/[\\^$.|?*+()[\]{}]/g, (c) => `\\${c}`)
|
||||
return { sql: 't.text ~* ?', binding: `(^|[^[:alnum:]])${rest}` }
|
||||
}
|
||||
return { sql: 't.text ILIKE ?', binding: contains(word) }
|
||||
}
|
||||
|
||||
/** Searching and browsing the catalog, for the web interface and Newznab. */
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
@@ -93,10 +106,12 @@ class SearchService {
|
||||
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
||||
const bindings: unknown[] = []
|
||||
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
|
||||
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike.
|
||||
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike;
|
||||
// "^zed" only at a word's start.
|
||||
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
||||
where.push('r.id IN (SELECT t.release_id FROM release_texts t WHERE t.text ILIKE ?)')
|
||||
bindings.push(contains(word))
|
||||
const match = wordMatch(word)
|
||||
where.push(`r.id IN (SELECT t.release_id FROM release_texts t WHERE ${match.sql})`)
|
||||
bindings.push(match.binding)
|
||||
}
|
||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||
if (params.complete) where.push('r.complete')
|
||||
@@ -304,9 +319,9 @@ class SearchService {
|
||||
// History by articles; the pass: the history ranges before those deferred behind them (priority
|
||||
// 2000 apart; e.g. groups added later that go first).
|
||||
const { rows: ranges } = await db.rawQuery(
|
||||
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
|
||||
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
|
||||
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
|
||||
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
|
||||
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
|
||||
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
|
||||
export const SETTLE_COMPLETE_HOURS = 6
|
||||
|
||||
/**
|
||||
* Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
|
||||
* lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
|
||||
* never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
|
||||
* hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
|
||||
* announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
|
||||
* settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
|
||||
* candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
|
||||
*/
|
||||
class Settler {
|
||||
async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`UPDATE releases SET settled_at = now()
|
||||
WHERE settled_at IS NULL AND complete
|
||||
AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
|
||||
RETURNING id`,
|
||||
[quietHours, quietHours]
|
||||
)
|
||||
if (rows.length > 0)
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[rows.map((r: { id: string }) => Number(r.id))]
|
||||
)
|
||||
return rows.length
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new Settler()
|
||||
@@ -0,0 +1,343 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { readdir, stat, statfs } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import type { Lane } from '#lib/edge/index'
|
||||
|
||||
export interface RangeBand {
|
||||
band: string
|
||||
planned: number
|
||||
plannedArticles: number
|
||||
leased: number
|
||||
done: number
|
||||
doneArticles: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
export interface HeldGroup {
|
||||
node: string
|
||||
group: string
|
||||
lane: Lane
|
||||
range: { start: number; end: number; next: number } | null
|
||||
heartbeatSeconds: number
|
||||
snapshotNext: number | null
|
||||
snapshotSeconds: number | null
|
||||
snapshotMb: number | null
|
||||
rangesDone: number
|
||||
headers: number
|
||||
}
|
||||
|
||||
export interface SnapshotFile {
|
||||
group: string
|
||||
lane: Lane
|
||||
mb: number
|
||||
hours: number
|
||||
}
|
||||
|
||||
export interface Status {
|
||||
at: string
|
||||
catalog: {
|
||||
tablespace: string
|
||||
gb: number
|
||||
capGb: number
|
||||
full: boolean
|
||||
freeGb: number
|
||||
}
|
||||
bands: RangeBand[]
|
||||
lastHour: {
|
||||
indexedArticles: number
|
||||
doneRanges: number
|
||||
storedArticles: number
|
||||
storedRanges: number
|
||||
}
|
||||
corpus: { pendingRanges: number; pendingArticles: number }
|
||||
spool: { ranges: number; chunks: number }
|
||||
tail: { groups: number; ranges: number; articles: number }
|
||||
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
|
||||
ancient: { groups: number; articles: number }
|
||||
held: HeldGroup[]
|
||||
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
|
||||
releases: { estimated: number; provisional: number }
|
||||
/** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
|
||||
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
|
||||
judge: { dirty: number }
|
||||
nodes: { name: string; lastSeenSeconds: number | null }[]
|
||||
}
|
||||
|
||||
const seconds = (since: Date | string | null) =>
|
||||
since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
|
||||
|
||||
/**
|
||||
* What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
|
||||
* NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
|
||||
* with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
|
||||
* filesystem; nothing is kept between calls.
|
||||
*/
|
||||
class StatusService {
|
||||
async collect(): Promise<Status> {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
const cap = coordinatorConfig.catalogMaxNvmeGb
|
||||
// The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
|
||||
// chunk partitions and the probe cache on the HDD, minutes on homelab.
|
||||
const { rows: space } = await db.rawQuery(
|
||||
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
|
||||
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
|
||||
)
|
||||
const tablespaceGb = Number(space[0].bytes) / 1e9
|
||||
const fs = await statfs('/').catch(() => null)
|
||||
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
|
||||
|
||||
// The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
|
||||
// capture too, at their history priorities), the history by its thousand.
|
||||
const { rows: bands } = await db.rawQuery(
|
||||
`SELECT CASE WHEN priority <= ? THEN 'tail'
|
||||
WHEN priority < 0 THEN 'raised'
|
||||
WHEN capture AND priority <= 2 THEN 'window ' || priority
|
||||
WHEN priority < 1000 THEN 'history 0-999'
|
||||
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
|
||||
count(*) FILTER (WHERE status = 'planned') AS planned,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
|
||||
count(*) FILTER (WHERE status = 'leased') AS leased,
|
||||
count(*) FILTER (WHERE status = 'done') AS done,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM ranges GROUP BY 1
|
||||
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
|
||||
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
|
||||
[tail, tail]
|
||||
)
|
||||
const { rows: hour } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
|
||||
count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
|
||||
count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
|
||||
count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
|
||||
count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
|
||||
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
|
||||
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
|
||||
FROM ranges`,
|
||||
[tail, tail, tail]
|
||||
)
|
||||
const h = hour[0]
|
||||
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const { rows: held } = await db.rawQuery(
|
||||
`SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
|
||||
r.start_id, r.end_id, r.next_id, l.group_id
|
||||
FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
|
||||
LEFT JOIN ranges r ON r.id = l.range_id
|
||||
WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
|
||||
ORDER BY l.lane, g.name`,
|
||||
[ttl]
|
||||
)
|
||||
const snapshotSizes = new Map<string, number>()
|
||||
const files: SnapshotFile[] = []
|
||||
const names = new Map<number, string>(
|
||||
(await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
|
||||
)
|
||||
for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
|
||||
const m = /^(\d+)(-tail)?\.gz$/.exec(file)
|
||||
if (!m) continue
|
||||
const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
|
||||
if (!s) continue
|
||||
const lane: Lane = m[2] ? 'tail' : 'history'
|
||||
snapshotSizes.set(`${m[1]}:${lane}`, s.size)
|
||||
files.push({
|
||||
group: names.get(Number(m[1])) ?? m[1],
|
||||
lane,
|
||||
mb: s.size / 1e6,
|
||||
hours: (Date.now() - s.mtimeMs) / 3600_000,
|
||||
})
|
||||
}
|
||||
files.sort((a, b) => b.mb - a.mb)
|
||||
|
||||
// The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
|
||||
// homelab); the rest are small partial indexes.
|
||||
const { rows: releases } = await db.rawQuery(
|
||||
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
|
||||
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
|
||||
(SELECT count(*) FROM judge_dirty) AS dirty`
|
||||
)
|
||||
const r = releases[0]
|
||||
const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
|
||||
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
catalog: {
|
||||
tablespace: space[0].tablespace,
|
||||
gb: tablespaceGb,
|
||||
capGb: cap,
|
||||
full: cap > 0 && tablespaceGb > cap,
|
||||
freeGb,
|
||||
},
|
||||
bands: bands.map((b: any) => ({
|
||||
band: b.band,
|
||||
planned: Number(b.planned),
|
||||
plannedArticles: Number(b.planned_articles),
|
||||
leased: Number(b.leased),
|
||||
done: Number(b.done),
|
||||
doneArticles: Number(b.done_articles),
|
||||
failed: Number(b.failed),
|
||||
})),
|
||||
lastHour: {
|
||||
indexedArticles: Number(h.indexed),
|
||||
doneRanges: Number(h.done_ranges),
|
||||
storedArticles: Number(h.stored),
|
||||
storedRanges: Number(h.stored_ranges),
|
||||
},
|
||||
corpus: {
|
||||
pendingRanges: Number(h.corpus_pending),
|
||||
pendingArticles: Number(h.corpus_pending_articles),
|
||||
},
|
||||
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
|
||||
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
|
||||
tail: {
|
||||
groups: Number(h.tail_groups),
|
||||
ranges: Number(h.tail_ranges),
|
||||
articles: Number(h.tail_articles),
|
||||
},
|
||||
held: held.map((l: any) => {
|
||||
const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
|
||||
return {
|
||||
node: l.node,
|
||||
group: l.group,
|
||||
lane: l.lane,
|
||||
range:
|
||||
l.start_id !== null
|
||||
? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
|
||||
: null,
|
||||
heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
|
||||
snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
|
||||
snapshotSeconds: seconds(l.snapshot_at),
|
||||
snapshotMb: size === undefined ? null : size / 1e6,
|
||||
rangesDone: Number(l.ranges_done),
|
||||
headers: Number(l.headers),
|
||||
}
|
||||
}),
|
||||
snapshots: {
|
||||
count: files.length,
|
||||
gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
|
||||
oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
|
||||
largest: files.slice(0, 5),
|
||||
},
|
||||
releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
|
||||
probes: {
|
||||
ready: Number(r.ready),
|
||||
waiting: Number(r.waiting),
|
||||
pendingWaiting: Number(r.pending_waiting),
|
||||
pendingNamed: Number(r.pending_named),
|
||||
},
|
||||
judge: { dirty: Number(r.dirty) },
|
||||
nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
|
||||
}
|
||||
}
|
||||
|
||||
/** The status as a text panel. */
|
||||
render(s: Status): string {
|
||||
const n = (v: number) => Math.round(v).toLocaleString('en-US')
|
||||
const b = (v: number) =>
|
||||
v >= 1e9
|
||||
? `${(v / 1e9).toFixed(2)} B`
|
||||
: v >= 1e6
|
||||
? `${(v / 1e6).toFixed(1)} M`
|
||||
: v >= 1e3
|
||||
? `${(v / 1e3).toFixed(0)} k`
|
||||
: String(v)
|
||||
const age = (sec: number | null) =>
|
||||
sec === null
|
||||
? '-'
|
||||
: sec < 90
|
||||
? `${sec} s`
|
||||
: sec < 5400
|
||||
? `${Math.round(sec / 60)} min`
|
||||
: `${(sec / 3600).toFixed(1)} h`
|
||||
const lines: string[] = []
|
||||
lines.push(`Status at ${s.at}`)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
|
||||
)
|
||||
lines.push(
|
||||
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
|
||||
)
|
||||
lines.push(
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
|
||||
)
|
||||
lines.push('')
|
||||
const table = (header: string[], rows: string[][]) => {
|
||||
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
|
||||
const line = (r: string[]) =>
|
||||
r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(' ')
|
||||
lines.push(line(header))
|
||||
for (const r of rows) lines.push(line(r))
|
||||
}
|
||||
lines.push('Ranges by band:')
|
||||
table(
|
||||
['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
|
||||
s.bands.map((x) => [
|
||||
x.band,
|
||||
n(x.planned),
|
||||
b(x.plannedArticles),
|
||||
n(x.leased),
|
||||
n(x.done),
|
||||
b(x.doneArticles),
|
||||
n(x.failed),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(`Held groups (${s.held.length}):`)
|
||||
if (s.held.length > 0)
|
||||
table(
|
||||
[
|
||||
'group',
|
||||
'lane',
|
||||
'node',
|
||||
'range',
|
||||
'at',
|
||||
'heartbeat',
|
||||
'snapshot',
|
||||
'size',
|
||||
'done',
|
||||
'headers',
|
||||
],
|
||||
s.held.map((x) => [
|
||||
x.group,
|
||||
x.lane,
|
||||
x.node,
|
||||
x.range ? `${x.range.start}-${x.range.end}` : '-',
|
||||
x.range ? String(x.range.next) : '-',
|
||||
age(x.heartbeatSeconds),
|
||||
x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
|
||||
x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
|
||||
n(x.rangesDone),
|
||||
b(x.headers),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
|
||||
)
|
||||
lines.push(
|
||||
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
|
||||
)
|
||||
lines.push(
|
||||
`Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
|
||||
)
|
||||
lines.push(
|
||||
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
export default new StatusService()
|
||||
@@ -4,6 +4,7 @@ import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { Prefetcher } from '#services/prefetcher'
|
||||
import Planner from '#services/planner'
|
||||
import Leases from '#services/leases'
|
||||
import Settler from '#services/settler'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
@@ -101,6 +102,13 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
|
||||
}
|
||||
try {
|
||||
const settled = await Settler.settleComplete()
|
||||
if (settled > 0)
|
||||
this.logger.info(`Settled ${settled} complete provisional releases quiet for hours.`)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Settling complete releases failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
@@ -117,8 +125,11 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
}
|
||||
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
|
||||
tailedAt = Date.now()
|
||||
const { groups, ranges } = await Planner.planTail()
|
||||
if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
|
||||
const { groups, ranges, extended } = await Planner.planTail()
|
||||
if (ranges + extended > 0)
|
||||
this.logger.info(
|
||||
`Tail: ${ranges} ranges added, ${extended} grown, for ${groups} groups.`
|
||||
)
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Coordinator round failed: ${error.message}`)
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import StatusService from '#services/status'
|
||||
|
||||
/**
|
||||
* What the indexer is doing (app/services/status.ts): the catalog's room, the ranges by band, the last hour's
|
||||
* rates, the held groups and their snapshots, the phase-2 backlogs. Reads only.
|
||||
*/
|
||||
export default class CoordinatorStatus extends BaseCommand {
|
||||
static commandName = 'coordinator:status'
|
||||
static description =
|
||||
'Prints what the indexer is doing: catalog room, ranges by band, rates, held groups, backlogs'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'JSON instead of the text panel' })
|
||||
declare json: boolean
|
||||
|
||||
async run() {
|
||||
const status = await StatusService.collect()
|
||||
this.logger.log(this.json ? JSON.stringify(status, null, 2) : StatusService.render(status))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { describeClass } from '#lib/release/index'
|
||||
|
||||
/**
|
||||
* The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and
|
||||
* `target_bytes` (the first target's bytes, when its candidates carry them). The finished rows (named, exhausted,
|
||||
* naming) in one scan, seeding each class's record, which gets its verdict (dropped) as the ladder gives it; the
|
||||
* waiting rows in windows of created_at by their index (a scan per window starved the prober on the HDD); then
|
||||
* pending_classes' `waiting` counted afresh from the claim's index. --report shows the record.
|
||||
*/
|
||||
export default class PendingClasses extends BaseCommand {
|
||||
static commandName = 'pending:classes'
|
||||
static description =
|
||||
'Classes the pending releases from before the classes and recounts pending_classes; --report shows the record'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Hours of created_at per window' })
|
||||
declare hours: number
|
||||
|
||||
@flags.boolean({ description: 'Show the classes with the best and worst records, write nothing' })
|
||||
declare report: boolean
|
||||
|
||||
async run() {
|
||||
if (this.report) return this.showReport()
|
||||
const hours = this.hours ?? 6
|
||||
const started = Date.now()
|
||||
// The finished rows (named, exhausted, naming; 6.5M on 2026-11-02) in one pass: a scan of the table, no index
|
||||
// reaches them by status; they seed each class's record.
|
||||
const { rows: seeded } = await db.rawQuery(
|
||||
`WITH classed AS (
|
||||
UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
|
||||
target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
|
||||
WHERE klass = 0 AND status IN (1, 3, 4)
|
||||
RETURNING tool, group_id, klass, status),
|
||||
upserted AS (
|
||||
INSERT INTO pending_classes (tool, group_id, klass, probed, named)
|
||||
SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed GROUP BY 1, 2, 3
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named
|
||||
RETURNING 1)
|
||||
SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM upserted) AS classes`
|
||||
)
|
||||
this.logger.info(
|
||||
`Finished rows: ${seeded[0].classed} classed into ${seeded[0].classes} classes (${Math.round((Date.now() - started) / 1000)} s).`
|
||||
)
|
||||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET dropped = true, swept_at = NULL WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`,
|
||||
[pendingMinProbes, pendingMaxNamed]
|
||||
)
|
||||
// The waiting rows by windows of created_at (pending_releases_waiting_idx reaches them).
|
||||
const { rows: span } = await db.rawQuery(
|
||||
`SELECT (SELECT min(created_at) FROM pending_releases WHERE status = 0) AS lo, (SELECT max(created_at) FROM pending_releases WHERE status = 0) AS hi`
|
||||
)
|
||||
if (span[0].lo) {
|
||||
let from = new Date(span[0].lo)
|
||||
const hi = new Date(span[0].hi)
|
||||
let classed = 0
|
||||
while (from <= hi) {
|
||||
const to = new Date(from.getTime() + hours * 3600_000)
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
|
||||
target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
|
||||
WHERE status = 0 AND created_at >= ? AND created_at < ? AND klass = 0`,
|
||||
[from.toISOString(), to.toISOString()]
|
||||
)
|
||||
classed += result.rowCount ?? 0
|
||||
from = to
|
||||
this.logger.info(
|
||||
`Waiting rows up to ${to.toISOString().slice(0, 16)}: ${classed} classed (${Math.round((Date.now() - started) / 1000)} s).`
|
||||
)
|
||||
}
|
||||
}
|
||||
// The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here).
|
||||
await db.rawQuery(`
|
||||
INSERT INTO pending_classes (tool, group_id, klass, waiting)
|
||||
SELECT tool, group_id, klass, count(*) FROM pending_releases WHERE status = 0 AND finished GROUP BY 1, 2, 3
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = excluded.waiting`)
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes c SET waiting = 0 WHERE waiting > 0 AND NOT EXISTS (
|
||||
SELECT 1 FROM pending_releases p WHERE p.status = 0 AND p.finished AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass)`
|
||||
)
|
||||
const { rows: totals } = await db.rawQuery(
|
||||
`SELECT count(*) AS classes, coalesce(sum(waiting), 0) AS waiting, count(*) FILTER (WHERE dropped) AS dropped FROM pending_classes`
|
||||
)
|
||||
this.logger.info(
|
||||
`Done: ${totals[0].classes} classes, ${totals[0].waiting} waiting, ${totals[0].dropped} dropped.`
|
||||
)
|
||||
}
|
||||
|
||||
private async showReport() {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named, c.waiting, c.dropped FROM pending_classes c JOIN groups g ON g.id = c.group_id
|
||||
WHERE c.probed >= 20 ORDER BY c.named::float / c.probed DESC, c.probed DESC`
|
||||
)
|
||||
const line = (r: any) =>
|
||||
` ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} probed ${String(r.probed).padStart(7)} named ${String(Math.round((100 * r.named) / r.probed)).padStart(3)}% waiting ${String(r.waiting).padStart(8)}${r.dropped ? ' dropped' : ''}`
|
||||
this.logger.info(`${rows.length} classes with 20+ climbs; the best:`)
|
||||
for (const r of rows.slice(0, 25)) this.logger.info(line(r))
|
||||
this.logger.info('The worst:')
|
||||
for (const r of rows.slice(-25)) this.logger.info(line(r))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import Planner from '#services/planner'
|
||||
|
||||
/**
|
||||
* Merges adjacent planned tail ranges (Planner.mergeTail): the tail planner added a sliver a group every round
|
||||
* while nothing leased the tail; now it grows the group's open range instead, and this packs what is there.
|
||||
*/
|
||||
export default class RangesMergeTail extends BaseCommand {
|
||||
static commandName = 'ranges:merge-tail'
|
||||
static description = 'Merges adjacent planned tail ranges into ranges of RANGE_ARTICLES'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'Only count what would be merged' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const { candidates, merged, dropped } = await Planner.mergeTail(this.dryRun)
|
||||
this.logger.info(
|
||||
`${candidates} planned tail ranges: ${dropped} ${this.dryRun ? 'would merge' : 'merged'} into ${merged} grown ranges (${candidates - dropped} left).`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,13 @@ export default class RangesPlan extends BaseCommand {
|
||||
|
||||
async run() {
|
||||
try {
|
||||
const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
|
||||
this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
|
||||
if (this.tail) {
|
||||
const { groups, ranges, extended } = await Planner.planTail()
|
||||
this.logger.info(`Tail: ${ranges} ranges added, ${extended} grown, over ${groups} groups.`)
|
||||
} else {
|
||||
const { groups, ranges } = await Planner.planAll()
|
||||
this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
|
||||
}
|
||||
} finally {
|
||||
await NntpPool.close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { isUnreadableName } from '#lib/release/index'
|
||||
|
||||
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||
const CURSOR = 'readable'
|
||||
|
||||
/**
|
||||
* The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
|
||||
* readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
|
||||
* false, so the readable filter hides them (an inner name from the prober still shows them), and their probe
|
||||
* candidates marked unreadable, the prober's first tiers. By the release name,
|
||||
* which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
|
||||
* Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
|
||||
*/
|
||||
export default class ReleasesReadable extends BaseCommand {
|
||||
static commandName = 'releases:readable'
|
||||
static description =
|
||||
'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Release ids per batch' })
|
||||
declare batch: number
|
||||
|
||||
@flags.number({ description: 'Stop after this id (a sample)' })
|
||||
declare toId: number
|
||||
|
||||
@flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
|
||||
declare apply: boolean
|
||||
|
||||
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||
declare restart: boolean
|
||||
|
||||
async run() {
|
||||
const batch = this.batch ?? 20_000
|
||||
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
|
||||
CURSOR,
|
||||
])
|
||||
let after =
|
||||
this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||
let looked = 0
|
||||
let changed = 0
|
||||
const sample: string[] = []
|
||||
const started = Date.now()
|
||||
while (after < hi) {
|
||||
const to = Math.min(after + batch, hi)
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
|
||||
[after, to]
|
||||
)
|
||||
looked += rows.length
|
||||
const ids: number[] = []
|
||||
for (const r of rows) {
|
||||
if (!isUnreadableName(r.name)) continue
|
||||
ids.push(Number(r.id))
|
||||
if (sample.length < 40) sample.push(r.name)
|
||||
}
|
||||
changed += ids.length
|
||||
if (this.apply) {
|
||||
if (ids.length > 0) {
|
||||
await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
|
||||
ids,
|
||||
])
|
||||
// Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)).
|
||||
await db.rawQuery(
|
||||
`UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`,
|
||||
[ids]
|
||||
)
|
||||
}
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, String(to)]
|
||||
)
|
||||
}
|
||||
after = to
|
||||
if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
|
||||
this.logger.info(
|
||||
`Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
|
||||
)
|
||||
}
|
||||
}
|
||||
this.logger.info(
|
||||
`${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
|
||||
)
|
||||
if (!this.apply) for (const n of sample) this.logger.info(` ${n}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { classKey, describeClass } from '#lib/release/index'
|
||||
|
||||
/**
|
||||
* Publishes a rule set whose dropped classes are the pending ladder's (pending_classes.dropped; LOG.md (130)): the
|
||||
* edge keeps unnamed releases of those classes out of the pending store from then on, their explored sample
|
||||
* excepted. Run again as the record grows; unchanged classes publish nothing.
|
||||
*/
|
||||
export default class RulesClasses extends BaseCommand {
|
||||
static commandName = 'rules:classes'
|
||||
static description = "Publishes a rule set with the pending ladder's dropped classes"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'Show what would be published, write nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named FROM pending_classes c JOIN groups g ON g.id = c.group_id WHERE c.dropped ORDER BY 1, 2, 3`
|
||||
)
|
||||
const next = new Set<string>(rows.map((r: any) => classKey(r.tool, r.grp, Number(r.klass))))
|
||||
const { version, priorities: _p, ...current } = await RuleSets.current()
|
||||
const before = new Set(current.droppedClasses ?? [])
|
||||
const added = [...next].filter((c) => !before.has(c))
|
||||
const removed = [...before].filter((c) => !next.has(c))
|
||||
if (added.length === 0 && removed.length === 0) {
|
||||
this.logger.info(
|
||||
`Unchanged: rule set ${version} has the ${before.size} dropped classes already.`
|
||||
)
|
||||
return
|
||||
}
|
||||
for (const r of rows.slice(0, 40)) {
|
||||
this.logger.info(
|
||||
` ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} ${r.named} of ${r.probed} named`
|
||||
)
|
||||
}
|
||||
if (this.dryRun) {
|
||||
this.logger.info(
|
||||
`Would publish ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
|
||||
)
|
||||
return
|
||||
}
|
||||
const published = await RuleSets.publish(
|
||||
{ ...current, droppedClasses: [...next].sort() },
|
||||
`${next.size} dropped classes from the pending ladder`
|
||||
)
|
||||
this.logger.info(
|
||||
`Rule set ${published}: ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
|
||||
)
|
||||
}
|
||||
}
|
||||
+11
-2
@@ -33,11 +33,20 @@ export default {
|
||||
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
||||
/**
|
||||
* The ranges group leases hand out (REBUILD.md design 1), by priority: the tail is negative, the capture windows
|
||||
* 0–2 (2016, 2021, 2025), the rest of the history above (its position in the group, or 4000+). Bounds so a
|
||||
* deployment can run one window first (homelab 2026-10-26: 0..0, the 2016 window); unbounded by default.
|
||||
* 0–2 (2016, 2021, 2025), the history by its position in the group (0 oldest … 1000 newest: its first buckets
|
||||
* share 0–2 with the windows) or 4000+. Bounds so a deployment can run one window first, with
|
||||
* LEASE_CAPTURE_ONLY so the history's 0 isn't taken along (homelab 2026-10-26: 0..0 and capture only, the 2016
|
||||
* window; LOG.md (118)); unbounded and any range by default.
|
||||
*/
|
||||
leasePriorityMin: env.get('LEASE_MIN_PRIORITY', -1_000_000),
|
||||
leasePriorityMax: env.get('LEASE_MAX_PRIORITY', 1_000_000),
|
||||
leaseCaptureOnly: env.get('LEASE_CAPTURE_ONLY', false),
|
||||
/**
|
||||
* How long a range's stream waits for a chunk the prefetcher hasn't spooled yet (app/controllers/node_controller.ts).
|
||||
* Long: a chunk that keeps coming in isn't given up (only a stalled one is), and some groups' chunks take 20+
|
||||
* minutes (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
|
||||
*/
|
||||
streamChunkWaitMs: env.get('STREAM_CHUNK_WAIT_MINUTES', 60) * 60 * 1000,
|
||||
/** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
|
||||
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
|
||||
nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Lease lanes (REBUILD.md, "Voyager 3: window indexing", 1, amended 2026-10-27): a group has two cursors, each with
|
||||
* its own holder, window and snapshot. The `history` lane works the ranges the lease bounds hand out (the windows,
|
||||
* the history), the `tail` lane the tail planner's (priority at or below the tail's): a thread on the tail never
|
||||
* waits behind a group's history, and a held history group's tail isn't stuck behind its days of ranges.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`ALTER TABLE group_leases ADD COLUMN lane varchar(8) NOT NULL DEFAULT 'history'`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
|
||||
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id, lane)`)
|
||||
// Counts the leases handed out: odd and even ones start from the other half of the order's mix.
|
||||
this.schema.raw(`CREATE SEQUENCE group_lease_turns`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP SEQUENCE IF EXISTS group_lease_turns`)
|
||||
this.schema.raw(`DELETE FROM group_leases WHERE lane <> 'history'`)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP COLUMN lane`)
|
||||
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id)`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* One "ancient" range a group, articles 1 … (its first range − 1): what had expired on the server before the
|
||||
* group was planned (the planner cuts from the low watermark). Status 'ancient': nothing to fetch, never leased,
|
||||
* so the tiling is complete from article 1 and the span below the low watermark is on record (user, 2026-10-28).
|
||||
* Planner.historyRows adds it for groups planned from now on.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`INSERT INTO ranges (group_id, start_id, end_id, next_id, priority, status)
|
||||
SELECT group_id, 1, min(start_id) - 1, 1, 0, 'ancient' FROM ranges GROUP BY group_id HAVING min(start_id) > 1`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DELETE FROM ranges WHERE status = 'ancient'`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The pending ladder by class (REBUILD.md decision f; LOG.md (130)): a pending release's class (lib/release/classes.ts:
|
||||
* size bucket × name shape × partial, 1..70; 0 for rows from before) and the bytes of its first target, so the
|
||||
* claim takes the classes with the best record first and the cheapest article within one; pending_classes per
|
||||
* (tool, group, class) how many climbed to the end and how many got a name, `waiting` how many wait (kept by the
|
||||
* batch store, the claim and the discard; recounted by pending:classes), `dropped` as pending_signatures.
|
||||
* The SQL pending_class() is the TypeScript's pendingClass, for the batch store's insert and the backfill.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION pending_class(name text, bytes bigint, partial boolean) RETURNS smallint LANGUAGE sql IMMUTABLE AS $$
|
||||
SELECT ((CASE WHEN bytes < 1000000 THEN 0 WHEN bytes < 10000000 THEN 1 WHEN bytes < 100000000 THEN 2 WHEN bytes < 1000000000 THEN 3 ELSE 4 END) * 14
|
||||
+ (CASE WHEN s ~ '^[0-9a-fA-F]{16,}$' THEN 1
|
||||
WHEN s ~ '^[0-9]{6,}$' THEN 2
|
||||
WHEN s ~ '^[a-z2-7]{20,}$' AND s ~ '[2-7]' THEN 3
|
||||
WHEN s ~ '^[A-Za-z]+$' THEN 4
|
||||
WHEN s ~ '^[A-Za-z0-9]+$' THEN 5
|
||||
WHEN s ~ '^[A-Za-z0-9._-]+$' THEN 6
|
||||
ELSE 7 END - 1) * 2
|
||||
+ (CASE WHEN partial THEN 1 ELSE 0 END) + 1)::smallint
|
||||
FROM (SELECT regexp_replace(name, '\\.[A-Za-z0-9]{1,4}$', '') AS s) x
|
||||
$$`)
|
||||
this.schema.raw(
|
||||
`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS klass smallint NOT NULL DEFAULT 0`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS target_bytes bigint`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE IF NOT EXISTS pending_classes (
|
||||
tool varchar(16) NOT NULL,
|
||||
group_id integer NOT NULL,
|
||||
klass smallint NOT NULL,
|
||||
probed integer NOT NULL DEFAULT 0,
|
||||
named integer NOT NULL DEFAULT 0,
|
||||
waiting integer NOT NULL DEFAULT 0,
|
||||
dropped boolean NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (tool, group_id, klass)
|
||||
)`)
|
||||
// The claim by class: the cheapest target first within one, the oldest among equals.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_class_idx ON pending_releases (tool, group_id, klass, target_bytes, created_at) WHERE status = 0`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_class_idx`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS pending_classes`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS target_bytes`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS klass`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS pending_class(text, bigint, boolean)`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The prober's start puts a previous run's claims back (PendingLadder.release: status 9 → 0), which scanned the
|
||||
* 58M-row pending table on the HDD for 20+ minutes after every deploy (LOG.md (130)): a partial index on the few
|
||||
* taken rows makes it instant.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_taken_idx ON pending_releases (release_key) WHERE status = 9`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_taken_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The prober's start puts a previous run's claims back (probe_candidates status taken → waiting), which scanned
|
||||
* the 7.8 GB candidates table on the HDD for 4 minutes after every deploy (LOG.md (133)): a partial index on the
|
||||
* few taken rows makes it instant.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_taken_idx ON probe_candidates (id) WHERE status = 'taken'`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_taken_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The ladder's discard sweep (PendingLadder.discard) reads a dropped signature's or class's waiting rows once:
|
||||
* `swept_at` marks it done (null again when the record drops it anew). Before, every sweep read every dropped
|
||||
* signature's rows over again, minutes of HDD each, inside the claim (LOG.md (133)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('pending_signatures', (table) => {
|
||||
table.timestamp('swept_at', { useTz: true }).nullable()
|
||||
})
|
||||
this.schema.alterTable('pending_classes', (table) => {
|
||||
table.timestamp('swept_at', { useTz: true }).nullable()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('pending_signatures', (table) => {
|
||||
table.dropColumn('swept_at')
|
||||
})
|
||||
this.schema.alterTable('pending_classes', (table) => {
|
||||
table.dropColumn('swept_at')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The discard sweep of a dropped signature (tool, group, poster) read the whole pending_releases table per
|
||||
* signature: the planner took a parallel seq scan over the poster test, four minutes on the HDD that starved the
|
||||
* lanes' claims (LOG.md (134)). The waiting rows by signature make it an index range. By the poster's md5: a
|
||||
* poster of 2.7 kB broke the first build (a btree key is at most a third of a page), which left an invalid index
|
||||
* behind that IF NOT EXISTS would have taken for the real one; the drop clears it.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_signature_idx`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_signature_idx ON pending_releases (group_id, tool, md5(poster)) WHERE status = 0`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_signature_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* probe_candidates, with its indexes, in the catalog's tablespace (default_tablespace: nvme on homelab), out of
|
||||
* CHUNK_TABLESPACE where migration 6000 put it: the prober's catalog claim updates it from random points every
|
||||
* minute, and on the HDD, beside the pending ladder's sweep and autovacuum, a claim took 2–3 minutes with all the
|
||||
* lanes waiting (LOG.md 2026-11-03 (136)). Homelab moved it by hand (edge, node-api and the workers stopped; ~8 GB);
|
||||
* a no-op where it's there already, or without a default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(
|
||||
`SELECT current_setting('default_tablespace') AS target`
|
||||
)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE (c.relname = 'probe_candidates' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'probe_candidates'::regclass))
|
||||
AND c.relkind IN ('r', 'i') AND coalesce(t.spcname, 'pg_default') <> ?
|
||||
ORDER BY c.relkind DESC, c.relname`,
|
||||
[target]
|
||||
)
|
||||
for (const { relname, relkind } of rows) {
|
||||
await db.rawQuery(
|
||||
`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Everything the prober reads or writes in the catalog's tablespace (default_tablespace: nvme on homelab), out of
|
||||
* CHUNK_TABLESPACE (the HDD): pending_releases with its two HDD indexes (migration 6300), release_inspections
|
||||
* (6000), probe_cache with its toast. The edge's batch upserts into pending_releases took minutes each on the HDD and
|
||||
* the prober's flushes waited on them (LOG.md 2026-11-03 (137)); and with these on the NVMe the prober can run
|
||||
* through the night without the HDD. Homelab moved them by hand (edge, node-api and the workers stopped; ~66 GB);
|
||||
* a no-op where they're there already, or without a default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(
|
||||
`SELECT current_setting('default_tablespace') AS target`
|
||||
)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
for (const table of ['pending_releases', 'release_inspections', 'probe_cache']) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE (c.relname = ? OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = ?::regclass))
|
||||
AND c.relkind IN ('r', 'i') AND coalesce(t.spcname, 'pg_default') <> ?
|
||||
ORDER BY c.relkind DESC, c.relname`,
|
||||
[table, table, target]
|
||||
)
|
||||
for (const { relname, relkind } of rows) {
|
||||
await db.rawQuery(
|
||||
`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -178,6 +178,8 @@ services:
|
||||
# The window worker (REBUILD.md design 1 and 5): a group per thread, the window's snapshot to the coordinator
|
||||
# this often; the coordinator keeps them under SNAPSHOT_DIR (the spool's snapshots/ on bulk by default).
|
||||
EDGE_SNAPSHOT_MINUTES: ${EDGE_SNAPSHOT_MINUTES:-30}
|
||||
# Of the threads, this many on the groups' tail lane (the live tail; the rest work the history lane).
|
||||
EDGE_TAIL_THREADS: ${EDGE_TAIL_THREADS:-1}
|
||||
stop_grace_period: 60s
|
||||
depends_on:
|
||||
node-api:
|
||||
|
||||
+3
-2
@@ -47,8 +47,9 @@ echo "== recreating app services"
|
||||
docker compose stop edge
|
||||
if [ -e "$CACHE/paused" ]; then
|
||||
# Paused for the night (~/bin/usenet-rebuild-pause.sh): the workers stay stopped, the resume starts the new build.
|
||||
echo "== paused: coordinator, prober, merger, judge, spotter, spot-inspector and edge left stopped"
|
||||
docker compose up -d --no-deps node-api web
|
||||
# The prober runs through the night (2026-11-03, LOG.md (137)), so it gets the new build now.
|
||||
echo "== paused: coordinator, merger, judge, spotter, spot-inspector and edge left stopped"
|
||||
docker compose up -d --no-deps node-api web prober
|
||||
else
|
||||
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
||||
if [ -e "$CACHE/no-edge" ]; then
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Installed as ~/bin/usenet-rebuild-pause.sh (by hand). Nightly (homelab crontab, 23:45): voyager 2's processing paused for the night; usenet-rebuild-resume.sh (07:45)
|
||||
# starts it again. Postgres, web and node-api stay up. Takes the deploy lock, so it never runs during a CI deploy, and
|
||||
# leaves .cache/paused, so a deploy in the night doesn't start the workers again.
|
||||
# starts it again. Postgres, web, node-api and the prober stay up: since 2026-11-03 (LOG.md (137)) everything the
|
||||
# prober touches is on the NVMe, so it probes through the night without the HDD. Takes the deploy lock, so it never
|
||||
# runs during a CI deploy, and leaves .cache/paused, so a deploy in the night doesn't start the workers again.
|
||||
cd /mnt/bulk/usenet-rebuild || exit 1
|
||||
exec 9>.cache/deploy.lock
|
||||
flock 9
|
||||
touch .cache/paused
|
||||
date "+%F %T pausing"
|
||||
docker compose stop edge
|
||||
docker compose stop coordinator prober merger judge spotter spot-inspector
|
||||
docker compose stop coordinator merger judge spotter spot-inspector
|
||||
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||
+25
-5
@@ -234,9 +234,20 @@ them:
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--tail-threads T] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
```
|
||||
|
||||
- **Two lanes** (2026-10-27): `--tail-threads T` (EDGE_TAIL_THREADS; one of
|
||||
two or more threads by default) puts that many threads on the groups'
|
||||
*tail* lane (`?lane=tail` on the group routes): the tail planner's
|
||||
ranges, through a window and snapshot of their own
|
||||
(`<group>-tail.gz`). The other threads work the history lane: the
|
||||
windows and the history the coordinator's bounds hand out. A tail thread
|
||||
never wades through a group's history first, and a held history group's
|
||||
tail isn't stuck behind its days of ranges; the coordinator orders each
|
||||
lane's leases by alternating the group with the most planned articles
|
||||
and the longest unheld one.
|
||||
|
||||
- **One group per thread** (default one per core; REBUILD.md, "Voyager 3:
|
||||
window indexing", 1 and 5): a thread leases a group (`POST
|
||||
groups/lease`), takes its ranges in article order (`POST
|
||||
@@ -248,8 +259,11 @@ COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-nod
|
||||
snapshot (bincode, gzip) goes to the coordinator every
|
||||
`--snapshot-minutes` (EDGE_SNAPSHOT_MINUTES, 30) right after a batch,
|
||||
and when the group is given back: caught up (`next` answers 204: the
|
||||
tail), a stop, a lost lease or a failure; the next holder restores it
|
||||
(`GET groups/:id/snapshot`) and continues at its article. `--groups N`
|
||||
tail), a stop, a lost lease or a failure; a stop or a failed stream
|
||||
posts the open batch first, so the snapshot stands at its resume point
|
||||
(a batch the coordinator refused can't, and that window is lost); the
|
||||
next holder restores it (`GET groups/:id/snapshot`) and continues at
|
||||
its article. `--groups N`
|
||||
stops after N groups; `--ranges N` is the old per-range lease
|
||||
(`POST lease`), one range per thread without a carried window.
|
||||
- **As `commands/edge_node.ts`:** retries for network trouble and 5xx;
|
||||
@@ -270,5 +284,11 @@ COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-nod
|
||||
same (releases with every column, chunks, candidates, rule counts,
|
||||
fragments with their JSON).
|
||||
- **On homelab:** compose service `edge` (deploy/compose.yaml), the
|
||||
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8);
|
||||
it talks to `node-api`.
|
||||
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8),
|
||||
EDGE_TAIL_THREADS (1); it talks to `node-api`.
|
||||
- **End to end:** `tests/e2e/edge_node.spec.ts` runs this binary
|
||||
(`edge/target/release/edge-node`, or EDGE_NODE_BIN) against the test
|
||||
server, database, spool and corpus: a chunk the corpus fill stores while
|
||||
the stream waits, a failed stream and the restart that restores the
|
||||
snapshot, a restart whose snapshot file is gone, a takeover. `node ace
|
||||
test e2e` after `cargo build --release`; CI builds the binary first.
|
||||
+57
-38
@@ -5,7 +5,11 @@
|
||||
//! --snapshot-minutes and when the group is given back, and the next holder restores it. A group with no range
|
||||
//! left (caught up: the tail) is given back and leased again later.
|
||||
//!
|
||||
//! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
//! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--tail-threads T] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
//!
|
||||
//! `--tail-threads T` (EDGE_TAIL_THREADS; 1 of 2 or more threads by default): that many of the threads lease the
|
||||
//! groups' *tail* lane (`?lane=tail`: the tail planner's ranges, with a window and snapshot of their own), the rest
|
||||
//! the history lane (the windows and the history the coordinator's bounds hand out).
|
||||
//!
|
||||
//! SIGTERM/SIGINT: each thread finishes the chunk, posts its batch, snapshots and gives its group back.
|
||||
//! `--ranges`: the old range leases (POST lease), one range per thread without a carried window (kept for a
|
||||
@@ -167,10 +171,10 @@ impl Api {
|
||||
}
|
||||
}
|
||||
|
||||
// Group leases (REBUILD.md design 1 and 5).
|
||||
// Group leases (REBUILD.md design 1 and 5), per lane (`history` or `tail`).
|
||||
|
||||
fn lease_group(&self) -> Result<Option<GroupLeaseJson>, ApiError> {
|
||||
let mut response = self.request("POST", "groups/lease", None, false, "application/json")?;
|
||||
fn lease_group(&self, lane: &str) -> Result<Option<GroupLeaseJson>, ApiError> {
|
||||
let mut response = self.request("POST", &format!("groups/lease?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
204 => Ok(None),
|
||||
200 => {
|
||||
@@ -181,8 +185,8 @@ impl Api {
|
||||
}
|
||||
}
|
||||
|
||||
fn next_range(&self, group: i64) -> Result<Option<RangeSpanJson>, ApiError> {
|
||||
let mut response = self.request("POST", &format!("groups/{group}/next"), None, false, "application/json")?;
|
||||
fn next_range(&self, group: i64, lane: &str) -> Result<Option<RangeSpanJson>, ApiError> {
|
||||
let mut response = self.request("POST", &format!("groups/{group}/next?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
204 => Ok(None),
|
||||
200 => {
|
||||
@@ -193,8 +197,8 @@ impl Api {
|
||||
}
|
||||
}
|
||||
|
||||
fn group_call(&self, group: i64, action: &str) -> Result<(), ApiError> {
|
||||
let response = self.request("POST", &format!("groups/{group}/{action}"), None, false, "application/json")?;
|
||||
fn group_call(&self, group: i64, lane: &str, action: &str) -> Result<(), ApiError> {
|
||||
let response = self.request("POST", &format!("groups/{group}/{action}?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
200..=299 => Ok(()),
|
||||
status => Err(ApiError::Other(format!("{action}: HTTP {status}"))),
|
||||
@@ -202,11 +206,11 @@ impl Api {
|
||||
}
|
||||
|
||||
/// The window's snapshot (bincode), gzipped, standing at `next`.
|
||||
fn put_snapshot(&self, group: i64, next: i64, snapshot: &[u8]) -> Result<usize, ApiError> {
|
||||
fn put_snapshot(&self, group: i64, lane: &str, next: i64, snapshot: &[u8]) -> Result<usize, ApiError> {
|
||||
let mut gz = GzEncoder::new(Vec::with_capacity(snapshot.len() / 3), flate2::Compression::new(1));
|
||||
gz.write_all(snapshot).unwrap();
|
||||
let body = gz.finish().unwrap();
|
||||
let url = format!("{}/api/v1/node/groups/{group}/snapshot?next={next}", self.base);
|
||||
let url = format!("{}/api/v1/node/groups/{group}/snapshot?next={next}&lane={lane}", self.base);
|
||||
let auth = format!("Bearer {}", self.token);
|
||||
for attempt in 1..=RETRIES {
|
||||
let result = self.agent.put(&url).header("Authorization", &auth).header("Content-Type", "application/octet-stream").send(&body[..]);
|
||||
@@ -227,8 +231,8 @@ impl Api {
|
||||
}
|
||||
|
||||
/// The group's snapshot: its bytes (inflated) and the article it stands at; None without one.
|
||||
fn get_snapshot(&self, group: i64) -> Result<Option<(i64, Vec<u8>)>, ApiError> {
|
||||
let response = self.request("GET", &format!("groups/{group}/snapshot"), None, false, "application/octet-stream")?;
|
||||
fn get_snapshot(&self, group: i64, lane: &str) -> Result<Option<(i64, Vec<u8>)>, ApiError> {
|
||||
let response = self.request("GET", &format!("groups/{group}/snapshot?lane={lane}"), None, false, "application/octet-stream")?;
|
||||
match response.status().as_u16() {
|
||||
404 => Ok(None),
|
||||
200 => {
|
||||
@@ -544,9 +548,10 @@ enum RangeOutcome {
|
||||
}
|
||||
|
||||
/// One range of a held group through the carried window: the chunks streamed, batches posted at the range's end at
|
||||
/// least, the range finished. The indexer keeps the window for the next range.
|
||||
/// least, the range finished. The indexer keeps the window for the next range. `group_name` is the label for the
|
||||
/// log (the tail lane's says so).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn run_group_range(api: &Arc<Api>, group_id: i64, group_name: &str, range: RangeSpanJson, indexer: &mut RangeIndexer, config: &Config, stopping: &AtomicBool, totals: &Arc<Mutex<Totals>>) -> RangeOutcome {
|
||||
fn run_group_range(api: &Arc<Api>, group_id: i64, lane: &str, group_name: &str, range: RangeSpanJson, indexer: &mut RangeIndexer, config: &Config, stopping: &AtomicBool, totals: &Arc<Mutex<Totals>>) -> RangeOutcome {
|
||||
let started = Instant::now();
|
||||
indexer.begin_range(range.next, range.end);
|
||||
let post_error: Arc<Mutex<Option<ApiError>>> = Arc::new(Mutex::new(None));
|
||||
@@ -597,7 +602,7 @@ fn run_group_range(api: &Arc<Api>, group_id: i64, group_name: &str, range: Range
|
||||
}
|
||||
if since.elapsed().as_millis() as i64 - last_contact.load(Ordering::Relaxed) > HEARTBEAT.as_millis() as i64 {
|
||||
last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
|
||||
let _ = api.group_call(group_id, "heartbeat");
|
||||
let _ = api.group_call(group_id, lane, "heartbeat");
|
||||
}
|
||||
match frame {
|
||||
Frame::KeepAlive => continue,
|
||||
@@ -629,9 +634,10 @@ fn run_group_range(api: &Arc<Api>, group_id: i64, group_name: &str, range: Range
|
||||
}
|
||||
}
|
||||
}
|
||||
// Stopped mid-range (or at a chunk that doesn't inflate): the batch so far goes out, so the snapshot stands at
|
||||
// a stored resume point, and a chunk reported corrupt is read again from its own start.
|
||||
if !reached_end && failure.is_none() && post_error.lock().unwrap().is_none() && indexer.next() > batch_from {
|
||||
// Stopped mid-range (a stop, a stream that failed, a chunk that doesn't inflate): the batch so far goes out,
|
||||
// so the snapshot stands at a stored resume point and the window survives the group going back; a chunk
|
||||
// reported corrupt is read again from its own start. Only a batch the coordinator refused stays open.
|
||||
if !reached_end && post_error.lock().unwrap().is_none() && indexer.next() > batch_from {
|
||||
let _ = post_tx.send(indexer.take());
|
||||
}
|
||||
drop(post_tx);
|
||||
@@ -705,9 +711,9 @@ fn run_group_range(api: &Arc<Api>, group_id: i64, group_name: &str, range: Range
|
||||
}
|
||||
|
||||
/// The window's snapshot to the coordinator, standing at the indexer's next article.
|
||||
fn snapshot(api: &Api, group_id: i64, group_name: &str, indexer: &RangeIndexer) {
|
||||
fn snapshot(api: &Api, group_id: i64, lane: &str, group_name: &str, indexer: &RangeIndexer) {
|
||||
match indexer.snapshot() {
|
||||
Ok(bytes) => match api.put_snapshot(group_id, indexer.next(), &bytes) {
|
||||
Ok(bytes) => match api.put_snapshot(group_id, lane, indexer.next(), &bytes) {
|
||||
Ok(gz) => log("info", &format!("{group_name}: snapshot at {} ({:.1} MB, {:.1} MB gzipped).", indexer.next(), bytes.len() as f64 / 1e6, gz as f64 / 1e6)),
|
||||
Err(e) => log("warn", &format!("{group_name}: snapshot failed: {e}")),
|
||||
},
|
||||
@@ -715,14 +721,16 @@ fn snapshot(api: &Api, group_id: i64, group_name: &str, indexer: &RangeIndexer)
|
||||
}
|
||||
}
|
||||
|
||||
/// One held group: its window restored from the snapshot (or new), its ranges one after another until it is
|
||||
/// caught up or the worker stops; then a snapshot, and the group given back.
|
||||
fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, config: &Config, stopping: &AtomicBool) {
|
||||
/// One held group (one lane of it): its window restored from the snapshot (or new), its ranges one after another
|
||||
/// until it is caught up or the worker stops; then a snapshot, and the group given back.
|
||||
fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, lane: &str, config: &Config, stopping: &AtomicBool) {
|
||||
let group_id = lease.group.id;
|
||||
let group_name = lease.group.name.clone();
|
||||
let name = lease.group.name.clone();
|
||||
// The log's label; the indexer gets the group's name itself (keys, xrefs).
|
||||
let group_name = if lane == "tail" { format!("{name} (tail)") } else { name.clone() };
|
||||
let rules = lease.rules.rules();
|
||||
let mut indexer = match lease.snapshot {
|
||||
Some(s) => match api.get_snapshot(group_id) {
|
||||
Some(s) => match api.get_snapshot(group_id, lane) {
|
||||
Ok(Some((next, bytes))) => match RangeIndexer::restore(&bytes, rules.clone(), 5) {
|
||||
Ok(ix) => {
|
||||
log("info", &format!("{group_name}: window restored from the snapshot at {next} ({:.1} MB).", bytes.len() as f64 / 1e6));
|
||||
@@ -730,20 +738,20 @@ fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, config: &Config, stopping: &
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name}: the snapshot at {} can't be restored ({e}); the window starts empty.", s.next));
|
||||
RangeIndexer::for_group(&group_name, rules.clone(), 5)
|
||||
RangeIndexer::for_group(&name, rules.clone(), 5)
|
||||
}
|
||||
},
|
||||
Ok(None) => {
|
||||
log("warn", &format!("{group_name}: the lease names a snapshot at {} but there is none; the window starts empty.", s.next));
|
||||
RangeIndexer::for_group(&group_name, rules.clone(), 5)
|
||||
RangeIndexer::for_group(&name, rules.clone(), 5)
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name}: snapshot: {e}; the group goes back."));
|
||||
let _ = api.group_call(group_id, "release");
|
||||
let _ = api.group_call(group_id, lane, "release");
|
||||
return;
|
||||
}
|
||||
},
|
||||
None => RangeIndexer::for_group(&group_name, rules.clone(), 5),
|
||||
None => RangeIndexer::for_group(&name, rules.clone(), 5),
|
||||
};
|
||||
let totals = Arc::new(Mutex::new(Totals::default()));
|
||||
let started = Instant::now();
|
||||
@@ -753,7 +761,7 @@ fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, config: &Config, stopping: &
|
||||
if stopping.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let range = match api.next_range(group_id) {
|
||||
let range = match api.next_range(group_id, lane) {
|
||||
Ok(Some(range)) => range,
|
||||
Ok(None) => {
|
||||
log("info", &format!("{group_name}: caught up after {ranges} ranges; the group goes back."));
|
||||
@@ -767,19 +775,19 @@ fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, config: &Config, stopping: &
|
||||
if range.next != indexer.next() && ranges > 0 {
|
||||
log("warn", &format!("{group_name}: the window stands at {} but the next range resumes at {}: the window carries on from there.", indexer.next(), range.next));
|
||||
}
|
||||
match run_group_range(api, group_id, &group_name, range, &mut indexer, config, stopping, &totals) {
|
||||
match run_group_range(api, group_id, lane, &group_name, range, &mut indexer, config, stopping, &totals) {
|
||||
RangeOutcome::Done => ranges += 1,
|
||||
// The same range again from the reported chunk: the stream waits for its new copy.
|
||||
RangeOutcome::Again => {}
|
||||
RangeOutcome::Stop => break,
|
||||
}
|
||||
if snapshotted.elapsed() >= config.snapshot_every {
|
||||
snapshot(api, group_id, &group_name, &indexer);
|
||||
snapshot(api, group_id, lane, &group_name, &indexer);
|
||||
snapshotted = Instant::now();
|
||||
}
|
||||
}
|
||||
snapshot(api, group_id, &group_name, &indexer);
|
||||
let _ = api.group_call(group_id, "release");
|
||||
snapshot(api, group_id, lane, &group_name, &indexer);
|
||||
let _ = api.group_call(group_id, lane, "release");
|
||||
let t = totals.lock().unwrap();
|
||||
log(
|
||||
"info",
|
||||
@@ -806,6 +814,12 @@ fn main() {
|
||||
.filter(|&n| n > 0)
|
||||
.unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1));
|
||||
let ranges: Option<usize> = flag("--ranges").map(|v| v.parse().unwrap());
|
||||
// Of the threads, this many on the groups' tail lane: one by default when there are two or more.
|
||||
let tail_threads: usize = flag("--tail-threads")
|
||||
.or_else(|| std::env::var("EDGE_TAIL_THREADS").ok())
|
||||
.map(|v| v.parse().unwrap())
|
||||
.unwrap_or(if threads >= 2 { 1 } else { 0 });
|
||||
assert!(tail_threads <= threads, "--tail-threads {tail_threads} is more than the {threads} threads");
|
||||
let max_mbit: Option<f64> = flag("--max-mbit").or_else(|| std::env::var("EDGE_MAX_MBIT").ok()).map(|v| v.parse().unwrap()).filter(|&m: &f64| m > 0.0);
|
||||
let config = Arc::new(Config {
|
||||
batch_articles: flag("--batch-articles").map(|v| v.parse().unwrap()).unwrap_or(500_000),
|
||||
@@ -818,12 +832,13 @@ fn main() {
|
||||
signal_hook::flag::register(signal_hook::consts::SIGTERM, stopping.clone()).unwrap();
|
||||
signal_hook::flag::register(signal_hook::consts::SIGINT, stopping.clone()).unwrap();
|
||||
let api = Arc::new(Api::new(&url, &token));
|
||||
log("info", &format!("Starting {threads} worker threads{}.", max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
|
||||
log("info", &format!("Starting {threads} worker threads{}{}.", if range_leases_wanted(&ranges) { String::new() } else { format!(", {tail_threads} on the tail") }, max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
|
||||
let groups: Option<usize> = flag("--groups").map(|v| v.parse().unwrap());
|
||||
let range_leases = ranges.is_some();
|
||||
let workers: Vec<_> = (0..threads)
|
||||
.map(|_| {
|
||||
.map(|i| {
|
||||
let (api, config, stopping) = (api.clone(), config.clone(), stopping.clone());
|
||||
let lane = if i < tail_threads { "tail" } else { "history" };
|
||||
std::thread::spawn(move || {
|
||||
let mut done = 0;
|
||||
if range_leases {
|
||||
@@ -843,9 +858,9 @@ fn main() {
|
||||
return;
|
||||
}
|
||||
while !stopping.load(Ordering::Relaxed) && groups.is_none_or(|n| done < n) {
|
||||
match api.lease_group() {
|
||||
match api.lease_group(lane) {
|
||||
Ok(Some(lease)) => {
|
||||
run_group(&api, lease, &config, &stopping);
|
||||
run_group(&api, lease, lane, &config, &stopping);
|
||||
done += 1;
|
||||
}
|
||||
Ok(None) if groups.is_some() => break,
|
||||
@@ -864,6 +879,10 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
fn range_leases_wanted(ranges: &Option<usize>) -> bool {
|
||||
ranges.is_some()
|
||||
}
|
||||
|
||||
fn idle(stopping: &AtomicBool) {
|
||||
let until = Instant::now() + IDLE;
|
||||
while Instant::now() < until && !stopping.load(Ordering::Relaxed) {
|
||||
|
||||
+72
-16
@@ -5,7 +5,7 @@
|
||||
|
||||
use crate::codec::{encode_chunk, ChunkFile};
|
||||
use crate::js::{self, sha1_hex};
|
||||
use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
|
||||
use crate::names::{self, is_random_file, is_unreadable_name, looks_scrambled, JudgedFile, Release, Rules, Verdict};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::rc::Rc;
|
||||
@@ -72,6 +72,8 @@ pub struct Candidate {
|
||||
pub kind: &'static str,
|
||||
pub message_id: String,
|
||||
pub file: String,
|
||||
/// The target article's bytes: the cheapest probe first within a class (LOG.md (130)).
|
||||
pub bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -1176,11 +1178,11 @@ impl RangeIndexer {
|
||||
let t = std::time::Instant::now();
|
||||
let done = self.aggregator.add(segments);
|
||||
unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
|
||||
self.files(done);
|
||||
self.files(done, false);
|
||||
self.frontier.update();
|
||||
let t = std::time::Instant::now();
|
||||
let gone = self.aggregator.evict(self.frontier.closed_before(), self.idle, false);
|
||||
self.files(gone);
|
||||
self.files(gone, false);
|
||||
self.close(false);
|
||||
unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
|
||||
self.batch.next = next;
|
||||
@@ -1191,7 +1193,7 @@ impl RangeIndexer {
|
||||
pub fn finish(&mut self) {
|
||||
self.frontier.settle();
|
||||
let gone = self.aggregator.evict(None, 0, true);
|
||||
self.files(gone);
|
||||
self.files(gone, true);
|
||||
self.close(true);
|
||||
self.batch.next = self.end + 1;
|
||||
}
|
||||
@@ -1243,12 +1245,16 @@ impl RangeIndexer {
|
||||
out
|
||||
}
|
||||
|
||||
/// Finished files (complete, or the window closed on them) into the open release of their key and upload.
|
||||
fn files(&mut self, files: Vec<Vec<Seg>>) {
|
||||
/// Finished files (complete, or the window closed on them) into the open release of their key and upload; `at_end`
|
||||
/// when the block ends on them.
|
||||
fn files(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
|
||||
let batch = self.aggregator.batch;
|
||||
for file in files {
|
||||
if file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1) && is_random_file(&file[0].file.name) {
|
||||
// One article of a multi-part random file (its upload per article: ngPost's, nyuu's): not a release.
|
||||
let lone = file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1);
|
||||
// A scrambled name only before the block's end, where the rest of a file can still be in the next block.
|
||||
if lone && (is_random_file(&file[0].file.name) || (!at_end && looks_scrambled(&file[0].file.name))) {
|
||||
// One article of a multi-part random file (its upload per article: ngPost's, nyuu's, and a random poster's
|
||||
// per article, 160k in alt.binaries.storage; LOG.md (122)): not a release.
|
||||
let tool = names::upload_tool(&file[0].message_id);
|
||||
self.count("per-article", tool.to_string(), 1, 0);
|
||||
continue;
|
||||
@@ -1419,9 +1425,21 @@ impl RangeIndexer {
|
||||
fn write(&mut self, key: &str, upload: usize, list: Vec<Vec<Seg>>, finished: bool, edge: bool, album: Option<String>) {
|
||||
let judged: Vec<JudgedFile> = list
|
||||
.iter()
|
||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster })
|
||||
.map(|f| JudgedFile {
|
||||
name: &f[0].file.name,
|
||||
total_parts: f[0].file.total_parts,
|
||||
groups: &f[0].file.groups,
|
||||
first_message_id: &f[0].message_id,
|
||||
poster: &f[0].file.poster,
|
||||
segments: f.len() as u32,
|
||||
bytes: f.iter().map(|s| s.bytes.max(0) as u64).sum(),
|
||||
})
|
||||
.collect();
|
||||
match names::judge(key, &judged, &self.rules) {
|
||||
// Settled for the single-archive rules: the window closed on it (a par2 can still join an open one). The
|
||||
// start edge (posted within the window of the block's start) doesn't count: a par2 comes after its rar, so
|
||||
// one in the previous block is rare (0 par2 siblings of 3,000 catalog hits, LOG.md (125)), and 7% of the
|
||||
// catalog's releases are start edges, which would escape the rule.
|
||||
match names::judge(key, &judged, finished, &self.rules) {
|
||||
Verdict::Keep => {
|
||||
let candidates = candidates_of(key, &list, upload);
|
||||
let mut chunk = chunk_of(key, &list, upload);
|
||||
@@ -1435,7 +1453,16 @@ impl RangeIndexer {
|
||||
}
|
||||
if !chunk.readable {
|
||||
// Unnamed: no chunk (its segments are read again once a probe names it), the tool for its signature.
|
||||
chunk.tool = Some(names::upload_tool(&list[0][0].message_id));
|
||||
let tool = names::upload_tool(&list[0][0].message_id);
|
||||
// A class the ladder found never names anything stays out of the pending store, its explored
|
||||
// sample excepted (so the rate keeps being measured); counted as garbage (LOG.md (130)).
|
||||
let klass = names::pending_class(&chunk.name, chunk.bytes.max(0) as u64, chunk.partial);
|
||||
if self.rules.dropped_classes.contains(&names::class_key(tool, &self.group, klass)) && !names::explored(key, self.rules.sample_percent) {
|
||||
let segments: usize = list.iter().map(|f| f.len()).sum();
|
||||
self.count("garbage", format!("{tool} {} #{klass}", self.group), segments as u64, list.len() as u64);
|
||||
return;
|
||||
}
|
||||
chunk.tool = Some(tool);
|
||||
chunk.payload = encode_chunk(&[]);
|
||||
}
|
||||
if chunk.readable {
|
||||
@@ -1544,7 +1571,7 @@ fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
||||
bytes,
|
||||
first_article,
|
||||
last_article,
|
||||
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
||||
readable: chunk_files.iter().any(|f| !is_unreadable_name(f.name)),
|
||||
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
||||
finished: true,
|
||||
edge: false,
|
||||
@@ -1558,7 +1585,7 @@ fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
||||
fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate> {
|
||||
let mut out = Vec::new();
|
||||
let mut tail: Option<(i64, &Vec<Seg>)> = None;
|
||||
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
||||
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone(), bytes: s.bytes };
|
||||
for f in files {
|
||||
let name = &f[0].file.name;
|
||||
let volume = names::seven_zip_volume(name);
|
||||
@@ -1579,10 +1606,15 @@ fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate>
|
||||
}
|
||||
}
|
||||
if let Some((_, f)) = tail {
|
||||
out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
|
||||
out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone(), bytes: f.last().unwrap().bytes });
|
||||
}
|
||||
// No set and no archive: the first random-named file's first segment, for its yEnc name or a twin by content
|
||||
// (hash_twins.ts). Any count of files: a hash-named release of several pieces (Bob's, LOG.md (140)) had no
|
||||
// target at all, so the ladder never probed it, never named it and never learned its class.
|
||||
if out.is_empty() {
|
||||
if let Some(f) = files.iter().find(|f| is_random_file(&f[0].file.name)) {
|
||||
out.push(c("lone", &f[0]));
|
||||
}
|
||||
if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
|
||||
out.push(c("lone", &files[0][0]));
|
||||
}
|
||||
out.truncate(4);
|
||||
out
|
||||
@@ -1711,6 +1743,30 @@ mod tests {
|
||||
assert_eq!(ix.pending(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_multi_file_random_release_gets_a_lone_target() {
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(GROUP.to_string(), 1);
|
||||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||||
// Three hash-named pieces of one hash-named upload, five segments each (more than a per-article post): no
|
||||
// archive, no par2, no 7z volume.
|
||||
let mut lines = Vec::new();
|
||||
for (i, name) in ["9e7fc2741cdd450287aff2733ca4c193", "c5ebdd5487f5b132c76e8d5c2572e560", "8114c42c9694380c8b25857e8b3331c3"].iter().enumerate() {
|
||||
for part in 0..5 {
|
||||
lines.push(line(100 + (i * 5 + part) as i64, &format!(r#"b3a75d234aa73891e78a70fc1f5496b1 [{}/3] - "{name}" yEnc ({}/5)"#, i + 1, part + 1), (i * 5 + part) as i64));
|
||||
}
|
||||
}
|
||||
ix.add_text(&lines.join("\n"), 10_001);
|
||||
ix.finish();
|
||||
let batch = ix.take();
|
||||
assert_eq!(batch.releases.len(), 1, "counts {:?}", batch.counts);
|
||||
let r = &batch.releases[0];
|
||||
assert!(!r.readable && r.files == 3 && r.segments == 15, "readable {} files {} segments {}", r.readable, r.files, r.segments);
|
||||
let kinds: Vec<_> = batch.candidates.iter().map(|c| (c.kind, c.file.as_str())).collect();
|
||||
assert_eq!(kinds, vec![("lone", "9e7fc2741cdd450287aff2733ca4c193")]);
|
||||
assert_eq!(batch.candidates[0].message_id, "u100@test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn album_templates_and_names() {
|
||||
assert_eq!(album_template("1481439079025_014.jpg.data").as_deref(), Some("#_#.jpg.data"));
|
||||
|
||||
@@ -62,6 +62,8 @@ pub struct RulesJson {
|
||||
#[serde(default)]
|
||||
pub dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub dropped_classes: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub per_article_max_parts: Option<u32>,
|
||||
@@ -71,6 +73,12 @@ pub struct RulesJson {
|
||||
pub drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub dropped_posters: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub drop_stub_archives: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub drop_nonsense_pieces: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub drop_tiny_archives: Option<bool>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
@@ -79,6 +87,7 @@ impl RulesJson {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
rules.dropped_classes = self.dropped_classes.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
@@ -94,6 +103,15 @@ impl RulesJson {
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
if let Some(d) = self.drop_stub_archives {
|
||||
rules.drop_stub_archives = d;
|
||||
}
|
||||
if let Some(d) = self.drop_nonsense_pieces {
|
||||
rules.drop_nonsense_pieces = d;
|
||||
}
|
||||
if let Some(d) = self.drop_tiny_archives {
|
||||
rules.drop_tiny_archives = d;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
+565
-6
@@ -449,6 +449,43 @@ fn finish_remainder(s: &str) -> String {
|
||||
js::trim(&out).to_string()
|
||||
}
|
||||
|
||||
/// displayName: a subject name as shown, brackets the counters left unmatched removed ("… Facialized 002/477] - …"
|
||||
/// gave "… Facialized ]"). Only the name: the key keeps the remainder.
|
||||
pub fn display_name(remainder: &str) -> String {
|
||||
let b = remainder.as_bytes();
|
||||
if !b.iter().any(|c| matches!(c, b'[' | b']' | b'(' | b')')) {
|
||||
return remainder.to_string();
|
||||
}
|
||||
let mut drop = vec![false; b.len()];
|
||||
for (open, close) in [(b'[', b']'), (b'(', b')')] {
|
||||
let mut opens: Vec<usize> = Vec::new();
|
||||
for (i, &c) in b.iter().enumerate() {
|
||||
if c == open {
|
||||
opens.push(i);
|
||||
} else if c == close && opens.pop().is_none() {
|
||||
drop[i] = true;
|
||||
}
|
||||
}
|
||||
for i in opens {
|
||||
drop[i] = true;
|
||||
}
|
||||
}
|
||||
if !drop.contains(&true) {
|
||||
return remainder.to_string();
|
||||
}
|
||||
let kept: Vec<u8> = b.iter().zip(&drop).filter(|(_, &d)| !d).map(|(&c, _)| c).collect();
|
||||
let kept = String::from_utf8(kept).expect("only ASCII brackets removed");
|
||||
let mut out = String::with_capacity(kept.len());
|
||||
for c in kept.chars() {
|
||||
if c == ' ' && out.ends_with(' ') {
|
||||
continue;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
let name = js::trim(&out);
|
||||
if name.is_empty() { remainder.to_string() } else { name.to_string() }
|
||||
}
|
||||
|
||||
/// counterTotal: the subject's file counter total, first form found first.
|
||||
pub fn counter_total(subject: &str) -> Option<String> {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
@@ -841,11 +878,11 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||
if !image && test(&RANDOM_VOLUME, name) && stem.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-') && looks_random(&stripped) {
|
||||
parts.push(&stem);
|
||||
}
|
||||
return Release { key: hash(&parts), name: remainder, source: "subject", file_total };
|
||||
return Release { key: hash(&parts), name: display_name(&remainder), source: "subject", file_total };
|
||||
}
|
||||
if let Some(total) = post_total.filter(|&n| n != 0 && !tag) {
|
||||
if is_post_token(&remainder) {
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: display_name(&remainder), source: "post", file_total };
|
||||
}
|
||||
}
|
||||
// A scene music post's index files ("00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg") under a bare counter go
|
||||
@@ -1028,6 +1065,228 @@ pub fn is_random_file(name: &str) -> bool {
|
||||
looks_random(&file_stem(name))
|
||||
}
|
||||
|
||||
/// Readable or not (lib/release/readable.ts): whether a name says what a release is, read token by token, so
|
||||
/// separators, spaces and random "extensions" don't hide a scramble that looks_random's single-token shapes miss
|
||||
/// (LOG.md (129)). The readable flag; the drop rules keep looks_random, which is narrower.
|
||||
/// Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed words.
|
||||
static NOISE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)^(addon|samples?|proof|subs?|nfo|part[0-9]+|vol[0-9]+|description|repost|rep|extras?|bonus|cd[0-9]*|dvd[0-9]*|dis[ck][0-9]*|par2|rar|zip|7z|mkv|mp4|avi|vol)$"));
|
||||
/// Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th.
|
||||
static CODE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"^([sS][0-9]{1,2}[eE][0-9]{1,3}|[xh]26[45]|[A-Za-z]+[0-9]+([a-z]{1,3}|x(64|86))?|[0-9]+[A-Za-z]{1,3}|[0-9]+[A-Za-z]+[0-9]+)$"));
|
||||
/// The NZBSPLIT tools' hash between their markers is no part of the name.
|
||||
static NZBSPLIT_HASH: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)__NZBS[P]LIT__[0-9a-f]{32}__NZBS[P]LIT__")); // "SP" is re()'s space class
|
||||
/// An inner run of 2+ capitals with lowercase on both sides ("tBWTuz", "YDoaROv"; not "3DMark03").
|
||||
static INNER_UPPER: LazyLock<Regex> = LazyLock::new(|| re(r"[a-z][A-Z]{2,}[a-z]"));
|
||||
|
||||
/// Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there.
|
||||
fn scene_case(t: &[u8]) -> Vec<u8> {
|
||||
let mut out = t.to_vec();
|
||||
for i in 1..t.len().saturating_sub(1) {
|
||||
if t[i] == b'i' && t[i - 1].is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
|
||||
out[i] = b'I';
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The signals of a scrambled token (readable.ts signals()).
|
||||
struct TokenSignals {
|
||||
/// Letter/digit and lower-to-upper switches.
|
||||
switches: usize,
|
||||
/// Case switches in both directions.
|
||||
case_switches: usize,
|
||||
/// Lone lowercase letters (not "i") between capitals ("NtpUgCmS").
|
||||
lone: usize,
|
||||
inner_upper: bool,
|
||||
max_consonants: usize,
|
||||
/// Vowels, a lowercase y too ("McCartney").
|
||||
vowels: usize,
|
||||
/// Runs of one character class.
|
||||
runs: usize,
|
||||
letters: usize,
|
||||
/// The longest lowercase run.
|
||||
max_lower: usize,
|
||||
}
|
||||
|
||||
fn token_signals(t: &[u8]) -> TokenSignals {
|
||||
let is_vowel = |c: u8| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u') || c == b'y';
|
||||
let class = |c: u8| if c.is_ascii_digit() { 0 } else if c.is_ascii_lowercase() { 1 } else { 2 };
|
||||
let mut s = TokenSignals { switches: 0, case_switches: 0, lone: 0, inner_upper: false, max_consonants: 0, vowels: 0, runs: 0, letters: 0, max_lower: 0 };
|
||||
let mut consonants = 0;
|
||||
let mut lower = 0;
|
||||
for i in 0..t.len() {
|
||||
let c = t[i];
|
||||
if c.is_ascii_alphabetic() && !is_vowel(c) && c.to_ascii_lowercase() != b'y' {
|
||||
consonants += 1;
|
||||
s.max_consonants = s.max_consonants.max(consonants);
|
||||
} else {
|
||||
consonants = 0;
|
||||
}
|
||||
if is_vowel(c) {
|
||||
s.vowels += 1;
|
||||
}
|
||||
if c.is_ascii_alphabetic() {
|
||||
s.letters += 1;
|
||||
}
|
||||
if c.is_ascii_lowercase() {
|
||||
lower += 1;
|
||||
s.max_lower = s.max_lower.max(lower);
|
||||
} else {
|
||||
lower = 0;
|
||||
}
|
||||
if i == 0 || class(t[i - 1]) != class(c) {
|
||||
s.runs += 1;
|
||||
}
|
||||
if i == 0 {
|
||||
continue;
|
||||
}
|
||||
let a = t[i - 1];
|
||||
if a.is_ascii_digit() != c.is_ascii_digit() {
|
||||
s.switches += 1;
|
||||
} else if a.is_ascii_lowercase() && c.is_ascii_uppercase() {
|
||||
s.switches += 1;
|
||||
}
|
||||
if (a.is_ascii_lowercase() && c.is_ascii_uppercase()) || (a.is_ascii_uppercase() && c.is_ascii_lowercase()) {
|
||||
s.case_switches += 1;
|
||||
}
|
||||
if i < t.len() - 1 && c.is_ascii_lowercase() && c != b'i' && a.is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
|
||||
s.lone += 1;
|
||||
}
|
||||
}
|
||||
s.inner_upper = INNER_UPPER.is_match(t);
|
||||
s
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
|
||||
pub enum TokenKind {
|
||||
Word,
|
||||
Random,
|
||||
Neutral,
|
||||
}
|
||||
|
||||
/// What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a
|
||||
/// packaging word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with readable.ts.
|
||||
pub fn token_kind(t: &[u8]) -> TokenKind {
|
||||
let n = t.len();
|
||||
if t.iter().all(u8::is_ascii_digit) || NOISE_TOKEN.is_match(t) || CODE_TOKEN.is_match(t) {
|
||||
return TokenKind::Neutral;
|
||||
}
|
||||
if t.iter().all(u8::is_ascii_alphabetic) {
|
||||
let s = token_signals(&scene_case(t));
|
||||
let capitalized = t[0].is_ascii_uppercase() && t[1..].iter().all(u8::is_ascii_lowercase);
|
||||
if s.case_switches == 0 || (s.case_switches == 1 && capitalized) {
|
||||
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
|
||||
if n >= 10 && s.vowels * 6 < n {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
return if n >= 3 { TokenKind::Word } else { TokenKind::Neutral };
|
||||
}
|
||||
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs.
|
||||
// Two CamelCase words with few vowels (FastFilms, BestCrypt, SchiffsModell) have four runs and no lone
|
||||
// letter; a scramble has more runs, a lone letter or an inner capital run.
|
||||
if n >= 6 && s.case_switches >= 2 && s.vowels * 4 < n && (s.runs >= 5 || s.lone >= 1 || s.inner_upper) {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
return TokenKind::Neutral;
|
||||
}
|
||||
if n >= 8 && t.iter().all(u8::is_ascii_hexdigit) && t.iter().filter(|c| c.is_ascii_digit()).count() >= 2 {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
if t.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) {
|
||||
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
|
||||
return if n >= 12 && token_signals(t).switches >= 6 { TokenKind::Random } else { TokenKind::Neutral };
|
||||
}
|
||||
// Words with numbers between them (Laura15Juli2026, Back4WinXP): every letter run a word or an acronym.
|
||||
if t.split(|c: &u8| !c.is_ascii_alphabetic()).filter(|w| !w.is_empty()).all(|w| {
|
||||
let vowels = w.iter().filter(|c| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u' | b'y')).count();
|
||||
(w.len() >= 2 && w.iter().all(u8::is_ascii_uppercase))
|
||||
|| (w.len() >= 2 && w[1..].iter().all(u8::is_ascii_lowercase) && (w.len() >= 3 || w[0].is_ascii_lowercase()) && vowels * 5 >= w.len())
|
||||
}) {
|
||||
return TokenKind::Neutral;
|
||||
}
|
||||
let s = token_signals(&scene_case(t));
|
||||
if n >= 5 && s.switches >= 3 {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
if n >= 6 && s.switches >= 2 && (s.lone >= 1 || s.inner_upper || s.max_consonants >= 4) {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
|
||||
if n >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.max_lower <= 2 {
|
||||
return TokenKind::Random;
|
||||
}
|
||||
TokenKind::Neutral
|
||||
}
|
||||
|
||||
/// A file or release name no one can read (readable.ts isUnreadableName): looks_random's shapes, or more scrambled
|
||||
/// tokens than words by letter count ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not
|
||||
/// "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p").
|
||||
pub fn is_unreadable_name(name: &str) -> bool {
|
||||
let stem = file_stem(name);
|
||||
let stem = NZBSPLIT_HASH.replace(stem.as_bytes(), &b" "[..]);
|
||||
if looks_random(&String::from_utf8_lossy(&stem)) {
|
||||
return true;
|
||||
}
|
||||
let mut words = 0;
|
||||
let mut random = 0;
|
||||
for t in stem.split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) {
|
||||
match token_kind(t) {
|
||||
TokenKind::Word => words += t.len(),
|
||||
TokenKind::Random => random += t.len(),
|
||||
TokenKind::Neutral => {}
|
||||
}
|
||||
}
|
||||
random > 0 && random > words
|
||||
}
|
||||
|
||||
/// The class of an unnamed release (lib/release/classes.ts pendingClass; LOG.md (130)): size bucket × name shape ×
|
||||
/// partial, 1..70. The lease's dropped classes ("tool\tgroup\tklass") are skipped at the edge.
|
||||
pub fn pending_class(name: &str, bytes: u64, partial: bool) -> u8 {
|
||||
let size = [1_000_000u64, 10_000_000, 100_000_000, 1_000_000_000].iter().filter(|&&b| bytes >= b).count() as u8;
|
||||
let s = match name.rfind('.') {
|
||||
Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
|
||||
_ => name,
|
||||
};
|
||||
let b = s.as_bytes();
|
||||
let all = |f: fn(&u8) -> bool| !b.is_empty() && b.iter().all(f);
|
||||
let shape: u8 = if b.len() >= 16 && all(u8::is_ascii_hexdigit) {
|
||||
1
|
||||
} else if b.len() >= 6 && all(u8::is_ascii_digit) {
|
||||
2
|
||||
} else if b.len() >= 20 && all(|c| c.is_ascii_lowercase() || (b'2'..=b'7').contains(c)) && b.iter().any(|c| (b'2'..=b'7').contains(c)) {
|
||||
3
|
||||
} else if all(u8::is_ascii_alphabetic) {
|
||||
4
|
||||
} else if all(u8::is_ascii_alphanumeric) {
|
||||
5
|
||||
} else if all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'-')) {
|
||||
6
|
||||
} else {
|
||||
7
|
||||
};
|
||||
size * 14 + (shape - 1) * 2 + u8::from(partial) + 1
|
||||
}
|
||||
|
||||
/// A class as droppedClasses lists it.
|
||||
pub fn class_key(tool: &str, group: &str, klass: u8) -> String {
|
||||
format!("{tool}\t{group}\t{klass}")
|
||||
}
|
||||
|
||||
/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
|
||||
/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
|
||||
/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
|
||||
pub fn looks_scrambled(name: &str) -> bool {
|
||||
let stem = file_stem(name);
|
||||
let (mut total, mut scrambled) = (0, 0);
|
||||
for token in stem.as_bytes().split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) {
|
||||
total += token.len();
|
||||
let switches = token.windows(2).filter(|w| (w[0].is_ascii_lowercase() && w[1].is_ascii_uppercase()) || w[0].is_ascii_digit() != w[1].is_ascii_digit()).count();
|
||||
if token.len() >= 5 && switches >= 3 {
|
||||
scrambled += token.len();
|
||||
}
|
||||
}
|
||||
total >= 8 && scrambled * 2 >= total
|
||||
}
|
||||
|
||||
static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
|
||||
static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||
@@ -1243,6 +1502,39 @@ pub fn is_sidecar(name: &str) -> bool {
|
||||
test(&SIDECAR, name)
|
||||
}
|
||||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||
/// An unsplit archive (the judge's ARCHIVE and VOLUME): one zip, rar or 7z that is not a volume of a set.
|
||||
static ARCHIVE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(zip|rar|7z)$"));
|
||||
/// "cneg" is "part" in ROT13: the a.b.erotica posts named that way are volumes of passworded videos (LOG.md (128)).
|
||||
static VOLUME: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.((part|cneg)[0-9]+\.rar|r[0-9][0-9]|z[0-9][0-9]|7z\.[0-9][0-9][0-9]|[0-9][0-9][0-9])$"));
|
||||
/// A poster at a free mail provider (the judge's WEBMAIL): the stub floods' throwaway accounts.
|
||||
static WEBMAIL: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\.")
|
||||
});
|
||||
/// A name that reads as a release (the judge's RELEASE_LIKE): a year, a scene tag, a group suffix or dotted words.
|
||||
static RELEASE_LIKE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
|
||||
});
|
||||
|
||||
/// RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way.
|
||||
static RELEASE_LIKE_SPACED: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
|
||||
});
|
||||
|
||||
/// The stub-archive rule (LOG.md (123)-(126), (128)): one unsplit archive, no par2, settled, posted to 2+ groups,
|
||||
/// under STUB_MAX_BYTES (the fakes stop at ~90 MB; the hash-named uploads, real albums and passworded videos above
|
||||
/// it don't), and its articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's
|
||||
/// 250 kB default; a one-article release always has small articles, so e-books and sidecars are exempt), or 4+
|
||||
/// groups, or a webmail poster.
|
||||
pub const STUB_ARTICLE_BYTES: u64 = 262_144;
|
||||
pub const STUB_MIN_BYTES: u64 = 1_000_000;
|
||||
pub const STUB_MAX_BYTES: u64 = 100_000_000;
|
||||
pub const STUB_WIDE_GROUPS: usize = 4;
|
||||
/// The nonsense-piece rule: one article holding one archive of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a
|
||||
/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts).
|
||||
pub const NONSENSE_MIN_BYTES: u64 = 100_000;
|
||||
pub const NONSENSE_MAX_BYTES: u64 = 1_000_000;
|
||||
/// The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars.
|
||||
pub const TINY_MIN_BYTES: u64 = 2_000;
|
||||
|
||||
pub struct JudgedFile<'a> {
|
||||
pub name: &'a str,
|
||||
@@ -1250,6 +1542,79 @@ pub struct JudgedFile<'a> {
|
||||
pub groups: &'a [String],
|
||||
pub first_message_id: &'a str,
|
||||
pub poster: &'a str,
|
||||
/// Articles seen and their bytes (the overview's :bytes).
|
||||
pub segments: u32,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
/// One unsplit archive and nothing else.
|
||||
fn single_archive<'a, 'b>(files: &'b [JudgedFile<'a>]) -> Option<&'b JudgedFile<'a>> {
|
||||
match files {
|
||||
[f] if test(&ARCHIVE, f.name) && !test(&VOLUME, f.name) => Some(f),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way
|
||||
/// the stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the
|
||||
/// stub-archive rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)).
|
||||
static STUB_EXEMPT_GROUPS: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)"));
|
||||
|
||||
/// The legs of the stub-archive rule a settled single archive under STUB_MAX_BYTES trips ("A" article size, "X" 4+
|
||||
/// groups, "W" webmail), empty when none.
|
||||
pub fn stub_archive_legs(f: &JudgedFile) -> String {
|
||||
let mut legs = String::new();
|
||||
if f.groups.len() < 2 || f.bytes >= STUB_MAX_BYTES || f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) {
|
||||
return legs;
|
||||
}
|
||||
let article = f.bytes / u64::from(f.segments.max(1));
|
||||
if article <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES {
|
||||
legs.push('A');
|
||||
}
|
||||
if f.groups.len() >= STUB_WIDE_GROUPS {
|
||||
legs.push('X');
|
||||
}
|
||||
if test(&WEBMAIL, f.poster) {
|
||||
legs.push('W');
|
||||
}
|
||||
legs
|
||||
}
|
||||
|
||||
/// An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece.
|
||||
static EBOOK: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b"));
|
||||
|
||||
/// A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the
|
||||
/// newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces).
|
||||
pub fn is_nonsense_piece(f: &JudgedFile) -> bool {
|
||||
f.total_parts == 1
|
||||
&& f.segments == 1
|
||||
&& (NONSENSE_MIN_BYTES..NONSENSE_MAX_BYTES).contains(&f.bytes)
|
||||
&& !f.name.contains(' ')
|
||||
&& test(&RELEASE_LIKE, &file_stem(f.name))
|
||||
&& !test(&EBOOK, f.name)
|
||||
&& !looks_scrambled(f.name)
|
||||
&& !is_random_file(f.name)
|
||||
}
|
||||
|
||||
/// Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
|
||||
/// exe is the content), covers and sidecars.
|
||||
static TINY_CONTENT: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b")
|
||||
});
|
||||
|
||||
/// A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
|
||||
/// unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
|
||||
/// subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
|
||||
pub fn is_tiny_archive(f: &JudgedFile) -> bool {
|
||||
f.total_parts == 1
|
||||
&& f.segments == 1
|
||||
&& (TINY_MIN_BYTES..NONSENSE_MIN_BYTES).contains(&f.bytes)
|
||||
&& !f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g))
|
||||
&& test(&RELEASE_LIKE_SPACED, &file_stem(f.name))
|
||||
&& !test(&EBOOK, f.name)
|
||||
&& !test(&TINY_CONTENT, f.name)
|
||||
&& !looks_scrambled(f.name)
|
||||
&& !is_random_file(f.name)
|
||||
}
|
||||
|
||||
pub enum Verdict {
|
||||
@@ -1265,12 +1630,19 @@ pub struct Rules {
|
||||
pub priorities: rustc_hash::FxHashMap<String, u32>,
|
||||
/// "tool\tgroup"
|
||||
pub dropped_signatures: rustc_hash::FxHashSet<String>,
|
||||
/// "tool\tgroup\tklass" (pending_class): unnamed releases of these classes don't go to the pending store.
|
||||
pub dropped_classes: rustc_hash::FxHashSet<String>,
|
||||
pub dropped_tools: rustc_hash::FxHashSet<String>,
|
||||
pub per_article_max_parts: u32,
|
||||
pub sample_percent: f64,
|
||||
pub drop_executables: bool,
|
||||
/// Fragments of the poster lowercased without punctuation (droppedPosters).
|
||||
pub dropped_posters: Vec<String>,
|
||||
/// The stub-archive rule (dropStubArchives), the nonsense-piece rule (dropNonsensePieces) and the tiny-archive
|
||||
/// rule (dropTinyArchives).
|
||||
pub drop_stub_archives: bool,
|
||||
pub drop_nonsense_pieces: bool,
|
||||
pub drop_tiny_archives: bool,
|
||||
}
|
||||
|
||||
impl Rules {
|
||||
@@ -1280,11 +1652,15 @@ impl Rules {
|
||||
version: 1,
|
||||
priorities,
|
||||
dropped_signatures: Default::default(),
|
||||
dropped_classes: Default::default(),
|
||||
dropped_tools: ["ngPost", "hex32", "yuun", "JBinUp", "random"].iter().map(|t| t.to_string()).collect(),
|
||||
per_article_max_parts: 3,
|
||||
sample_percent: 1.0,
|
||||
drop_executables: true,
|
||||
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
|
||||
drop_stub_archives: true,
|
||||
drop_nonsense_pieces: true,
|
||||
drop_tiny_archives: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1294,8 +1670,9 @@ pub fn poster_words(poster: &str) -> String {
|
||||
poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
|
||||
}
|
||||
|
||||
/// RuleSet.judge.
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||||
/// RuleSet.judge. `settled`: the window closed on the release (a par2 can still join an open one), which the
|
||||
/// single-archive rules need.
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rules) -> Verdict {
|
||||
let sample = explored(release_key, rules.sample_percent);
|
||||
if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
|
||||
let words = poster_words(first.poster);
|
||||
@@ -1309,6 +1686,20 @@ pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict
|
||||
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||||
return Verdict::Drop { rule: "executable", key, sample };
|
||||
}
|
||||
if let (true, Some(f)) = (settled, single_archive(files)) {
|
||||
if rules.drop_stub_archives {
|
||||
let legs = stub_archive_legs(f);
|
||||
if !legs.is_empty() {
|
||||
return Verdict::Drop { rule: "stub-archive", key: legs, sample };
|
||||
}
|
||||
}
|
||||
if rules.drop_nonsense_pieces && is_nonsense_piece(f) {
|
||||
return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample };
|
||||
}
|
||||
if rules.drop_tiny_archives && is_tiny_archive(f) {
|
||||
return Verdict::Drop { rule: "tiny-archive", key: upload_tool(f.first_message_id).to_string(), sample };
|
||||
}
|
||||
}
|
||||
if files.iter().any(|f| !is_random_file(f.name)) {
|
||||
return Verdict::Keep;
|
||||
}
|
||||
@@ -1339,6 +1730,29 @@ mod tests {
|
||||
assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scrambled_names() {
|
||||
for n in ["nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa", "PNSVWSV-xLaNv6o5qS840e3lwSZyH", "Biq-YsXu8G33ktt"] {
|
||||
assert!(looks_scrambled(n), "{n}");
|
||||
}
|
||||
// Short ones of the spam too (8% of alt.binaries.storage's): too few switches to tell.
|
||||
for n in ["A1Monitor v6.2.2", "PDF2Word v1.3", "Some.Show.S01E01.1080p.WEB.part1.rar", "MacGyver.S01E01.mkv", "Jimmy and the Crawler (mobi)", "Q25m.PDUYtbA", "82WH.2iuTK"] {
|
||||
assert!(!looks_scrambled(n), "{n}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_names() {
|
||||
let porn4you = "webmaster@hdbangvidz.com (Porn4You)";
|
||||
let a = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 002/477] - 0178.jpg", "0178.jpg");
|
||||
let b = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 003/477] - 0179.jpg", "0179.jpg");
|
||||
assert_eq!(a.name, "Babe with Plump Bubble Butt Fucked and Facialized");
|
||||
assert_eq!(a.key, b.key);
|
||||
assert_eq!(display_name("[Group] Show (2020)"), "[Group] Show (2020)");
|
||||
assert_eq!(display_name("( Show [x] name"), "Show [x] name");
|
||||
assert_eq!(display_name("]"), "]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remainders() {
|
||||
assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release");
|
||||
@@ -1451,6 +1865,46 @@ mod tests {
|
||||
assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_classes() {
|
||||
use super::pending_class;
|
||||
// Size bucket × shape × partial: a 5 MB hex name whole, the same partial, a 500 MB scramble with separators.
|
||||
assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, false), 15);
|
||||
assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, true), 16);
|
||||
assert_eq!(pending_class("Zx5AjNaK-mh7CFg6Hf8uu", 500_000_000, false), 3 * 14 + 5 * 2 + 1);
|
||||
assert_eq!(pending_class("1650777167", 100, false), 3);
|
||||
assert_eq!(pending_class("mrrtmzrzmi4dgmrzmyytqmlbztczbx", 2_000_000_000, false), 4 * 14 + 2 * 2 + 1);
|
||||
assert_eq!(pending_class("PsNdnYiugfZ.rar", 20_000_000, false), 2 * 14 + 3 * 2 + 1);
|
||||
assert_eq!(pending_class("n5d1ueIf34X", 20_000_000, true), 2 * 14 + 4 * 2 + 2);
|
||||
assert_eq!(pending_class("sgkra vU7u2yuw9LDLCjDu04pOez", 999, false), 13);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unreadable_names() {
|
||||
use super::is_unreadable_name;
|
||||
// Scrambles with separators, spaces, a random "extension", a packaging word, a hex with an addon (LOG.md (129)).
|
||||
for n in [
|
||||
"Zx5AjNaK-mh7CFg6Hf8uu.part01.rar", "yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU", "PsNdnYiugfZ.rar", "phfplC9y07U2yrj", "n5d1ueIf34X",
|
||||
"tJqr1tq7xK", "Description - Lt5nHjlo3RiuBb3C.par2", "bd5b4a59f3aa87114dcf75c0-addon.rar", "mrrtmzrzmi4dgmrzmyytqmlbztczbx",
|
||||
"AA-NaR16302830434zxP32108309f3Ab", "_QBs01_MqZYnYwM_iAyRquhc-", "sgkra vU7u2yuw9LDLCjDu04pOez", "Jk5Pud.zQdNj", "7uKsD_NRgT-G_Uc_p8",
|
||||
"NtpUgCmS_Xa1B2Pk", "{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}", "IkYovjrtZrItjQ.mkv", "ff75ce3f16c746c18b2d78cf.6c52f",
|
||||
] {
|
||||
assert!(is_unreadable_name(n), "{n}");
|
||||
}
|
||||
// Real names: scene tags, CamelCase, codes, leet, an acronym, a hash beside words.
|
||||
for n in [
|
||||
"Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune", "Paul McCartney - Band on the Run",
|
||||
"ShesSoWild - Ester Ana 1080p", "MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS", "cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar",
|
||||
"Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing", "Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON",
|
||||
"Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr", "Eclipse_IDE_2023-12_R_win32_x86_64.zip", "PartsHPGL 1.06c", "Go4Customer Fax Machine v1.2b",
|
||||
"AdBeGone 1.2", "3DMark03 320", "COD4MW2REMASTERED", "Movie.2020.1080p-GRP [a3f9e2b1]", "EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps",
|
||||
"iPhone", "Some.Show.S01E01.1080p.mkv", "FastFilms 2.04", "BestCrypt NP v4.0", "EasyHTML 2.35 German", "SchiffsModell - September 2026",
|
||||
"CrossStitcher - Issue 440, October 2026", "Laura15Juli2026", "IHaveAWife - Isa Bella", "DickHDDaily - Santana Red", "1904-SmithsWork-Hasluck",
|
||||
] {
|
||||
assert!(!is_unreadable_name(n), "{n}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_names() {
|
||||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||||
@@ -1491,8 +1945,8 @@ mod poster_rule_tests {
|
||||
|
||||
fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
|
||||
let groups = vec!["alt.binaries.erotica".to_string()];
|
||||
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
|
||||
match judge("k", &files, rules) {
|
||||
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster, segments: 1, bytes: 200_000 }];
|
||||
match judge("k", &files, true, rules) {
|
||||
Verdict::Keep => None,
|
||||
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||
}
|
||||
@@ -1512,6 +1966,111 @@ mod poster_rule_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod single_archive_rule_tests {
|
||||
use super::{judge, JudgedFile, Rules, Verdict};
|
||||
|
||||
const NYUU: &str = "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu";
|
||||
|
||||
fn verdict(name: &str, poster: &str, groups: usize, segments: u32, bytes: u64, settled: bool, rules: &Rules) -> Option<(&'static str, String)> {
|
||||
let groups: Vec<String> = (0..groups).map(|i| format!("alt.binaries.g{i}")).collect();
|
||||
let files = [JudgedFile { name, total_parts: segments, groups: &groups, first_message_id: NYUU, poster, segments, bytes }];
|
||||
match judge("k", &files, settled, rules) {
|
||||
Verdict::Keep => None,
|
||||
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_archives_by_article_size_groups_or_webmail() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com (Jahnunsinos007)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "AW".to_string())));
|
||||
assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "Yenc@power-post.org (Yenc-PP-A&A)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "A".to_string())));
|
||||
// Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone).
|
||||
assert_eq!(verdict("Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar", "Yenc@power-post.org (Yenc-PP-A&A)", 5, 2, 319_000, true, &rules), Some(("stub-archive", "X".to_string())));
|
||||
// A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave.
|
||||
assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 400, 260_000_000, true, &rules), None, "over 100 MB without the article-size leg");
|
||||
assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 100, 65_000_000, true, &rules), Some(("stub-archive", "W".to_string())));
|
||||
// Settled only: a par2 can still join an open release.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, false, &rules), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_and_single_group_and_split_archives_are_kept() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// An e-book: one or two small articles always have small articles (LOG.md (126)).
|
||||
assert_eq!(verdict("Maeve Binchy - Tara Road (epub).rar", "kie@home.now (KiE)", 2, 2, 521_000, true, &rules), None);
|
||||
// One group: most real single rars.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 1, 48, 12_000_000, true, &rules), None);
|
||||
// Modern 700 kB articles, no webmail, two groups.
|
||||
assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "poster@example.org", 2, 100, 70_000_000, true, &rules), None);
|
||||
// A volume, not a single archive; a hash-named 2 GB upload from a gmail-looking poster.
|
||||
assert_eq!(verdict("Some.Movie.2014.part01.rar", "a@gmail.com", 2, 48, 12_000_000, true, &rules), None);
|
||||
assert_eq!(verdict("cbcd32359460df1fa9b7d01d8720.rar", "b648de0d1c0f7a9a77be273d@gmail.com", 2, 3163, 2_340_000_000, true, &rules), None);
|
||||
// ROT13-named volumes of passworded videos in a.b.erotica, 258 kB articles, two groups, 1-270 MB; and the
|
||||
// 100 MB cap on every leg (LOG.md (128)).
|
||||
assert_eq!(verdict("Ghful.17.12.07.Qnvvfl.Fgbar.cneg3.rar", "nospam@nospam.nl (EvH)", 2, 5, 1_147_000, true, &rules), None);
|
||||
assert_eq!(verdict("Ghful.16.07.05.Wbwb.Xvff.cneg1.rar", "nospam@nospam.nl (EvH)", 2, 1051, 271_000_000, true, &rules), None);
|
||||
assert_eq!(verdict("Tinasoft EasyCafe 2.2.14 by OmerSalaj.rar", "Seven7i@mail.com ({Seven7i})", 4, 463, 117_000_000, true, &rules), None);
|
||||
assert_eq!(verdict("Tipard DVD Ripper - 10 0 12 x86 x64 incl Patch.rar", "Seven7i@mail.com ({Seven7i})", 4, 244, 62_000_000, true, &rules), Some(("stub-archive", "AXW".to_string())));
|
||||
let off = Rules { drop_stub_archives: false, ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, true, &off), None);
|
||||
// A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)).
|
||||
let groups = vec!["alt.binaries.sounds.karaoke".to_string(), "alt.binaries.test".to_string()];
|
||||
let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl.zip", total_parts: 13, groups: &groups, first_message_id: NYUU, poster: "home@home.com (TexasDj)", segments: 13, bytes: 3_300_000 }];
|
||||
assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonsense_pieces() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// One 790 kB article under a release-like name: the newznzb.* pieces.
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
|
||||
// A 3 kB cue/log sidecar rar with a year in its name: under this rule's floor (the tiny-archive rule's outside the music groups).
|
||||
assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
|
||||
// A book (a year or a format in its name, written with spaces), a dotted one too: not a piece.
|
||||
assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None);
|
||||
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None);
|
||||
assert_eq!(verdict("Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar", "saggir@domain.com", 1, 1, 113_000, true, &rules), None);
|
||||
// Two articles, or no release-like name, or scrambled: not this rule.
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 2, 790_000, true, &rules), None);
|
||||
assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
|
||||
assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
|
||||
let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_archives() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// One article of 2-100 kB under a release name, dotted or with spaces: a RAR holding "<title>.exe" (LOG.md (131)).
|
||||
assert_eq!(verdict("Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar", "Yanc@power-post.org <Yanc@power-post.org>", 1, 1, 15_084, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
|
||||
assert_eq!(verdict("Cicero Magazin fuer politische Kultur Mai 05 2016.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 17_478, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
|
||||
// A scene name without a year, a tag or a group suffix: not release-like, so kept (the flood's porn names mostly carry one).
|
||||
assert_eq!(verdict("[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar", "Yanc@power-post.org", 2, 1, 23_449, true, &rules), None);
|
||||
assert_eq!(verdict("Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 7_869, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
|
||||
// Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
|
||||
assert_eq!(verdict("The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar", "henk@example.org", 1, 1, 38_000, true, &rules), None);
|
||||
assert_eq!(verdict("Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar", "poster@example.org", 1, 1, 60_000, true, &rules), None);
|
||||
assert_eq!(verdict("Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
|
||||
assert_eq!(verdict("Some Band - Album (2011) covers.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
|
||||
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 38_000, true, &rules), None);
|
||||
// A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's.
|
||||
let groups = vec!["alt.binaries.sounds.karaoke".to_string()];
|
||||
let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl (2017).zip", total_parts: 1, groups: &groups, first_message_id: NYUU, poster: "home@home.com", segments: 1, bytes: 60_000 }];
|
||||
assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
|
||||
assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
|
||||
assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
|
||||
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 2, 20_000, true, &rules), None);
|
||||
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 1_500, true, &rules), None);
|
||||
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 150_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
|
||||
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, false, &rules), None);
|
||||
let off = Rules { drop_tiny_archives: false, ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, true, &off), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod image_set_tests {
|
||||
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
||||
|
||||
+2
-1
@@ -44,6 +44,7 @@ struct WireCandidate<'a> {
|
||||
kind: &'a str,
|
||||
message_id: &'a str,
|
||||
file: &'a str,
|
||||
bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -97,7 +98,7 @@ fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRel
|
||||
}
|
||||
|
||||
fn candidate(c: &Candidate) -> WireCandidate<'_> {
|
||||
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
|
||||
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file, bytes: c.bytes }
|
||||
}
|
||||
|
||||
/// toWire(batch) as JSON bytes.
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
|
||||
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
|
||||
export { toWire, fromWire, type WireBatch } from './wire.js'
|
||||
export type { Lease, GroupLease, NextRange } from './lease.js'
|
||||
export type { Lease, GroupLease, Lane, NextRange } from './lease.js'
|
||||
export type { EdgeSegment, FileMeta } from './scan.js'
|
||||
export { throttle } from './throttle.js'
|
||||
export { filterOverviews, undelta, type ChunkDrops } from './corpus_filter.js'
|
||||
+11
-2
@@ -7,11 +7,20 @@ export interface Lease {
|
||||
}
|
||||
|
||||
/**
|
||||
* A group lease (REBUILD.md, "Voyager 3: window indexing", 1): the group a worker thread holds, the rules, and the
|
||||
* snapshot of the group's window when there is one (its next article; the body comes from GET groups/:id/snapshot).
|
||||
* A group's two cursors (REBUILD.md, "Voyager 3: window indexing", 1): `history` works the ranges the lease bounds
|
||||
* hand out (the windows, the history), `tail` the tail planner's ranges. Each lane has its own holder, window and
|
||||
* snapshot; the group routes take `?lane=` (history by default).
|
||||
*/
|
||||
export type Lane = 'history' | 'tail'
|
||||
|
||||
/**
|
||||
* A group lease (REBUILD.md, "Voyager 3: window indexing", 1): the group a worker thread holds, the lane, the rules,
|
||||
* and the snapshot of the lane's window when there is one (its next article; the body comes from GET
|
||||
* groups/:id/snapshot).
|
||||
*/
|
||||
export interface GroupLease {
|
||||
group: { id: number; name: string; priority: number }
|
||||
lane: Lane
|
||||
rules: Rules
|
||||
snapshot: { next: number } | null
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { hash } from 'node:crypto'
|
||||
import type { Overview } from '../nntp/index.js'
|
||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||
import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
|
||||
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
|
||||
import { classKey, isUnreadableName, pendingClass } from '../release/index.js'
|
||||
import { explored } from '../rules/upload_tools.js'
|
||||
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
|
||||
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
||||
|
||||
@@ -58,11 +60,13 @@ export interface ProbeCandidate {
|
||||
kind: CandidateKind
|
||||
messageId: string
|
||||
file: string
|
||||
/** The target article's bytes: the cheapest probe first within a class (LOG.md (130)). */
|
||||
bytes?: number
|
||||
}
|
||||
|
||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||
export interface RuleCount {
|
||||
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
||||
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive' | 'incomplete'
|
||||
key: string
|
||||
articles: number
|
||||
files: number
|
||||
@@ -123,6 +127,11 @@ export interface RangeIndexerOptions {
|
||||
* chunks seen so far), as uploads in busy groups spread over minutes.
|
||||
*/
|
||||
edgeArticles?: number
|
||||
/**
|
||||
* A file given up short of parts is a file of its release all the same (partial), as the Rust edge's window writes
|
||||
* it, not a fragment: the pending namer reads such a release again (LOG.md 2026-10-29 (121)).
|
||||
*/
|
||||
partialFiles?: boolean
|
||||
}
|
||||
|
||||
const MAX_CANDIDATES = 4
|
||||
@@ -241,12 +250,34 @@ export class RangeIndexer {
|
||||
this.completed = new Map()
|
||||
for (const [key, all] of byRelease) {
|
||||
uploadsOf(all).forEach((list, upload) => {
|
||||
// Settled as far as this indexer knows: it judges a release's files as each batch completes them (the Rust
|
||||
// window passes whether the window closed on the release and no block edge cut it).
|
||||
const verdict = this.rules.judge(
|
||||
key,
|
||||
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster }))
|
||||
list.map((f) => ({
|
||||
name: f.segments[0].file.name,
|
||||
totalParts: f.segments[0].totalParts,
|
||||
groups: f.segments[0].file.groups,
|
||||
firstMessageId: f.segments[0].messageId,
|
||||
poster: f.segments[0].file.poster,
|
||||
segments: f.segments.length,
|
||||
bytes: f.segments.reduce((n, s) => n + s.bytes, 0),
|
||||
})),
|
||||
true
|
||||
)
|
||||
if (verdict.rule === null) {
|
||||
this.batch.releases.push(chunkOf(key, list, upload))
|
||||
const chunk = chunkOf(key, list, upload)
|
||||
if (!chunk.readable) {
|
||||
// Unnamed: a class the ladder found never names anything stays out of the pending store, its explored
|
||||
// sample excepted (so the rate keeps being measured); counted as garbage (LOG.md (130)).
|
||||
const tool = uploadTool(list[0].segments[0].messageId)
|
||||
const klass = pendingClass(chunk.name, chunk.bytes, chunk.partial === true)
|
||||
if (this.rules.classes.has(classKey(tool, this.options.group, klass)) && !explored(key, this.options.rules.samplePercent)) {
|
||||
this.count('garbage', `${tool} ${this.options.group} #${klass}`, list.reduce((n, f) => n + f.segments.length, 0), list.length)
|
||||
return
|
||||
}
|
||||
}
|
||||
this.batch.releases.push(chunk)
|
||||
this.batch.candidates.push(...candidatesOf(key, list, upload))
|
||||
return
|
||||
}
|
||||
@@ -274,6 +305,20 @@ export class RangeIndexer {
|
||||
else byFile.set(s.fileKey, [s])
|
||||
}
|
||||
for (const list of byFile.values()) {
|
||||
if (this.options.partialFiles) {
|
||||
// As the Rust window's files(): one article of a multi-part random file is no release; a scrambled name only
|
||||
// before the end, where the rest of the file can still come.
|
||||
const name = list[0].file.name
|
||||
if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && (isRandomFile(name) || (!atEnd && isScrambledFile(name)))) {
|
||||
this.count('per-article', uploadTool(list[0].messageId), 1, 0)
|
||||
continue
|
||||
}
|
||||
// One segment per part (two copies of a file share its key here), the first article seen.
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const s of list.sort((a, b) => a.articleNumber - b.articleNumber)) if (!parts.has(s.number)) parts.set(s.number, s)
|
||||
this.files([{ fileKey: list[0].fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }])
|
||||
continue
|
||||
}
|
||||
// One article of a "file" of many parts under a random name: a random subject per article, which
|
||||
// never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09).
|
||||
if (list.length === 1 && isRandomFile(list[0].file.name)) {
|
||||
@@ -428,8 +473,9 @@ export function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[], uploa
|
||||
bytes,
|
||||
firstArticle,
|
||||
lastArticle,
|
||||
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
|
||||
readable: chunkFiles.some((f) => !isUnreadableName(f.name)),
|
||||
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
||||
partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
|
||||
payload: encodeChunk(chunkFiles),
|
||||
}
|
||||
}
|
||||
@@ -451,12 +497,12 @@ export function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[],
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if (volume === 0 && f.segments.length > 1 && !tail) tail = { volume, file: f }
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name })
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name, bytes: f.segments[0].bytes })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name, bytes: f.segments[0].bytes })
|
||||
}
|
||||
if (tail) out.push({ releaseKey: key, upload, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name })
|
||||
if (tail) out.push({ releaseKey: key, upload, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name, bytes: tail.file.segments.at(-1)!.bytes })
|
||||
if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].file.name)) {
|
||||
out.push({ releaseKey: key, upload, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name })
|
||||
out.push({ releaseKey: key, upload, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name, bytes: files[0].segments[0].bytes })
|
||||
}
|
||||
return out.slice(0, MAX_CANDIDATES)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { parseRarHeaders } from '#lib/rar/index'
|
||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||
import { parseZipStart } from '#lib/zip/index'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
import { fileStem, looksRandom } from '#lib/release/index'
|
||||
import { isUnreadableName } from '#lib/release/index'
|
||||
|
||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||
export interface Inspection {
|
||||
@@ -139,7 +139,7 @@ export function usableName(inspection: Inspection, kind: string): string | null
|
||||
const usable = (n: string | undefined) => {
|
||||
if (!n) return null
|
||||
const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null
|
||||
return name && !PACKAGING.test(name) && !isUnreadableName(name) ? name : null
|
||||
}
|
||||
const named = (inspection.names ?? []).map(usable).find((n) => n)
|
||||
if (named) return named
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* The class of an unnamed release (REBUILD.md decision f; LOG.md (130)): what the headers say about its chance of a
|
||||
* name, from the pending ladder's own record. Size bucket × name shape × partial, 1..70, kept in step with
|
||||
* names.rs pending_class and the SQL pending_class() (migration 6800): the batch store sets it with the SQL, the
|
||||
* edge skips the classes a lease marks dropped.
|
||||
*/
|
||||
|
||||
/** Size buckets: under 1 MB, 10 MB, 100 MB, 1 GB, and from 1 GB. */
|
||||
export const SIZE_BUCKETS = [1_000_000, 10_000_000, 100_000_000, 1_000_000_000]
|
||||
/** Name shapes, 1..7: hex, digits, base32, letters only, mixed letters and digits, with separators, other. */
|
||||
export const SHAPES = [
|
||||
'hex',
|
||||
'digits',
|
||||
'base32',
|
||||
'letters',
|
||||
'alnum',
|
||||
'separators',
|
||||
'other',
|
||||
] as const
|
||||
|
||||
export function sizeBucket(bytes: number): number {
|
||||
let i = 0
|
||||
while (i < SIZE_BUCKETS.length && bytes >= SIZE_BUCKETS[i]) i++
|
||||
return i
|
||||
}
|
||||
|
||||
export function nameShape(name: string): number {
|
||||
const s = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
if (/^[0-9a-fA-F]{16,}$/.test(s)) return 1
|
||||
if (/^[0-9]{6,}$/.test(s)) return 2
|
||||
if (/^[a-z2-7]{20,}$/.test(s) && /[2-7]/.test(s)) return 3
|
||||
if (/^[A-Za-z]+$/.test(s)) return 4
|
||||
if (/^[A-Za-z0-9]+$/.test(s)) return 5
|
||||
if (/^[A-Za-z0-9._-]+$/.test(s)) return 6
|
||||
return 7
|
||||
}
|
||||
|
||||
/** 1..70: (size bucket × 7 + shape − 1) × 2 + partial + 1. */
|
||||
export function pendingClass(name: string, bytes: number, partial: boolean): number {
|
||||
return sizeBucket(bytes) * 14 + (nameShape(name) - 1) * 2 + (partial ? 1 : 0) + 1
|
||||
}
|
||||
|
||||
/** A class as droppedClasses lists it: "tool\tgroup\tklass". */
|
||||
export function classKey(tool: string, group: string, klass: number): string {
|
||||
return `${tool}\t${group}\t${klass}`
|
||||
}
|
||||
|
||||
/** A class in words ("10-100M hex partial"). */
|
||||
export function describeClass(klass: number): string {
|
||||
const k = klass - 1
|
||||
const size = Math.floor(k / 14)
|
||||
const shape = SHAPES[Math.floor((k % 14) / 2)]
|
||||
const sizes = ['<1M', '1-10M', '10-100M', '100M-1G', '1G+']
|
||||
return `${sizes[size]} ${shape}${k % 2 ? ' partial' : ''}`
|
||||
}
|
||||
+25
-2
@@ -105,6 +105,29 @@ export function subjectRemainder(subject: string): string {
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* A subject name as shown: brackets the counters left unmatched removed ("… Facialized 002/477] - 0177.jpg", the
|
||||
* counter's "[" missing, gave "… Facialized ]"; LOG.md 2026-10-29 (121)). Only the name: the key keeps the remainder.
|
||||
*/
|
||||
export function displayName(remainder: string): string {
|
||||
if (!/[[\]()]/.test(remainder)) return remainder
|
||||
const drop = new Set<number>()
|
||||
for (const [open, close] of [['[', ']'], ['(', ')']]) {
|
||||
const opens: number[] = []
|
||||
for (let i = 0; i < remainder.length; i++) {
|
||||
if (remainder[i] === open) opens.push(i)
|
||||
else if (remainder[i] === close) {
|
||||
if (opens.length > 0) opens.pop()
|
||||
else drop.add(i)
|
||||
}
|
||||
}
|
||||
for (const i of opens) drop.add(i)
|
||||
}
|
||||
if (drop.size === 0) return remainder
|
||||
const name = remainder.split('').filter((_, i) => !drop.has(i)).join('').replace(/ {2,}/g, ' ').trim()
|
||||
return name || remainder
|
||||
}
|
||||
|
||||
/**
|
||||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||||
@@ -448,7 +471,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
if (!image && RANDOM_VOLUME.test(input.name) && /^[A-Za-z0-9_-]+$/.test(stem) && looksRandom(stem.replace(/[-_]/g, ''))) parts.push(stem)
|
||||
return {
|
||||
key: hash(...parts),
|
||||
name: remainder,
|
||||
name: displayName(remainder),
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
}
|
||||
@@ -456,7 +479,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||||
// [i/M] counter it groups the files of one post whose names are random.
|
||||
if (!tag && postTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', poster, remainder, postTotal), name: remainder, fileTotal, source: 'post' }
|
||||
return { key: hash('post', poster, remainder, postTotal), name: displayName(remainder), fileTotal, source: 'post' }
|
||||
}
|
||||
// A scene music post's index files under a bare counter go with its tracks: the artist and the total ("[01/13] -
|
||||
// "00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg"" with "[05/13] - "01-ammar-turk_uit_de_kroeg.mp3"": the
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
/**
|
||||
* Grouping of files into releases (downloads), independent of AdonisJS.
|
||||
*/
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
|
||||
export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
|
||||
export { looksRandom } from './random.js'
|
||||
export { isUnreadableName, tokenKind } from './readable.js'
|
||||
export { pendingClass, classKey, describeClass, nameShape, sizeBucket } from './classes.js'
|
||||
@@ -0,0 +1,148 @@
|
||||
import { fileStem } from './identity.js'
|
||||
import { looksRandom } from './random.js'
|
||||
|
||||
/**
|
||||
* Readable or not: whether a name says what a release is (the catalog's readable flag, the prober's inner names).
|
||||
* looksRandom (random.ts) knows the single-token shapes; this reads a name token by token, so separators, spaces
|
||||
* and random "extensions" don't hide a scramble (LOG.md (129)).
|
||||
*/
|
||||
/**
|
||||
* Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed
|
||||
* words ("Description - Lt5nHjlo3RiuBb3C", "bd5b4a59f3aa87114dcf75c0-addon").
|
||||
*/
|
||||
const NOISE =
|
||||
/^(addon|samples?|proof|subs?|nfo|part\d+|vol\d+|description|repost|rep|extras?|bonus|cd\d*|dvd\d*|dis[ck]\d*|par2|rar|zip|7z|mkv|mp4|avi|vol)$/i
|
||||
/** Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. */
|
||||
const CODE =
|
||||
/^([sS]\d{1,2}[eE]\d{1,3}|[xh]26[45]|[A-Za-z]+\d+([a-z]{1,3}|x(64|86))?|\d+[A-Za-z]{1,3}|\d+[A-Za-z]+\d+)$/
|
||||
/** The NZBSPLIT tools' hash between their markers is no part of the name. */
|
||||
const NZBSPLIT = /__NZBSPLIT__[0-9a-f]{32}__NZBSPLIT__/i
|
||||
|
||||
/** Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. */
|
||||
function sceneCase(t: string): string {
|
||||
let out = ''
|
||||
for (let i = 0; i < t.length; i++) {
|
||||
out +=
|
||||
t[i] === 'i' && i > 0 && i < t.length - 1 && /[A-Z]/.test(t[i - 1]) && /[A-Z]/.test(t[i + 1])
|
||||
? 'I'
|
||||
: t[i]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* The signals of a scrambled token: letter/digit and lower-to-upper switches; case switches in both directions; lone
|
||||
* lowercase letters between capitals ("NtpUgCmS"); an inner run of 2+ capitals with lowercase on both sides
|
||||
* ("tBWTuz", "YDoaROv"); the longest consonant run; vowels (a lowercase y too: "McCartney"); runs of one character class.
|
||||
*/
|
||||
function signals(t: string) {
|
||||
let switches = 0
|
||||
let caseSwitches = 0
|
||||
let lone = 0
|
||||
let consonants = 0
|
||||
let maxConsonants = 0
|
||||
for (let i = 0; i < t.length; i++) {
|
||||
const c = t[i]
|
||||
if (/[b-df-hj-np-tv-xz]/i.test(c)) maxConsonants = Math.max(maxConsonants, ++consonants)
|
||||
else consonants = 0
|
||||
if (i === 0) continue
|
||||
const a = t[i - 1]
|
||||
if (/\d/.test(a) !== /\d/.test(c)) switches++
|
||||
else if (/[a-z]/.test(a) && /[A-Z]/.test(c)) switches++
|
||||
if ((/[a-z]/.test(a) && /[A-Z]/.test(c)) || (/[A-Z]/.test(a) && /[a-z]/.test(c))) caseSwitches++
|
||||
if (
|
||||
i < t.length - 1 &&
|
||||
/[a-z]/.test(c) &&
|
||||
c !== 'i' &&
|
||||
/[A-Z]/.test(a) &&
|
||||
/[A-Z]/.test(t[i + 1])
|
||||
)
|
||||
lone++
|
||||
}
|
||||
const innerUpper = /[a-z][A-Z]{2,}[a-z]/.test(t)
|
||||
const vowels = (t.match(/[aeiouAEIOU]|y/g) ?? []).length
|
||||
const runs = (t.match(/[a-z]+|[A-Z]+|\d+/g) ?? []).length
|
||||
const letters = (t.match(/[A-Za-z]/g) ?? []).length
|
||||
const maxLower = Math.max(0, ...(t.match(/[a-z]+/g) ?? []).map((r) => r.length))
|
||||
return {
|
||||
switches,
|
||||
caseSwitches,
|
||||
lone,
|
||||
innerUpper,
|
||||
maxConsonants,
|
||||
vowels,
|
||||
runs,
|
||||
letters,
|
||||
maxLower,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a packaging
|
||||
* word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with names.rs token_kind.
|
||||
*/
|
||||
export function tokenKind(t: string): 'word' | 'random' | null {
|
||||
if (/^\d+$/.test(t) || NOISE.test(t) || CODE.test(t)) return null
|
||||
if (/^[A-Za-z]+$/.test(t)) {
|
||||
const s = signals(sceneCase(t))
|
||||
if (s.caseSwitches === 0 || (s.caseSwitches === 1 && /^[A-Z][a-z]+$/.test(t))) {
|
||||
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
|
||||
if (t.length >= 10 && s.vowels * 6 < t.length) return 'random'
|
||||
return t.length >= 3 ? 'word' : null
|
||||
}
|
||||
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble. Two CamelCase words with few vowels
|
||||
// (FastFilms, BestCrypt, SchiffsModell) have four runs and no lone letter; a scramble has more runs, a lone
|
||||
// letter or an inner capital run.
|
||||
if (
|
||||
t.length >= 6 &&
|
||||
s.caseSwitches >= 2 &&
|
||||
s.vowels * 4 < t.length &&
|
||||
(s.runs >= 5 || s.lone >= 1 || s.innerUpper)
|
||||
)
|
||||
return 'random'
|
||||
return null
|
||||
}
|
||||
if (t.length >= 8 && /^[0-9a-f]+$/i.test(t) && (t.match(/\d/g)?.length ?? 0) >= 2) return 'random'
|
||||
if (/^[A-Z0-9]+$/.test(t)) {
|
||||
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
|
||||
return t.length >= 12 && signals(t).switches >= 6 ? 'random' : null
|
||||
}
|
||||
// Words with numbers between them (Laura15Juli2026, Back4WinXP): every letter run a word or an acronym.
|
||||
if (
|
||||
(t.match(/[A-Za-z]+/g) ?? []).every(
|
||||
(w) =>
|
||||
/^[A-Z]{2,}$/.test(w) ||
|
||||
(/^[A-Z]?[a-z]{2,}$/.test(w) && (w.match(/[aeiouy]/g) ?? []).length * 5 >= w.length)
|
||||
)
|
||||
)
|
||||
return null
|
||||
const s = signals(sceneCase(t))
|
||||
if (t.length >= 5 && s.switches >= 3) return 'random'
|
||||
if (t.length >= 6 && s.switches >= 2 && (s.lone >= 1 || s.innerUpper || s.maxConsonants >= 4))
|
||||
return 'random'
|
||||
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
|
||||
if (t.length >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.maxLower <= 2)
|
||||
return 'random'
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* A file or release name no one can read: looksRandom's shapes, or more scrambled tokens than words by letter count
|
||||
* ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not "Paul McCartney - Band on the Run",
|
||||
* "ShesSoWild - Ester Ana 1080p"). Separators and a random "extension" of any length don't hide a scramble, which
|
||||
* looksRandom's single-token shapes miss (15% of the readable catalog on 2026-11-02; LOG.md (129)). The readable
|
||||
* flag; the drop rules keep looksRandom, which is narrower.
|
||||
*/
|
||||
export function isUnreadableName(name: string): boolean {
|
||||
const stem = fileStem(name).replace(NZBSPLIT, ' ')
|
||||
if (looksRandom(stem)) return true
|
||||
let words = 0
|
||||
let random = 0
|
||||
for (const t of stem.split(/[^A-Za-z0-9]+/)) {
|
||||
if (!t) continue
|
||||
const kind = tokenKind(t)
|
||||
if (kind === 'word') words += t.length
|
||||
else if (kind === 'random') random += t.length
|
||||
}
|
||||
return random > 0 && random > words
|
||||
}
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
||||
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
|
||||
export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, isTinyArchive, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, TINY_MIN_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js'
|
||||
+174
-4
@@ -12,6 +12,11 @@ export interface Rules {
|
||||
priorities: Record<string, number>
|
||||
/** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */
|
||||
droppedSignatures: string[]
|
||||
/**
|
||||
* Dropped classes of unnamed releases, as "tool\tgroup\tklass" (lib/release/classes.ts; the pending ladder's record,
|
||||
* LOG.md (130)): the edge keeps them out of the pending store, the explored sample excepted.
|
||||
*/
|
||||
droppedClasses: string[]
|
||||
/** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */
|
||||
droppedTools: string[]
|
||||
/** A random-named file of at most this many parts: a random subject per article, needs bodies. */
|
||||
@@ -30,6 +35,24 @@ export interface Rules {
|
||||
* again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)).
|
||||
*/
|
||||
droppedPosters: string[]
|
||||
/**
|
||||
* Drop settled releases of one unsplit archive without a par2, posted to 2+ groups, under STUB_MAX_BYTES, whose
|
||||
* articles are at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES, or posted to 4+ groups, or by
|
||||
* a webmail poster: the exe-stub and recycled-installer floods under film and software names (55% of the catalog
|
||||
* on 2026-10-30; 99.3% of the known fakes, 0 known content; LOG.md (123)-(126), (128)).
|
||||
*/
|
||||
dropStubArchives: boolean
|
||||
/**
|
||||
* Drop one-article archives of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a release-like, unscrambled name:
|
||||
* the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)).
|
||||
*/
|
||||
dropNonsensePieces: boolean
|
||||
/**
|
||||
* Drop one-article archives of TINY_MIN_BYTES to NONSENSE_MIN_BYTES under a release-like name (spaces allowed),
|
||||
* outside the exempt groups, without an e-book, subtitle, crack or sidecar word: the 2015-17 exe-stub flood in
|
||||
* miniature (a RAR holding "<title>.exe" at 2-60 kB; 1.65M catalog rows; LOG.md (131), (133)).
|
||||
*/
|
||||
dropTinyArchives: boolean
|
||||
}
|
||||
|
||||
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||
@@ -39,9 +62,37 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||
samplePercent: 1,
|
||||
dropExecutables: true,
|
||||
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
|
||||
dropStubArchives: true,
|
||||
dropNonsensePieces: true,
|
||||
dropTinyArchives: true,
|
||||
droppedClasses: [],
|
||||
}
|
||||
|
||||
export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
||||
/** The stub-archive rule's bounds (edge/src/names.rs): Power-Post's 250 kB default article, the fakes' size span. */
|
||||
export const STUB_ARTICLE_BYTES = 262144
|
||||
export const STUB_MIN_BYTES = 1_000_000
|
||||
export const STUB_MAX_BYTES = 100_000_000
|
||||
export const STUB_WIDE_GROUPS = 4
|
||||
/** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */
|
||||
export const NONSENSE_MIN_BYTES = 100_000
|
||||
export const NONSENSE_MAX_BYTES = 1_000_000
|
||||
/** The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. */
|
||||
export const TINY_MIN_BYTES = 2_000
|
||||
|
||||
export type Verdict =
|
||||
| { rule: null }
|
||||
| { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive'; key: string; sample: boolean }
|
||||
|
||||
/** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */
|
||||
export interface JudgedFile {
|
||||
name: string
|
||||
totalParts: number
|
||||
groups: string[]
|
||||
firstMessageId: string
|
||||
poster: string
|
||||
segments: number
|
||||
bytes: number
|
||||
}
|
||||
|
||||
/** A poster as droppedPosters matches it: lowercased, letters and digits only. */
|
||||
export function posterWords(poster: string): string {
|
||||
@@ -51,11 +102,117 @@ export function posterWords(poster: string): string {
|
||||
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
||||
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
||||
const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i
|
||||
/** One zip, rar or 7z that is not a volume of a set. */
|
||||
const ARCHIVE = /\.(zip|rar|7z)$/i
|
||||
/** "cneg" is "part" in ROT13: the a.b.erotica posts named that way are volumes of passworded videos (LOG.md (128)). */
|
||||
const VOLUME = /\.((part|cneg)\d+\.rar|r\d\d|z\d\d|7z\.\d\d\d|\d\d\d)$/i
|
||||
/** A poster at a free mail provider: the stub floods' throwaway accounts. */
|
||||
const WEBMAIL = /@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\./i
|
||||
/** A name that reads as a release: a year, a scene tag, a group suffix or dotted words. */
|
||||
const RELEASE_LIKE = /(\b(19|20)\d\d\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s\d\de\d\d|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v\d+(\.\d+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)/i
|
||||
|
||||
/**
|
||||
* Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way the
|
||||
* stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the stub-archive
|
||||
* rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)).
|
||||
*/
|
||||
const STUB_EXEMPT_GROUPS = /(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)/i
|
||||
|
||||
/** The single unsplit archive a release consists of, or null. */
|
||||
function singleArchive(files: JudgedFile[]): JudgedFile | null {
|
||||
return files.length === 1 && ARCHIVE.test(files[0].name) && !VOLUME.test(files[0].name) ? files[0] : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The legs of the stub-archive rule a settled single archive under STUB_MAX_BYTES (the fakes stop at ~90 MB;
|
||||
* hash-named uploads, real albums and passworded videos above it don't) trips: "A" (articles <= STUB_ARTICLE_BYTES,
|
||||
* release >= STUB_MIN_BYTES: a one-article release always has small articles, so e-books and sidecars are exempt),
|
||||
* "X" (4+ groups) and "W" (webmail poster). Empty when none.
|
||||
*/
|
||||
export function stubArchiveLegs(f: JudgedFile): string {
|
||||
if (f.groups.length < 2 || f.bytes >= STUB_MAX_BYTES || f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g))) return ''
|
||||
let legs = ''
|
||||
if (f.bytes / Math.max(f.segments, 1) <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES) legs += 'A'
|
||||
if (f.groups.length >= STUB_WIDE_GROUPS) legs += 'X'
|
||||
if (WEBMAIL.test(f.poster)) legs += 'W'
|
||||
return legs
|
||||
}
|
||||
|
||||
/** An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece. */
|
||||
const EBOOK = /\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b/i
|
||||
|
||||
/**
|
||||
* A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the
|
||||
* newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces).
|
||||
*/
|
||||
export function isNonsensePiece(f: JudgedFile): boolean {
|
||||
return (
|
||||
f.totalParts === 1 &&
|
||||
f.segments === 1 &&
|
||||
f.bytes >= NONSENSE_MIN_BYTES &&
|
||||
f.bytes < NONSENSE_MAX_BYTES &&
|
||||
!f.name.includes(' ') &&
|
||||
RELEASE_LIKE.test(fileStem(f.name)) &&
|
||||
!EBOOK.test(f.name) &&
|
||||
!isScrambledFile(f.name) &&
|
||||
!isRandomFile(f.name)
|
||||
)
|
||||
}
|
||||
|
||||
/** RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. */
|
||||
const RELEASE_LIKE_SPACED = new RegExp(RELEASE_LIKE.source.replace('[._-]', '[._ -]'), 'i')
|
||||
|
||||
/**
|
||||
* Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
|
||||
* exe is the content), covers and sidecars.
|
||||
*/
|
||||
const TINY_CONTENT =
|
||||
/\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b/i
|
||||
|
||||
/**
|
||||
* A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
|
||||
* unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
|
||||
* subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
|
||||
*/
|
||||
export function isTinyArchive(f: JudgedFile): boolean {
|
||||
return (
|
||||
f.totalParts === 1 &&
|
||||
f.segments === 1 &&
|
||||
f.bytes >= TINY_MIN_BYTES &&
|
||||
f.bytes < NONSENSE_MIN_BYTES &&
|
||||
!f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g)) &&
|
||||
RELEASE_LIKE_SPACED.test(fileStem(f.name)) &&
|
||||
!EBOOK.test(f.name) &&
|
||||
!TINY_CONTENT.test(f.name) &&
|
||||
!isScrambledFile(f.name) &&
|
||||
!isRandomFile(f.name)
|
||||
)
|
||||
}
|
||||
|
||||
export function isRandomFile(name: string) {
|
||||
return looksRandom(fileStem(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or
|
||||
* between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a
|
||||
* random-subject-per-article upload gives each article that looksRandom misses for their separators.
|
||||
*/
|
||||
export function isScrambledFile(name: string) {
|
||||
let total = 0
|
||||
let scrambled = 0
|
||||
for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) {
|
||||
total += token.length
|
||||
let switches = 0
|
||||
for (let i = 1; i < token.length; i++) {
|
||||
const [a, b] = [token[i - 1], token[i]]
|
||||
if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++
|
||||
}
|
||||
if (token.length >= 5 && switches >= 3) scrambled += token.length
|
||||
}
|
||||
return total >= 8 && scrambled * 2 >= total
|
||||
}
|
||||
|
||||
/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
|
||||
export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
|
||||
let owner: string | null = null
|
||||
@@ -68,20 +225,24 @@ export function ownerOf(groups: string[], priorities: Record<string, number>): s
|
||||
|
||||
export class RuleSet {
|
||||
private signatures: Set<string>
|
||||
readonly classes: Set<string>
|
||||
private tools: Set<string>
|
||||
private posters: string[]
|
||||
constructor(readonly rules: Rules) {
|
||||
this.posters = rules.droppedPosters ?? []
|
||||
this.signatures = new Set(rules.droppedSignatures)
|
||||
this.classes = new Set(rules.droppedClasses ?? [])
|
||||
this.tools = new Set(rules.droppedTools)
|
||||
}
|
||||
|
||||
/**
|
||||
* A release's verdict from its files: dropped when its poster is; when its content is bare
|
||||
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
||||
* dropped, as per-article random subjects when every file has few parts.
|
||||
* executables; when it is a settled single archive that is a stub or a nonsense piece; else kept as a whole when
|
||||
* any file has a readable name; else dropped as garbage when its signature or tool is dropped, as per-article
|
||||
* random subjects when every file has few parts. `settled`: the window closed on the release (a par2 can still
|
||||
* join an open one), which the single-archive rules need.
|
||||
*/
|
||||
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict {
|
||||
judge(releaseKey: string, files: JudgedFile[], settled = true): Verdict {
|
||||
const sample = explored(releaseKey, this.rules.samplePercent)
|
||||
if (this.posters.length > 0 && files.length > 0) {
|
||||
const words = posterWords(files[0].poster)
|
||||
@@ -94,6 +255,15 @@ export class RuleSet {
|
||||
return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample }
|
||||
}
|
||||
}
|
||||
const archive = settled ? singleArchive(files) : null
|
||||
if (archive) {
|
||||
if (this.rules.dropStubArchives) {
|
||||
const legs = stubArchiveLegs(archive)
|
||||
if (legs !== '') return { rule: 'stub-archive', key: legs, sample }
|
||||
}
|
||||
if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample }
|
||||
if (this.rules.dropTinyArchives && isTinyArchive(archive)) return { rule: 'tiny-archive', key: uploadTool(archive.firstMessageId), sample }
|
||||
}
|
||||
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
|
||||
const first = files[0]
|
||||
const tool = uploadTool(first.firstMessageId)
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@
|
||||
<label class="search-box">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24"><circle cx="11" cy="11" r="7" /><path d="m20 20-3.5-3.5" /></svg>
|
||||
<input id="q" type="search" autocomplete="off" spellcheck="false"
|
||||
placeholder="Search release names…" aria-label="Search" />
|
||||
placeholder="Search release names… (^word: at a word's start)" aria-label="Search" />
|
||||
<kbd aria-hidden="true">/</kbd>
|
||||
</label>
|
||||
<label class="select">
|
||||
|
||||
@@ -77,6 +77,8 @@ export default await Env.create(new URL('../', import.meta.url), {
|
||||
SNAPSHOT_DIR: Env.schema.string.optional(),
|
||||
LEASE_MIN_PRIORITY: Env.schema.number.optional(),
|
||||
LEASE_MAX_PRIORITY: Env.schema.number.optional(),
|
||||
LEASE_CAPTURE_ONLY: Env.schema.boolean.optional(),
|
||||
STREAM_CHUNK_WAIT_MINUTES: Env.schema.number.optional(),
|
||||
PENDING_SHARE: Env.schema.number.optional(),
|
||||
PENDING_MIN_PROBES: Env.schema.number.optional(),
|
||||
PENDING_MAX_NAMED: Env.schema.number.optional(),
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import Node from '#models/node'
|
||||
import Spool from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { GROUP, overviewChunk, runEdge, startEdge, waitFor } from '#tests/e2e/helpers/edge'
|
||||
|
||||
/**
|
||||
* The Rust edge worker against the real node API, database, spool and corpus (REBUILD.md, "Voyager 3: window
|
||||
* indexing", 1 and 5): what broke in production on 2026-10-27 (LOG.md (118)) — a stream waiting on a chunk the
|
||||
* corpus fill stored, a failed stream, a restart with and without the snapshot, a takeover.
|
||||
*/
|
||||
test.group('Edge worker end to end', (group) => {
|
||||
const saved = {
|
||||
spool: coordinatorConfig.spool.dir,
|
||||
corpus: coordinatorConfig.corpus.dir,
|
||||
snapshots: coordinatorConfig.snapshotDir,
|
||||
chunk: coordinatorConfig.chunkArticles,
|
||||
wait: coordinatorConfig.streamChunkWaitMs,
|
||||
}
|
||||
let dir = ''
|
||||
group.each.setup(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'e2e-'))
|
||||
coordinatorConfig.spool.dir = join(dir, 'spool')
|
||||
coordinatorConfig.corpus.dir = join(dir, 'corpus')
|
||||
coordinatorConfig.snapshotDir = join(dir, 'snapshots')
|
||||
coordinatorConfig.chunkArticles = 100
|
||||
return testUtils.db().truncate()
|
||||
})
|
||||
group.each.teardown(async () => {
|
||||
Object.assign(coordinatorConfig.spool, { dir: saved.spool })
|
||||
coordinatorConfig.corpus.dir = saved.corpus
|
||||
coordinatorConfig.snapshotDir = saved.snapshots
|
||||
coordinatorConfig.chunkArticles = saved.chunk
|
||||
coordinatorConfig.streamChunkWaitMs = saved.wait
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
/** A group with one range of three chunks and a node with a token. */
|
||||
async function setup(opts: { capture?: boolean } = {}) {
|
||||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||
const [r] = await db
|
||||
.table('ranges')
|
||||
.insert({
|
||||
group_id: g.id,
|
||||
start_id: 1,
|
||||
end_id: 300,
|
||||
next_id: 1,
|
||||
priority: 0,
|
||||
capture: opts.capture ?? false,
|
||||
})
|
||||
.returning('id')
|
||||
const node = await Node.create({ name: 'e2e-node' })
|
||||
const token = (await Node.accessTokens.create(node)).value!.release()
|
||||
return { groupId: Number(g.id), rangeId: Number(r.id), nodeId: Number(node.id), token }
|
||||
}
|
||||
const range = async (id: number) => (await db.from('ranges').where('id', id).firstOrFail()) as any
|
||||
const lease = async (groupId: number) =>
|
||||
(await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', 'history')
|
||||
.first()) as any
|
||||
/** The release whose first article is `at` (a release's name isn't the file's: the identity strips it). */
|
||||
const releaseAt = async (at: number) =>
|
||||
(await db.from('releases').where('first_article', at).first()) as any
|
||||
const names = async () =>
|
||||
JSON.stringify(
|
||||
await db
|
||||
.from('releases')
|
||||
.select('name', 'first_article', 'last_article', 'segments', 'complete')
|
||||
.orderBy('first_article')
|
||||
)
|
||||
const has = (lines: string[], text: string) => lines.some((l) => l.includes(text))
|
||||
|
||||
test('a range streamed from the spool and the corpus, with a chunk the corpus fill stores while the stream waits', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const { groupId, rangeId, token } = await setup({ capture: true })
|
||||
await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
|
||||
await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
|
||||
const run = await startEdge(token)
|
||||
// The worker holds the group and streams: chunk 1 goes out, chunk 2 is waited for — the corpus fill stores it.
|
||||
await waitFor(
|
||||
'the group to be held',
|
||||
async () =>
|
||||
(await lease(groupId))?.range_id !== null && (await lease(groupId))?.range_id !== undefined
|
||||
)
|
||||
await new Promise((r) => setTimeout(r, 700))
|
||||
await Corpus.write(
|
||||
{ group: GROUP, start: 1, end: 300 },
|
||||
{ from: 101, to: 200 },
|
||||
overviewChunk(101, 200),
|
||||
3
|
||||
)
|
||||
const { code, lines } = { ...(await run.exit), lines: run.lines }
|
||||
assert.equal(code, 0, lines.join('\n'))
|
||||
assert.isTrue(has(lines, 'caught up after 1 ranges'), lines.join('\n'))
|
||||
assert.isTrue(has(lines, 'snapshot at 301'), lines.join('\n'))
|
||||
assert.include(await range(rangeId), { status: 'done' })
|
||||
assert.equal(Number((await range(rangeId)).headers), 300)
|
||||
const l = await lease(groupId)
|
||||
assert.isNull(l.node_id)
|
||||
assert.equal(Number(l.snapshot_next), 301)
|
||||
assert.equal(Number(l.ranges_done), 1)
|
||||
// The window closed on the early files (the frontier is a day past them); every closed release is settled.
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.select('name', 'settled_at', 'first_article', 'last_article', 'segments')
|
||||
assert.isAtLeast(releases.length, 10)
|
||||
assert.isTrue(releases.every((r) => r.settled_at !== null))
|
||||
const spanning = releases.find((r) => Number(r.first_article) === 100)
|
||||
assert.isDefined(spanning, await names())
|
||||
assert.deepEqual(
|
||||
[Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
|
||||
[100, 101, 2],
|
||||
'the file over the chunk boundary came through the carried window'
|
||||
)
|
||||
})
|
||||
|
||||
test('a stream that fails: the batch so far is posted, the window snapshotted, the group given back; the next run restores it', async ({
|
||||
assert,
|
||||
}) => {
|
||||
coordinatorConfig.streamChunkWaitMs = 1500
|
||||
const { groupId, rangeId, token } = await setup()
|
||||
await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
|
||||
const first = await runEdge(token)
|
||||
assert.equal(first.code, 0, first.lines.join('\n'))
|
||||
assert.isTrue(has(first.lines, 'the group goes back'), first.lines.join('\n'))
|
||||
assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
|
||||
let r = await range(rangeId)
|
||||
assert.include(r, { status: 'planned', node_id: null })
|
||||
assert.equal(
|
||||
Number(r.next_id),
|
||||
101,
|
||||
'the batch for chunk 1 was stored before the group went back'
|
||||
)
|
||||
let l = await lease(groupId)
|
||||
assert.isNull(l.node_id)
|
||||
assert.equal(Number(l.snapshot_next), 101)
|
||||
const before = await db.from('releases').count('* as n').first()
|
||||
assert.isAtLeast(Number(before!.n), 10)
|
||||
assert.isNull(await releaseAt(100), 'the file over the boundary is still open in the window')
|
||||
|
||||
// The next run: the window restored, the rest of the range streamed, the open file completed.
|
||||
coordinatorConfig.streamChunkWaitMs = saved.wait
|
||||
await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
|
||||
await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
|
||||
const second = await runEdge(token)
|
||||
assert.equal(second.code, 0, second.lines.join('\n'))
|
||||
assert.isTrue(
|
||||
has(second.lines, 'window restored from the snapshot at 101'),
|
||||
second.lines.join('\n')
|
||||
)
|
||||
r = await range(rangeId)
|
||||
assert.equal(r.status, 'done')
|
||||
assert.equal(Number(r.headers), 300)
|
||||
l = await lease(groupId)
|
||||
assert.equal(Number(l.snapshot_next), 301)
|
||||
const spanning = await releaseAt(100)
|
||||
assert.isNotNull(spanning, await names())
|
||||
assert.deepEqual(
|
||||
[Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
|
||||
[100, 101, 2]
|
||||
)
|
||||
})
|
||||
|
||||
test('a restart whose snapshot file is gone: the window starts empty and the file over the seam ends incomplete', async ({
|
||||
assert,
|
||||
}) => {
|
||||
coordinatorConfig.streamChunkWaitMs = 1500
|
||||
const { groupId, rangeId, token } = await setup()
|
||||
await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
|
||||
const first = await runEdge(token)
|
||||
assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
|
||||
await rm(GroupLeases.snapshotPath(groupId), { force: true })
|
||||
|
||||
coordinatorConfig.streamChunkWaitMs = saved.wait
|
||||
await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
|
||||
await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
|
||||
const second = await runEdge(token)
|
||||
assert.equal(second.code, 0, second.lines.join('\n'))
|
||||
assert.isTrue(
|
||||
has(second.lines, 'but there is none; the window starts empty'),
|
||||
second.lines.join('\n')
|
||||
)
|
||||
assert.equal((await range(rangeId)).status, 'done')
|
||||
// Part 1 was in the lost window: one seam. Part 2 alone is a file short of a part.
|
||||
assert.isNull(await releaseAt(100), await names())
|
||||
const half = await releaseAt(101)
|
||||
assert.isNotNull(half, await names())
|
||||
assert.deepEqual([Number(half.segments), half.complete], [1, false], await names())
|
||||
})
|
||||
|
||||
test("a takeover: the stale holder's next batch is refused, its snapshot too, and the group stays the new holder's", async ({
|
||||
assert,
|
||||
}) => {
|
||||
const { groupId, rangeId, token, nodeId } = await setup()
|
||||
await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
|
||||
const run = await startEdge(token)
|
||||
// Chunk 1 posted; the stream now waits for chunk 2.
|
||||
await waitFor('the first batch', async () => Number((await range(rangeId)).next_id) === 101)
|
||||
// The holder looks dead (its group and range heartbeats stale): another node takes the group over (no snapshot
|
||||
// yet: the range goes back to its resume point).
|
||||
const stale = new Date(Date.now() - 24 * 3600_000)
|
||||
await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: stale })
|
||||
await db.from('ranges').where('id', rangeId).update({ heartbeat_at: stale })
|
||||
const other = await Node.create({ name: 'e2e-other' })
|
||||
const taken = await GroupLeases.lease(Number(other.id))
|
||||
assert.equal(taken?.group.id, groupId)
|
||||
assert.isNull(taken?.snapshot)
|
||||
assert.include(await range(rangeId), { status: 'planned', node_id: null })
|
||||
// The old holder gets chunk 2 all the same, posts, and is refused.
|
||||
await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
|
||||
await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
|
||||
const { code } = await run.exit
|
||||
const lines = run.lines
|
||||
assert.equal(code, 0, lines.join('\n'))
|
||||
assert.isTrue(has(lines, 'snapshot failed'), lines.join('\n'))
|
||||
assert.isTrue(
|
||||
has(lines, 'not leased to this node') || has(lines, 'no longer this node'),
|
||||
lines.join('\n')
|
||||
)
|
||||
const l = await lease(groupId)
|
||||
assert.equal(Number(l.node_id), Number(other.id))
|
||||
assert.isNull(l.snapshot_next)
|
||||
const r = await range(rangeId)
|
||||
assert.equal(r.status, 'planned')
|
||||
assert.equal(Number(r.next_id), 101, 'nothing past the first batch was stored')
|
||||
assert.notEqual(Number(r.node_id), nodeId)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { access } from 'node:fs/promises'
|
||||
import { resolve } from 'node:path'
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import env from '#start/env'
|
||||
|
||||
export const GROUP = 'alt.binaries.e2e'
|
||||
|
||||
/** The edge worker binary: EDGE_NODE_BIN, else the release or debug build under edge/target. */
|
||||
export async function edgeBinary(): Promise<string> {
|
||||
const candidates = [
|
||||
process.env.EDGE_NODE_BIN,
|
||||
'edge/target/release/edge-node',
|
||||
'edge/target/debug/edge-node',
|
||||
]
|
||||
.filter((c): c is string => !!c)
|
||||
.map((c) => resolve(c))
|
||||
for (const path of candidates) {
|
||||
try {
|
||||
await access(path)
|
||||
return path
|
||||
} catch {}
|
||||
}
|
||||
throw new Error(
|
||||
`No edge-node binary (looked at ${candidates.join(', ')}): build it with "cargo build --release --manifest-path edge/Cargo.toml --bin edge-node", or set EDGE_NODE_BIN`
|
||||
)
|
||||
}
|
||||
|
||||
/** The test server's address for the worker (tests/bootstrap.ts starts it on HOST:PORT). */
|
||||
export function coordinatorUrl(): string {
|
||||
const host = env.get('HOST', 'localhost')
|
||||
return `http://${host === '0.0.0.0' ? '127.0.0.1' : host}:${env.get('PORT', 3333)}`
|
||||
}
|
||||
|
||||
export interface EdgeRun {
|
||||
exit: Promise<{ code: number | null; signal: string | null }>
|
||||
lines: string[]
|
||||
kill: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* The real edge worker against the test server: one thread on the history lane, one group, a batch per chunk
|
||||
* (100 articles in these tests), a snapshot after every range.
|
||||
*/
|
||||
export async function startEdge(token: string, args: string[] = []): Promise<EdgeRun> {
|
||||
const bin = await edgeBinary()
|
||||
// The test server first (tests/bootstrap.ts starts it for this suite): a worker that can't connect retries for
|
||||
// minutes before it says so.
|
||||
const probe = await fetch(`${coordinatorUrl()}/api/v1/node/lease`, { method: 'POST' }).catch(
|
||||
(e) => e as Error
|
||||
)
|
||||
if (probe instanceof Error || probe.status !== 401)
|
||||
throw new Error(
|
||||
`The test server doesn't answer at ${coordinatorUrl()}: ${probe instanceof Error ? probe.message : `HTTP ${probe.status}`}`
|
||||
)
|
||||
const child = spawn(
|
||||
bin,
|
||||
[
|
||||
'--threads',
|
||||
'1',
|
||||
'--tail-threads',
|
||||
'0',
|
||||
'--groups',
|
||||
'1',
|
||||
'--batch-articles',
|
||||
'100',
|
||||
'--snapshot-minutes',
|
||||
'0',
|
||||
...args,
|
||||
],
|
||||
{
|
||||
env: { ...process.env, COORDINATOR_URL: coordinatorUrl(), NODE_TOKEN: token },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}
|
||||
)
|
||||
const lines: string[] = []
|
||||
let rest = ''
|
||||
const collect = (chunk: Buffer) => {
|
||||
rest += chunk.toString('utf8')
|
||||
const parts = rest.split('\n')
|
||||
rest = parts.pop() ?? ''
|
||||
lines.push(...parts)
|
||||
}
|
||||
child.stdout.on('data', collect)
|
||||
child.stderr.on('data', collect)
|
||||
const exit = new Promise<{ code: number | null; signal: string | null }>((done) => {
|
||||
child.on('exit', (code, signal) => {
|
||||
if (rest) lines.push(rest)
|
||||
done({ code, signal })
|
||||
})
|
||||
})
|
||||
return { exit, lines, kill: () => child.kill('SIGKILL') }
|
||||
}
|
||||
|
||||
/** Runs the worker to its end (at most `timeoutMs`), its log lines and exit code. */
|
||||
export async function runEdge(token: string, args: string[] = [], timeoutMs = 60_000) {
|
||||
const run = await startEdge(token, args)
|
||||
const timer = setTimeout(() => run.kill(), timeoutMs)
|
||||
const result = await run.exit
|
||||
clearTimeout(timer)
|
||||
return { ...result, lines: run.lines }
|
||||
}
|
||||
|
||||
/** Polls `condition` every 100 ms until it holds or `timeoutMs` passed (then throws). */
|
||||
export async function waitFor(what: string, condition: () => Promise<boolean>, timeoutMs = 30_000) {
|
||||
const until = Date.now() + timeoutMs
|
||||
while (Date.now() < until) {
|
||||
if (await condition()) return
|
||||
await new Promise((r) => setTimeout(r, 100))
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${what}`)
|
||||
}
|
||||
|
||||
const BASE_MS = Date.UTC(2026, 9, 5, 10)
|
||||
|
||||
/** A file name without digits (a number sequence would make an album of a poster's files): letters from `k`. */
|
||||
export function nameOf(k: number): string {
|
||||
let s = ''
|
||||
let n = k
|
||||
do {
|
||||
s = String.fromCharCode(97 + (n % 26)) + s
|
||||
n = Math.floor(n / 26)
|
||||
} while (n > 0)
|
||||
return `Set.${s}.mkv`
|
||||
}
|
||||
|
||||
/**
|
||||
* The overview lines of articles `from`..`to`, one an hour from 2026-10-05: article 1 is a one-part file, and from 2
|
||||
* on each even article is part 1 of a two-part file whose part 2 is the next (odd) article — so a file spans every
|
||||
* chunk boundary at an even `to` (100 and 101, 200 and 201: the window has to carry it). Dates an hour apart make
|
||||
* the posting frontier move a day every 24 articles, so the window closes on files as it goes.
|
||||
*/
|
||||
export function overviewLines(from: number, to: number): string[] {
|
||||
const lines: string[] = []
|
||||
for (let n = from; n <= to; n++) {
|
||||
const date = new Date(BASE_MS + n * 3600_000).toUTCString().replace(' GMT', ' +0000')
|
||||
const [name, part] =
|
||||
n === 1 ? [nameOf(1), '(1/1)'] : n % 2 === 0 ? [nameOf(n), '(1/2)'] : [nameOf(n - 1), '(2/2)']
|
||||
lines.push(
|
||||
`${n}\t"${name}" yEnc ${part}\tposter <p@x>\t${date}\t<a${n}@e2e>\t\t700000\t5000\tXref: news.example ${GROUP}:${n}`
|
||||
)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
/** One XOVER chunk as the news server sends it (deflated). */
|
||||
export function overviewChunk(from: number, to: number): Buffer {
|
||||
return deflateSync(Buffer.from(overviewLines(from, to).join('\r\n') + '\r\n'))
|
||||
}
|
||||
@@ -431,10 +431,10 @@ test.group('BatchStore', (group) => {
|
||||
assert,
|
||||
}) => {
|
||||
await storeIn(GROUP, 1, [
|
||||
...upload(10, 'TsIgNBewuFbuwWy.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'),
|
||||
...upload(20, 'TsIgNBewuFbuwWy.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'),
|
||||
...upload(30, 'TsIgNBewuFbuwWy.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'),
|
||||
...upload(40, 'TsIgNBewuFbuwWy.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'),
|
||||
...upload(10, 'Some.Show.S01E04.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'),
|
||||
...upload(20, 'Some.Show.S01E04.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'),
|
||||
...upload(30, 'Some.Show.S01E04.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'),
|
||||
...upload(40, 'Some.Show.S01E04.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'),
|
||||
...upload(
|
||||
50,
|
||||
'Some.Show.S01E03.part01.rar',
|
||||
@@ -455,8 +455,8 @@ test.group('BatchStore', (group) => {
|
||||
candidates.map((c) => [c.kind, c.file]),
|
||||
[
|
||||
['archive', 'Some.Show.S01E03.part01.rar'],
|
||||
['archive', 'TsIgNBewuFbuwWy.7z.001'],
|
||||
['7z-tail', 'TsIgNBewuFbuwWy.7z.004'],
|
||||
['archive', 'Some.Show.S01E04.7z.001'],
|
||||
['7z-tail', 'Some.Show.S01E04.7z.004'],
|
||||
]
|
||||
)
|
||||
})
|
||||
@@ -538,6 +538,38 @@ test.group('BatchStore', (group) => {
|
||||
assert.include(await db.from('pending_releases').firstOrFail(), { files: 2, segments: '5' })
|
||||
})
|
||||
|
||||
test('a finished unnamed release without a probe target is exhausted at once, not waiting', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const { rangeId, nodeId } = await rangeIn(GROUP, 1)
|
||||
// Two hash-named pieces: the edge gives the first a lone target; without one the ladder could never claim it.
|
||||
const batch = batchOf(GROUP, 1, [
|
||||
...upload(
|
||||
10,
|
||||
'9e7fc2741cdd450287aff2733ca4c193',
|
||||
4,
|
||||
'b3a75d234aa73891e78a70fc1f5496b1 [1/2]',
|
||||
'2026-10-05T10:00:00Z'
|
||||
),
|
||||
...upload(
|
||||
20,
|
||||
'c5ebdd5487f5b132c76e8d5c2572e560',
|
||||
4,
|
||||
'b3a75d234aa73891e78a70fc1f5496b1 [2/2]',
|
||||
'2026-10-05T10:01:00Z'
|
||||
),
|
||||
])
|
||||
assert.lengthOf(batch.releases, 1)
|
||||
assert.isFalse(batch.releases[0].readable)
|
||||
const stored = await BatchStore.store(rangeId, nodeId, { ...batch, candidates: [] })
|
||||
assert.include(stored, { releases: 1, newReleases: 0, candidates: 0, pending: 1 })
|
||||
const pending = await db.from('pending_releases').firstOrFail()
|
||||
assert.include(pending, { files: 2, finished: true, status: 3 })
|
||||
assert.deepEqual(pending.candidates, [])
|
||||
const klass = await db.from('pending_classes').firstOrFail()
|
||||
assert.include(klass, { waiting: 0 })
|
||||
})
|
||||
|
||||
test("an album's members stay findable by their own names (release_texts kind member)", async ({
|
||||
assert,
|
||||
}) => {
|
||||
|
||||
@@ -42,7 +42,7 @@ test.group('Group leases', (g) => {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test('a lease takes the group with the most urgent planned range; its ranges come in article order; caught up it gives nothing', async ({
|
||||
test('a lease takes the group with the most urgent planned range of its lane; its ranges come in article order; caught up it gives nothing', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const a = await group('alt.binaries.gl.a', 1)
|
||||
@@ -56,33 +56,73 @@ test.group('Group leases', (g) => {
|
||||
const n1 = await node('gl-1')
|
||||
const n2 = await node('gl-2')
|
||||
|
||||
// The history lane: both groups have priority 0 work; a has the most planned articles (300 to 100).
|
||||
const first = await GroupLeases.lease(n1)
|
||||
assert.equal(first?.group.id, b, 'the tail first')
|
||||
assert.equal(first?.group.id, a, 'the most planned articles first')
|
||||
assert.equal(first?.lane, 'history')
|
||||
assert.isNull(first?.snapshot)
|
||||
// Held: the next node gets the other group; a third gets nothing.
|
||||
const second = await GroupLeases.lease(n2)
|
||||
assert.equal(second?.group.id, a)
|
||||
assert.equal(second?.group.id, b)
|
||||
assert.isNull(await GroupLeases.lease(await node('gl-3')))
|
||||
|
||||
// Ranges in article order, whatever their priority.
|
||||
assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b1, start: 1, end: 100, next: 1 })
|
||||
assert.include(await rangeRow(b1), { status: 'leased', node_id: n1 })
|
||||
// Ranges in article order, whatever their priority; the tail range isn't the history lane's.
|
||||
assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 101 })
|
||||
assert.include(await rangeRow(b2), { status: 'leased', node_id: n2 })
|
||||
// Asked again before finishing: the same range.
|
||||
assert.equal((await GroupLeases.next(b, n1))?.range.id, b1)
|
||||
await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
|
||||
assert.equal((await GroupLeases.next(b, n1))?.range.id, b2)
|
||||
assert.equal((await GroupLeases.next(b, n2))?.range.id, b2)
|
||||
// A chunk of b2 that doesn't inflate, reported after the batch before it: the range is asked for again and
|
||||
// comes back at its resume point (the chunk fetched plain next time).
|
||||
await db.from('ranges').where('id', b2).update({ next_id: 151 })
|
||||
await Leases.corrupt(b2, n1, 151)
|
||||
await Leases.corrupt(b2, n2, 151)
|
||||
assert.include(await rangeRow(b2), { status: 'planned', node_id: null })
|
||||
assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b2, start: 101, end: 200, next: 151 })
|
||||
assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 151 })
|
||||
assert.deepEqual((await rangeRow(b2)).plain_chunks.map(Number), [151])
|
||||
await db.from('ranges').where('id', b2).update({ status: 'done', next_id: 201 })
|
||||
assert.isNull(await GroupLeases.next(b, n1), 'caught up')
|
||||
assert.isNull(await GroupLeases.next(b, n2), 'caught up: the tail range is the other lane\'s')
|
||||
assert.include(await rangeRow(b1), { status: 'planned' })
|
||||
|
||||
// The tail lane: group b's tail, held by a third node while n2 holds b's history.
|
||||
const n3 = await node('gl-3b')
|
||||
const tail = await GroupLeases.lease(n3, 'tail')
|
||||
assert.equal(tail?.group.id, b)
|
||||
assert.equal(tail?.lane, 'tail')
|
||||
assert.isNull(await GroupLeases.lease(await node('gl-3c'), 'tail'), 'no other group has tail work')
|
||||
assert.deepEqual((await GroupLeases.next(b, n3, 'tail'))?.range, { id: b1, start: 1, end: 100, next: 1 })
|
||||
// The history lane is n2's.
|
||||
await assert.rejects(() => GroupLeases.next(b, n3), LeaseLost)
|
||||
await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
|
||||
assert.isNull(await GroupLeases.next(b, n3, 'tail'), 'the tail lane is caught up')
|
||||
// Each lane its own snapshot.
|
||||
await GroupLeases.putSnapshot(b, n3, 101, Readable.from([Buffer.from('tail at 101')]), 'tail')
|
||||
await GroupLeases.putSnapshot(b, n2, 201, Readable.from([Buffer.from('history at 201')]))
|
||||
assert.equal((await GroupLeases.getSnapshot(b, n3, 'tail'))?.next, 101)
|
||||
assert.equal((await GroupLeases.getSnapshot(b, n2))?.next, 201)
|
||||
await GroupLeases.release(b, n3, 'tail')
|
||||
assert.include(await GroupLeases.held(b, n2), { node_id: n2 })
|
||||
await assert.rejects(() => GroupLeases.held(b, n3, 'tail'), LeaseLost)
|
||||
|
||||
assert.deepEqual([a1, a2, a3].length, 3)
|
||||
await assert.rejects(() => GroupLeases.next(a, n1), LeaseLost)
|
||||
await assert.rejects(() => GroupLeases.next(a, n2), LeaseLost)
|
||||
})
|
||||
|
||||
test('the order mixes the group with the most planned articles and the longest unheld one', async ({ assert }) => {
|
||||
// Four groups with priority 0 work: c was held before (5000 articles left), a, b and d never (100, 1000, 50).
|
||||
const a = await group('alt.binaries.gl.o.a', 1)
|
||||
const b = await group('alt.binaries.gl.o.b', 2)
|
||||
const c = await group('alt.binaries.gl.o.c', 3)
|
||||
const d = await group('alt.binaries.gl.o.d', 4)
|
||||
await range(a, 1, 0)
|
||||
for (let i = 0; i < 10; i++) await range(b, 1 + 100 * i, 0)
|
||||
for (let i = 0; i < 50; i++) await range(c, 1 + 100 * i, 0)
|
||||
await db.table('ranges').insert({ group_id: d, start_id: 1, end_id: 50, next_id: 1, priority: 0 })
|
||||
await db.table('group_leases').insert({ group_id: c, lane: 'history', node_id: null, heartbeat_at: new Date(Date.now() - 3600_000) })
|
||||
// By articles c, b, a, d; by age a, b, d (never held, by group priority), c. The leases alternate which
|
||||
// ranking goes first: c, a, b, d (or a, c, b, d when the sequence stands on the other parity).
|
||||
await db.rawQuery(`SELECT setval('group_lease_turns', 2)`)
|
||||
const order = []
|
||||
for (const name of ['o1', 'o2', 'o3', 'o4']) order.push((await GroupLeases.lease(await node(name)))?.group.id)
|
||||
assert.deepEqual(order, [c, a, b, d])
|
||||
})
|
||||
|
||||
test('a stale holder is taken over and the ranges after the snapshot are read again; a release keeps the resume point', async ({
|
||||
@@ -133,20 +173,51 @@ test.group('Group leases', (g) => {
|
||||
assert.equal((await GroupLeases.next(a, n1))?.range.id, r4)
|
||||
})
|
||||
|
||||
test('capture only: the history ranges sharing a window priority are neither leased nor handed out', async ({
|
||||
assert,
|
||||
}) => {
|
||||
// Group e: history at priority 0 only (the planner's oldest bucket); group f: a 2016 window range (capture)
|
||||
// between two history ranges at 0.
|
||||
const e = await group('alt.binaries.gl.e', 1)
|
||||
await range(e, 1, 0)
|
||||
const f = await group('alt.binaries.gl.f', 2)
|
||||
await range(f, 1, 0)
|
||||
const f2 = await range(f, 101, 0, { capture: true })
|
||||
await range(f, 201, 0)
|
||||
const n1 = await node('gl-8')
|
||||
const was = coordinatorConfig.leaseCaptureOnly
|
||||
coordinatorConfig.leaseCaptureOnly = true
|
||||
try {
|
||||
assert.equal((await GroupLeases.lease(n1))?.group.id, f, 'the group with a capture range, not the older one')
|
||||
assert.equal((await GroupLeases.next(f, n1))?.range.id, f2)
|
||||
await db.from('ranges').where('id', f2).update({ status: 'done', next_id: 201 })
|
||||
assert.isNull(await GroupLeases.next(f, n1), 'the history range after the window is not handed out')
|
||||
assert.isNull(await GroupLeases.lease(await node('gl-9')))
|
||||
} finally {
|
||||
coordinatorConfig.leaseCaptureOnly = was
|
||||
}
|
||||
assert.equal((await GroupLeases.lease(await node('gl-10')))?.group.id, e, 'any range again without the flag')
|
||||
})
|
||||
|
||||
test('a snapshot older than a done range makes that range be read again from its start', async ({ assert }) => {
|
||||
const a = await group('alt.binaries.gl.d', 1)
|
||||
const r1 = await range(a, 1, 0)
|
||||
const r2 = await range(a, 101, 0)
|
||||
const n1 = await node('gl-6')
|
||||
await GroupLeases.lease(n1)
|
||||
// A range after the snapshot's article done long before the snapshot was taken (the old range leases did it):
|
||||
// not the window's, it stays done.
|
||||
const r3 = await range(a, 201, 0)
|
||||
await db.from('ranges').where('id', r3).update({ status: 'done', next_id: 301, done_at: new Date(Date.now() - 48 * 3600_000) })
|
||||
// The worker snapshotted at the start, finished r1, and crashed before its next snapshot: r1 is read again.
|
||||
await GroupLeases.putSnapshot(a, n1, 1, Readable.from([Buffer.from('window at 1')]))
|
||||
await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101 })
|
||||
await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101, done_at: new Date() })
|
||||
await db.from('group_leases').where('group_id', a).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
|
||||
await GroupLeases.lease(await node('gl-7'))
|
||||
assert.equal((await rangeRow(r1)).status, 'planned')
|
||||
assert.equal(Number((await rangeRow(r1)).next_id), 1)
|
||||
assert.equal((await rangeRow(r2)).status, 'planned')
|
||||
assert.equal(Number((await rangeRow(r2)).next_id), 101)
|
||||
assert.equal((await rangeRow(r3)).status, 'done')
|
||||
})
|
||||
})
|
||||
@@ -145,6 +145,23 @@ test.group('Node API', (group) => {
|
||||
;(await client.post(`/api/v1/node/groups/${groupId}/release`).bearerToken(token)).assertStatus(200)
|
||||
;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
|
||||
assert.include(await db.from('ranges').where('id', r2.id).firstOrFail(), { status: 'planned', node_id: null })
|
||||
|
||||
// The tail lane (`?lane=tail`): the group's tail range, with a cursor and snapshot of its own.
|
||||
const [t] = await db.table('ranges').insert({ group_id: g.id, start_id: 401, end_id: 500, next_id: 401, priority: -3 }).returning('id')
|
||||
;(await client.post('/api/v1/node/groups/lease?lane=sideways').bearerToken(token)).assertStatus(400)
|
||||
const tail = await client.post('/api/v1/node/groups/lease?lane=tail').bearerToken(token)
|
||||
tail.assertStatus(200)
|
||||
assert.equal(tail.body().lane, 'tail')
|
||||
assert.equal(Number(tail.body().group.id), groupId)
|
||||
assert.deepEqual((await client.post(`/api/v1/node/groups/${groupId}/next?lane=tail`).bearerToken(token)).body().range, { id: Number(t.id), start: 401, end: 500, next: 401 })
|
||||
;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
|
||||
;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat?lane=tail`).bearerToken(token)).assertStatus(200)
|
||||
;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot?next=401&lane=tail`, Buffer.from('tail at 401'))).assertStatus(200)
|
||||
const gotTail = await client.get(`/api/v1/node/groups/${groupId}/snapshot?lane=tail`).bearerToken(token)
|
||||
gotTail.assertStatus(200)
|
||||
assert.equal(gotTail.header('x-snapshot-next'), '401')
|
||||
;(await client.post(`/api/v1/node/groups/${groupId}/release?lane=tail`).bearerToken(token)).assertStatus(200)
|
||||
assert.include(await db.from('ranges').where('id', t.id).firstOrFail(), { status: 'planned', node_id: null })
|
||||
} finally {
|
||||
coordinatorConfig.corpus.dir = saved.corpus
|
||||
coordinatorConfig.snapshotDir = saved.snapshots
|
||||
@@ -153,6 +170,38 @@ test.group('Node API', (group) => {
|
||||
}
|
||||
})
|
||||
|
||||
test('a chunk the corpus fill stores while the stream waits for it comes from the corpus', async ({ client, assert }) => {
|
||||
const corpus = await mkdtemp(join(tmpdir(), 'corpus-'))
|
||||
const saved = coordinatorConfig.corpus.dir
|
||||
coordinatorConfig.corpus.dir = corpus
|
||||
try {
|
||||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||
// A capture range not in the corpus: its first chunk spooled by the leased run, the second claimed by
|
||||
// the corpus fill, which stores it in the corpus only, after the stream started.
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 200, next_id: 1, capture: true }).returning('id')
|
||||
const rangeId = Number(r.id)
|
||||
await Spool.write(rangeId, { from: 1, to: 100 }, chunk(11))
|
||||
const node = await Node.create({ name: 'n1' })
|
||||
const token = (await Node.accessTokens.create(node)).value!.release()
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'leased', node_id: node.id, leased_at: new Date(), heartbeat_at: new Date() })
|
||||
|
||||
const pending = client.get(`/api/v1/node/ranges/${rangeId}/stream`).bearerToken(token)
|
||||
setTimeout(() => Corpus.write({ group: GROUP, start: 1, end: 200 }, { from: 101, to: 200 }, chunk(111), 2), 800)
|
||||
const stream = await pending
|
||||
stream.assertStatus(200)
|
||||
const raw = stream.body() as unknown
|
||||
const body: Buffer = Buffer.isBuffer(raw) ? raw : Buffer.from(stream.text(), 'binary')
|
||||
const frames = []
|
||||
for await (const frame of decodeFrames([body])) frames.push(frame)
|
||||
assert.deepEqual(frames.map((f) => [f.from, f.to]), [[1, 100], [101, 200]])
|
||||
assert.equal(parseOverviews(unzipSync(frames[1].data).toString('utf8')).length, 3)
|
||||
assert.isFalse(await Spool.has(rangeId, { from: 101, to: 200 }))
|
||||
} finally {
|
||||
coordinatorConfig.corpus.dir = saved
|
||||
await rm(corpus, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a chunk that does not inflate: reported, removed from the spool, the range goes back', async ({ client, assert }) => {
|
||||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, spooled_chunks: 3 }).returning('id')
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { pendingClass } from '#lib/release/index'
|
||||
|
||||
test.group('Pending classes / SQL pending_class()', () => {
|
||||
test('the SQL function agrees with pendingClass', async ({ assert }) => {
|
||||
const cases: [string, number, boolean][] = [
|
||||
['c04212a389cf69edc54767355ab5285f', 5_000_000, false],
|
||||
['c04212a389cf69edc54767355ab5285f', 5_000_000, true],
|
||||
['Zx5AjNaK-mh7CFg6Hf8uu', 500_000_000, false],
|
||||
['1650777167', 100, false],
|
||||
['mrrtmzrzmi4dgmrzmyytqmlbztczbx', 2_000_000_000, false],
|
||||
['PsNdnYiugfZ.rar', 20_000_000, false],
|
||||
['n5d1ueIf34X', 20_000_000, true],
|
||||
['sgkra vU7u2yuw9LDLCjDu04pOez', 999, false],
|
||||
['Description - Lt5nHjlo3RiuBb3C', 40_000_000, false],
|
||||
['ABCDEFGHIJKLMNOP', 0, false],
|
||||
]
|
||||
for (const [name, bytes, partial] of cases) {
|
||||
const { rows } = await db.rawQuery(`SELECT pending_class(?, ?::bigint, ?) AS k`, [
|
||||
name,
|
||||
bytes,
|
||||
partial,
|
||||
])
|
||||
assert.equal(Number(rows[0].k), pendingClass(name, bytes, partial), name)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -9,7 +9,7 @@ let n = 0
|
||||
|
||||
async function pending(
|
||||
name: string,
|
||||
candidates: { kind: string; messageId: string; file: string }[],
|
||||
candidates: { kind: string; messageId: string; file: string; bytes?: number }[],
|
||||
extra: Record<string, unknown> = {}
|
||||
) {
|
||||
n++
|
||||
@@ -206,3 +206,198 @@ test.group('Pending ladder', (group) => {
|
||||
assert.lengthOf(await db.from('pending_signatures'), 0)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('Pending ladder / classes', (group) => {
|
||||
group.each.setup(async () => {
|
||||
PendingLadder.forget()
|
||||
const rollback = await testUtils.db().withGlobalTransaction()
|
||||
groupId = Number(
|
||||
(
|
||||
await db
|
||||
.table('groups')
|
||||
.insert({ name: 'alt.binaries.classes', priority: 1 })
|
||||
.returning('id')
|
||||
)[0].id
|
||||
)
|
||||
n = 0
|
||||
return rollback
|
||||
})
|
||||
|
||||
test('a claim takes the class with the best record first, the cheapest target within it, the rest oldest first', async ({
|
||||
assert,
|
||||
}) => {
|
||||
// Class 15 (1-10 MB hex, whole) named 9 of 10; class 16 (partial) named 0 of 50; class 0: rows from before.
|
||||
await db.table('pending_classes').multiInsert([
|
||||
{ tool: 'other', group_id: groupId, klass: 15, probed: 10, named: 9, waiting: 2 },
|
||||
{ tool: 'other', group_id: groupId, klass: 16, probed: 50, named: 0, waiting: 1 },
|
||||
])
|
||||
const old = await pending('00000000000000000000000000000031', [
|
||||
{ kind: 'lone', messageId: '<o@x>', file: 'o' },
|
||||
])
|
||||
const poor = await pending(
|
||||
'00000000000000000000000000000032',
|
||||
[{ kind: 'lone', messageId: '<p@x>', file: 'p' }],
|
||||
{ klass: 16 }
|
||||
)
|
||||
const bigTarget = await pending(
|
||||
'00000000000000000000000000000033',
|
||||
[{ kind: 'archive', messageId: '<b@x>', file: 'b.rar', bytes: 700_000 }],
|
||||
{ klass: 15, target_bytes: 700_000 }
|
||||
)
|
||||
const smallTarget = await pending(
|
||||
'00000000000000000000000000000034',
|
||||
[{ kind: 'archive', messageId: '<s@x>', file: 's.rar', bytes: 50_000 }],
|
||||
{ klass: 15, target_bytes: 50_000 }
|
||||
)
|
||||
const probes = await PendingLadder.claim(10)
|
||||
assert.deepEqual(
|
||||
probes.map((p) => [p.releaseKey, p.klass]),
|
||||
[
|
||||
[smallTarget, 15],
|
||||
[bigTarget, 15],
|
||||
[poor, 16],
|
||||
[old, 0],
|
||||
]
|
||||
)
|
||||
const waiting = await db
|
||||
.from('pending_classes')
|
||||
.where({ group_id: groupId })
|
||||
.orderBy('klass')
|
||||
.select('klass', 'waiting')
|
||||
assert.deepEqual(
|
||||
waiting.map((c) => [c.klass, c.waiting]),
|
||||
[
|
||||
[15, 0],
|
||||
[16, 0],
|
||||
]
|
||||
)
|
||||
// Outcomes count on the class; a probe given back waits again.
|
||||
await PendingLadder.record([
|
||||
{
|
||||
probe: probes[0],
|
||||
status: 'done',
|
||||
inspection: inspection('s.rar', ['Some.Movie.2016.1080p.mkv']),
|
||||
},
|
||||
{ probe: probes[1], status: 'waiting' },
|
||||
{ probe: probes[2], status: 'missing' },
|
||||
])
|
||||
const classes = await db.from('pending_classes').where({ group_id: groupId }).orderBy('klass')
|
||||
assert.deepEqual(
|
||||
classes.map((c) => [c.klass, c.probed, c.named, c.waiting, c.dropped]),
|
||||
[
|
||||
[15, 11, 10, 1, false],
|
||||
[16, 51, 0, 0, false],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('a class that names nothing is dropped and its waiting releases discarded, the other classes untouched', async ({
|
||||
assert,
|
||||
}) => {
|
||||
await db
|
||||
.table('pending_classes')
|
||||
.insert({ tool: 'other', group_id: groupId, klass: 16, probed: 99, named: 0, waiting: 1 })
|
||||
const last = await pending(
|
||||
'00000000000000000000000000000041',
|
||||
[{ kind: 'lone', messageId: '<z@x>', file: 'z' }],
|
||||
{ klass: 16 }
|
||||
)
|
||||
const [probe] = await PendingLadder.claim(1)
|
||||
assert.equal(probe.releaseKey, last)
|
||||
await PendingLadder.record([{ probe, status: 'missing' }])
|
||||
assert.include(
|
||||
await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail(),
|
||||
{ probed: 100, named: 0, dropped: true }
|
||||
)
|
||||
const waiting = await pending(
|
||||
'00000000000000000000000000000042',
|
||||
[{ kind: 'lone', messageId: '<w@x>', file: 'w' }],
|
||||
{ klass: 16 }
|
||||
)
|
||||
const other = await pending(
|
||||
'00000000000000000000000000000043',
|
||||
[{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
|
||||
{ klass: 15 }
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET waiting = waiting + 1 WHERE group_id = ? AND klass = 16`,
|
||||
[groupId]
|
||||
)
|
||||
await db.table('rule_sets').insert({
|
||||
version: 98,
|
||||
rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
|
||||
note: 'test',
|
||||
})
|
||||
assert.equal(await PendingLadder.discard(), 1)
|
||||
const statusOf = async (key: string) =>
|
||||
(
|
||||
await db
|
||||
.from('pending_releases')
|
||||
.where('release_key', Buffer.from(key, 'hex'))
|
||||
.firstOrFail()
|
||||
).status
|
||||
assert.equal(await statusOf(waiting), PENDING.discarded)
|
||||
assert.equal(await statusOf(other), PENDING.waiting)
|
||||
assert.equal(
|
||||
(await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail())
|
||||
.waiting,
|
||||
0
|
||||
)
|
||||
})
|
||||
|
||||
test('a twin by content names a release whose own probe did not', async ({ assert }) => {
|
||||
// A catalog release with a readable name holds the same first 16 KiB and size.
|
||||
const [poster] = await db.table('posters').insert({ poster: 'Twin <twin@x>' }).returning('id')
|
||||
const [set] = await db
|
||||
.table('group_sets')
|
||||
.insert({ groups: JSON.stringify(['alt.binaries.classes']) })
|
||||
.returning('id')
|
||||
const [twin] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
release_key: Buffer.from('aa'.repeat(20), 'hex'),
|
||||
identity_key: Buffer.from('bb'.repeat(20), 'hex'),
|
||||
group_id: groupId,
|
||||
group_set_id: set.id,
|
||||
poster_id: poster.id,
|
||||
name: 'Some.Movie.2016.1080p.BluRay-GRP',
|
||||
source: 3,
|
||||
posted_at: new Date('2026-10-01T00:00:00Z'),
|
||||
first_article: 1,
|
||||
last_article: 2,
|
||||
files: 2,
|
||||
segments: 2,
|
||||
bytes: 1000,
|
||||
readable: true,
|
||||
})
|
||||
.returning('id')
|
||||
await db.table('file_hashes').insert({
|
||||
candidate_id: 777_001,
|
||||
release_id: twin.id,
|
||||
hash16k: 'deadbeef'.repeat(4),
|
||||
size: 1000,
|
||||
})
|
||||
const key = await pending('00000000000000000000000000000051', [
|
||||
{ kind: 'lone', messageId: '<t@x>', file: '00000000000000000000000000000051' },
|
||||
])
|
||||
const [probe] = await PendingLadder.claim(1)
|
||||
assert.equal(probe.releaseKey, key)
|
||||
await PendingLadder.record([
|
||||
{
|
||||
probe,
|
||||
status: 'done',
|
||||
inspection: {
|
||||
...inspection('00000000000000000000000000000051'),
|
||||
hash16k: 'deadbeef'.repeat(4),
|
||||
},
|
||||
},
|
||||
])
|
||||
assert.include(
|
||||
await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
|
||||
{
|
||||
status: PENDING.naming,
|
||||
named: 'Some.Movie.2016.1080p.BluRay-GRP',
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,142 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import Planner from '#services/planner'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
const TAIL = coordinatorConfig.tail.priority
|
||||
|
||||
test.group('Planner / tail ranges', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('applyTail grows an open range, or adds the slice when the range was taken meanwhile', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const [g] = await db
|
||||
.table('groups')
|
||||
.insert({
|
||||
name: 'alt.binaries.tail',
|
||||
priority: 1,
|
||||
tail: true,
|
||||
planned_at: new Date('2026-10-27T08:00:00Z'),
|
||||
})
|
||||
.returning('id')
|
||||
const [open] = await db
|
||||
.table('ranges')
|
||||
.insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, priority: TAIL })
|
||||
.returning('id')
|
||||
const [taken] = await db
|
||||
.table('ranges')
|
||||
.insert({
|
||||
group_id: g.id,
|
||||
start_id: 301,
|
||||
end_id: 400,
|
||||
next_id: 301,
|
||||
priority: TAIL,
|
||||
status: 'leased',
|
||||
node_id: 1,
|
||||
})
|
||||
.returning('id')
|
||||
const now = new Date('2026-10-27T09:00:00Z')
|
||||
const planned = await Planner.applyTail(
|
||||
{
|
||||
rows: [
|
||||
{ group_id: Number(g.id), start_id: 2001, end_id: 2500, next_id: 2001, priority: TAIL },
|
||||
],
|
||||
extended: [
|
||||
{ id: Number(open.id), group_id: Number(g.id), from: 301, end_id: 2000 },
|
||||
{ id: Number(taken.id), group_id: Number(g.id), from: 401, end_id: 600 },
|
||||
],
|
||||
},
|
||||
now
|
||||
)
|
||||
assert.deepEqual(planned, { groups: 1, ranges: 2, extended: 1 })
|
||||
const ranges = await db
|
||||
.from('ranges')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('start_id')
|
||||
.select('start_id', 'end_id', 'status', 'priority')
|
||||
assert.deepEqual(
|
||||
ranges.map((r) => [Number(r.start_id), Number(r.end_id), r.status, Number(r.priority)]),
|
||||
[
|
||||
[1, 2000, 'planned', TAIL],
|
||||
[301, 400, 'leased', TAIL],
|
||||
[401, 600, 'planned', TAIL],
|
||||
[2001, 2500, 'planned', TAIL],
|
||||
]
|
||||
)
|
||||
const updated = await db.from('groups').where('id', g.id).firstOrFail()
|
||||
assert.equal(Number(updated.high), 2500)
|
||||
assert.equal(new Date(updated.planned_at).toISOString(), now.toISOString())
|
||||
})
|
||||
|
||||
test('mergeTail packs adjacent planned tail ranges and leaves taken, spooled, capture and history ranges alone', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const saved = coordinatorConfig.rangeArticles
|
||||
coordinatorConfig.rangeArticles = 1000
|
||||
try {
|
||||
const [g] = await db
|
||||
.table('groups')
|
||||
.insert({ name: 'alt.binaries.tail', priority: 1, tail: true })
|
||||
.returning('id')
|
||||
const gid = Number(g.id)
|
||||
const range = (start_id: number, end_id: number, extra: Record<string, unknown> = {}) => ({
|
||||
group_id: gid,
|
||||
start_id,
|
||||
end_id,
|
||||
next_id: start_id,
|
||||
priority: TAIL,
|
||||
...extra,
|
||||
})
|
||||
await db.table('ranges').multiInsert([
|
||||
// History, then a run of five slivers (the fifth is spooled: a break), two more, a capture range, the tail's rest.
|
||||
range(1, 1000, { priority: 500, status: 'done' }),
|
||||
range(1001, 1100),
|
||||
range(1101, 1200),
|
||||
range(1201, 1300),
|
||||
range(1301, 1400),
|
||||
range(1401, 1500, { spooled_chunks: 1 }),
|
||||
range(1501, 1600),
|
||||
range(1601, 1700),
|
||||
range(1701, 1800, { capture: true }),
|
||||
range(1801, 2600),
|
||||
range(2601, 2700),
|
||||
range(2701, 2800, { status: 'leased', node_id: 1 }),
|
||||
])
|
||||
assert.deepEqual(await Planner.mergeTail(true), { candidates: 8, merged: 3, dropped: 5 })
|
||||
assert.equal(
|
||||
(await db.from('ranges').where('group_id', gid).count('* as n').first())!.n,
|
||||
'12'
|
||||
)
|
||||
assert.deepEqual(await Planner.mergeTail(), { candidates: 8, merged: 3, dropped: 5 })
|
||||
const ranges = await db
|
||||
.from('ranges')
|
||||
.where('group_id', gid)
|
||||
.orderBy('start_id')
|
||||
.select('start_id', 'end_id', 'next_id', 'status', 'priority', 'capture')
|
||||
assert.deepEqual(
|
||||
ranges.map((r) => [
|
||||
Number(r.start_id),
|
||||
Number(r.end_id),
|
||||
Number(r.next_id),
|
||||
r.status,
|
||||
Number(r.priority),
|
||||
r.capture,
|
||||
]),
|
||||
[
|
||||
[1, 1000, 1, 'done', 500, false],
|
||||
[1001, 1400, 1001, 'planned', TAIL, false],
|
||||
[1401, 1500, 1401, 'planned', TAIL, false],
|
||||
[1501, 1700, 1501, 'planned', TAIL, false],
|
||||
[1701, 1800, 1701, 'planned', TAIL, true],
|
||||
[1801, 2700, 1801, 'planned', TAIL, false],
|
||||
[2701, 2800, 2701, 'leased', TAIL, false],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(await Planner.mergeTail(), { candidates: 3, merged: 0, dropped: 0 })
|
||||
} finally {
|
||||
coordinatorConfig.rangeArticles = saved
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -163,6 +163,55 @@ test.group('ReleaseJudge', (group) => {
|
||||
assert.isNull(await db.from('release_verdicts').where('release_id', locked).first())
|
||||
})
|
||||
|
||||
test('judges tiny single-article files under release-like names', async ({ assert }) => {
|
||||
let n = 0
|
||||
const settled = new Date(Date.now() - 3 * 86400_000)
|
||||
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
||||
n++
|
||||
const key = `jt${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||
return Number(
|
||||
await insertRelease({
|
||||
release_key: key,
|
||||
identity_key: key,
|
||||
name,
|
||||
source: 'stem',
|
||||
groups: ['alt.binaries.boneless'],
|
||||
files: 1,
|
||||
segments: 1,
|
||||
bytes: 15_000,
|
||||
posted_at: settled,
|
||||
readable: true,
|
||||
settled_at: settled,
|
||||
...extra,
|
||||
})
|
||||
)
|
||||
}
|
||||
const dotted = await release('Velvet.S02E20.GERMAN.720p.HDTV.x264-ACED')
|
||||
const spaced = await release('Pacific Rim (2013) BDRip DVDr-sailo1')
|
||||
const subs = await release('Some.Film.2019.German.Subs-XYZ')
|
||||
const ebook = await release('Some.Book.2019.epub-GRP')
|
||||
const music = await release('Some.Album.2019.FLAC-GRP', { groups: ['alt.binaries.sounds.flac'] })
|
||||
const big = await release('Big.Film.2019.1080p-GRP', { bytes: 500_000 })
|
||||
const twoFiles = await release('Two.Films.2019.1080p-GRP', { files: 2, segments: 2 })
|
||||
const plain = await release('Holiday photo')
|
||||
const sidecar = await release('Side.Car.2019.1080p-GRP', { sidecar_only: true })
|
||||
const unreadable = await release('Xk9v.2019.Qpz-ab', { readable: false })
|
||||
|
||||
const round = await new ReleaseJudge().full()
|
||||
const verdicts = new Map(
|
||||
(await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [
|
||||
Number(v.release_id),
|
||||
v.rule,
|
||||
])
|
||||
)
|
||||
assert.equal(verdicts.get(dotted), 'tiny')
|
||||
assert.equal(verdicts.get(spaced), 'tiny')
|
||||
for (const id of [subs, ebook, music, big, twoFiles, plain, sidecar, unreadable]) {
|
||||
assert.notEqual(verdicts.get(id), 'tiny')
|
||||
}
|
||||
assert.equal(round.rules.tiny.added, 2)
|
||||
})
|
||||
|
||||
test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({
|
||||
assert,
|
||||
}) => {
|
||||
|
||||
@@ -138,4 +138,26 @@ test.group('Releases API', (group) => {
|
||||
const list = (await client.get('/api/v1/releases?q=locked movie')).body() as { releases: { passwordKnown: boolean }[] }
|
||||
assert.isTrue(list.releases[0].passwordKnown)
|
||||
})
|
||||
|
||||
test('a word with a leading ^ matches at a word start only; without it anywhere', async ({ client, assert }) => {
|
||||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||
const [n] = await db.table('nodes').insert({ name: 'api-node' }).returning('id')
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id')
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||
indexer.add([
|
||||
...file('Zed.Dead.2019.1080p.mkv', 'Zed.Dead.2019 [1/1]', 2, 100),
|
||||
...file('The_Zed_Files.S01E01.mkv', 'The_Zed_Files.S01E01 [1/1]', 2, 200),
|
||||
...file('Babe.Facialized.2017.mkv', 'Babe.Facialized.2017 [1/1]', 2, 300),
|
||||
], 1001)
|
||||
indexer.finish()
|
||||
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
||||
|
||||
const names = async (q: string) =>
|
||||
((await client.get(`/api/v1/releases?q=${encodeURIComponent(q)}`)).body() as { releases: { name: string }[] }).releases.map((x) => x.name).sort()
|
||||
assert.lengthOf(await names('zed'), 3)
|
||||
assert.deepEqual(await names('^zed'), ['The_Zed_Files S01E01', 'Zed Dead 2019'])
|
||||
assert.deepEqual(await names('^zed dead'), ['Zed Dead 2019'])
|
||||
// Regex characters in the word are literal.
|
||||
assert.deepEqual(await names('^ze+d'), [])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,112 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import Settler from '#services/settler'
|
||||
import { insertRelease } from '#tests/functional/helpers/releases'
|
||||
|
||||
const hoursAgo = (h: number) => new Date(Date.now() - h * 3600_000)
|
||||
|
||||
test.group('Settler', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('a complete provisional release quiet for six hours is settled, its waiting candidates made claimable', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const base = {
|
||||
groups: ['alt.binaries.test'],
|
||||
poster: 'p@x',
|
||||
first_article: 1,
|
||||
posted_at: hoursAgo(30),
|
||||
last_posted_at: hoursAgo(30),
|
||||
}
|
||||
// Complete and old: settled.
|
||||
const old = await insertRelease({
|
||||
...base,
|
||||
name: 'old',
|
||||
files: 3,
|
||||
file_total: 3,
|
||||
complete: true,
|
||||
readable: true,
|
||||
first_seen_at: hoursAgo(7),
|
||||
})
|
||||
// Complete but seen an hour ago: its window may still be carried.
|
||||
const recent = await insertRelease({
|
||||
...base,
|
||||
name: 'recent',
|
||||
files: 3,
|
||||
file_total: 3,
|
||||
complete: true,
|
||||
first_seen_at: hoursAgo(1),
|
||||
})
|
||||
// Complete, old, but its last part posted an hour ago: still arriving.
|
||||
const posting = await insertRelease({
|
||||
...base,
|
||||
name: 'posting',
|
||||
files: 3,
|
||||
file_total: 3,
|
||||
complete: true,
|
||||
first_seen_at: hoursAgo(7),
|
||||
last_posted_at: hoursAgo(1),
|
||||
})
|
||||
// Incomplete, and without a counter: the continuation's.
|
||||
const partial = await insertRelease({
|
||||
...base,
|
||||
name: 'partial',
|
||||
files: 2,
|
||||
file_total: 3,
|
||||
complete: false,
|
||||
first_seen_at: hoursAgo(7),
|
||||
})
|
||||
const uncounted = await insertRelease({
|
||||
...base,
|
||||
name: 'uncounted',
|
||||
files: 2,
|
||||
file_total: null,
|
||||
complete: null,
|
||||
first_seen_at: hoursAgo(7),
|
||||
})
|
||||
// Settled already: left alone.
|
||||
const settled = await insertRelease({
|
||||
...base,
|
||||
name: 'settled',
|
||||
files: 3,
|
||||
file_total: 3,
|
||||
complete: true,
|
||||
first_seen_at: hoursAgo(7),
|
||||
settled_at: hoursAgo(6),
|
||||
})
|
||||
await db.table('probe_candidates').multiInsert([
|
||||
{ release_id: old, kind: 'par2', message_id: '<old@x>', file: 'x.par2', status: 'waiting' },
|
||||
{
|
||||
release_id: partial,
|
||||
kind: 'par2',
|
||||
message_id: '<partial@x>',
|
||||
file: 'x.par2',
|
||||
status: 'waiting',
|
||||
},
|
||||
])
|
||||
|
||||
assert.equal(await Settler.settleComplete(), 1)
|
||||
const rows = await db
|
||||
.from('releases')
|
||||
.whereIn('id', [old, recent, posting, partial, uncounted, settled])
|
||||
.select('id', 'settled_at')
|
||||
const settledIds = rows
|
||||
.filter((r) => r.settled_at !== null)
|
||||
.map((r) => Number(r.id))
|
||||
.sort()
|
||||
assert.deepEqual(settledIds, [old, settled].sort())
|
||||
const candidates = await db
|
||||
.from('probe_candidates')
|
||||
.whereIn('release_id', [old, partial])
|
||||
.select('release_id', 'ready', 'unreadable')
|
||||
assert.deepEqual(
|
||||
candidates.map((c) => [Number(c.release_id), c.ready, c.unreadable]).sort(),
|
||||
[
|
||||
[old, true, false],
|
||||
[partial, false, false],
|
||||
].sort()
|
||||
)
|
||||
assert.equal(await Settler.settleComplete(), 0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,168 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import StatusService from '#services/status'
|
||||
|
||||
test.group('Status', (group) => {
|
||||
let dir = ''
|
||||
let snapshotDir = ''
|
||||
group.each.setup(async () => {
|
||||
const rollback = await testUtils.db().withGlobalTransaction()
|
||||
dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
|
||||
snapshotDir = coordinatorConfig.snapshotDir
|
||||
coordinatorConfig.snapshotDir = dir
|
||||
return rollback
|
||||
})
|
||||
group.each.teardown(async () => {
|
||||
coordinatorConfig.snapshotDir = snapshotDir
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test('the ranges by band, the held groups with their snapshots, the backlogs', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const [g] = await db
|
||||
.table('groups')
|
||||
.insert({ name: 'alt.binaries.status', priority: 1 })
|
||||
.returning('id')
|
||||
const [n] = await db
|
||||
.table('nodes')
|
||||
.insert({ name: 'status-node', last_seen_at: new Date() })
|
||||
.returning('id')
|
||||
const ranges = [
|
||||
{ start_id: 1, end_id: 100, next_id: 1, priority: -3 },
|
||||
{ start_id: 101, end_id: 200, next_id: 101, priority: -1 },
|
||||
{
|
||||
start_id: 201,
|
||||
end_id: 300,
|
||||
next_id: 301,
|
||||
priority: 0,
|
||||
capture: true,
|
||||
status: 'done',
|
||||
done_at: new Date(),
|
||||
},
|
||||
{
|
||||
start_id: 301,
|
||||
end_id: 400,
|
||||
next_id: 351,
|
||||
priority: 0,
|
||||
capture: true,
|
||||
status: 'leased',
|
||||
node_id: n.id,
|
||||
spooled_chunks: 2,
|
||||
},
|
||||
{
|
||||
start_id: 401,
|
||||
end_id: 500,
|
||||
next_id: 401,
|
||||
priority: 2,
|
||||
capture: true,
|
||||
stored_at: new Date(),
|
||||
},
|
||||
{ start_id: 501, end_id: 600, next_id: 501, priority: 500 },
|
||||
{ start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' },
|
||||
]
|
||||
const ids = await db
|
||||
.table('ranges')
|
||||
.multiInsert(ranges.map((r) => ({ group_id: g.id, ...r })))
|
||||
.returning('id')
|
||||
await db.table('group_leases').multiInsert([
|
||||
{
|
||||
group_id: g.id,
|
||||
lane: 'history',
|
||||
node_id: n.id,
|
||||
leased_at: new Date(),
|
||||
heartbeat_at: new Date(),
|
||||
range_id: ids[3].id,
|
||||
snapshot_next: 301,
|
||||
snapshot_at: new Date(Date.now() - 600_000),
|
||||
ranges_done: 1,
|
||||
headers: 100,
|
||||
},
|
||||
{
|
||||
group_id: g.id,
|
||||
lane: 'tail',
|
||||
node_id: null,
|
||||
heartbeat_at: new Date(Date.now() - 3600_000),
|
||||
snapshot_next: 1,
|
||||
snapshot_at: new Date(Date.now() - 3600_000),
|
||||
},
|
||||
])
|
||||
await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000))
|
||||
await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000))
|
||||
// A group planned from its low watermark 1000: its ancient range counts in no band and is never leased.
|
||||
const [old] = await db
|
||||
.table('groups')
|
||||
.insert({ name: 'alt.binaries.ancient', priority: 2 })
|
||||
.returning('id')
|
||||
await db.table('ranges').multiInsert([
|
||||
{ group_id: old.id, start_id: 1, end_id: 999, next_id: 1, priority: 0, status: 'ancient' },
|
||||
{ group_id: old.id, start_id: 1000, end_id: 1099, next_id: 1000, priority: 0 },
|
||||
])
|
||||
|
||||
const s = await StatusService.collect()
|
||||
assert.deepEqual(
|
||||
s.bands.map((b) => [
|
||||
b.band,
|
||||
b.planned,
|
||||
b.plannedArticles,
|
||||
b.leased,
|
||||
b.done,
|
||||
b.doneArticles,
|
||||
b.failed,
|
||||
]),
|
||||
[
|
||||
['tail', 1, 100, 0, 0, 0, 0],
|
||||
['raised', 1, 100, 0, 0, 0, 0],
|
||||
['window 0', 0, 0, 1, 1, 100, 0],
|
||||
['window 2', 1, 100, 0, 0, 0, 0],
|
||||
['history 0-999', 2, 200, 0, 0, 0, 0],
|
||||
['history 4000+', 0, 0, 0, 0, 0, 1],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(s.ancient, { groups: 1, articles: 999 })
|
||||
assert.deepEqual(s.lastHour, {
|
||||
indexedArticles: 100,
|
||||
doneRanges: 1,
|
||||
storedArticles: 100,
|
||||
storedRanges: 1,
|
||||
})
|
||||
assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 })
|
||||
assert.deepEqual(s.spool, { ranges: 1, chunks: 2 })
|
||||
assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 })
|
||||
assert.lengthOf(s.held, 1)
|
||||
assert.include(s.held[0], {
|
||||
node: 'status-node',
|
||||
group: 'alt.binaries.status',
|
||||
lane: 'history',
|
||||
snapshotNext: 301,
|
||||
snapshotMb: 2,
|
||||
rangesDone: 1,
|
||||
headers: 100,
|
||||
})
|
||||
assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 })
|
||||
assert.isAtLeast(s.held[0].snapshotSeconds!, 599)
|
||||
assert.equal(s.snapshots.count, 2)
|
||||
assert.deepEqual(
|
||||
s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]),
|
||||
[
|
||||
['alt.binaries.status', 'history', 2],
|
||||
['alt.binaries.status', 'tail', 0.001],
|
||||
]
|
||||
)
|
||||
assert.equal(s.releases.provisional, 0)
|
||||
assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 })
|
||||
assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0)
|
||||
assert.isAbove(s.catalog.gb, 0)
|
||||
|
||||
const text = StatusService.render(s)
|
||||
assert.include(text, 'window 0')
|
||||
assert.include(text, 'alt.binaries.status history status-node 301-400 351')
|
||||
assert.include(text, 'Snapshots: 2 files')
|
||||
assert.include(text, 'Ancient: 999 articles below the low watermark of 1 groups')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,36 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { historyRows } from '#services/planner'
|
||||
|
||||
test.group('Planner / history rows', () => {
|
||||
test('a group planned for the first time: the ancient range below the low watermark, then the slices', ({
|
||||
assert,
|
||||
}) => {
|
||||
const rows = historyRows(7, { low: 1001, high: 3500 }, 0, 1000)
|
||||
assert.deepEqual(
|
||||
rows.map((r) => [r.start_id, r.end_id, r.next_id, r.priority, r.status ?? 'planned']),
|
||||
[
|
||||
[1, 1000, 1, 0, 'ancient'],
|
||||
[1001, 2000, 1001, 0, 'planned'],
|
||||
[2001, 3000, 2001, 400, 'planned'],
|
||||
[3001, 3500, 3001, 800, 'planned'],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('no ancient range when the group starts at article 1, nor when planning again', ({
|
||||
assert,
|
||||
}) => {
|
||||
assert.deepEqual(
|
||||
historyRows(7, { low: 1, high: 1500 }, 0, 1000).map((r) => [r.start_id, r.end_id, r.status]),
|
||||
[
|
||||
[1, 1000, undefined],
|
||||
[1001, 1500, undefined],
|
||||
]
|
||||
)
|
||||
// Planned before up to 3500: only the new articles, no second ancient range.
|
||||
assert.deepEqual(
|
||||
historyRows(7, { low: 1200, high: 4100 }, 3500, 1000).map((r) => [r.start_id, r.end_id]),
|
||||
[[3501, 4100]]
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,16 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { queueLevels } from '#services/prober'
|
||||
|
||||
test.group('Prober / queue levels', () => {
|
||||
test('the next claim starts with eight pipeline rounds of every lane still queued, and fills to twice that', ({
|
||||
assert,
|
||||
}) => {
|
||||
assert.deepEqual(queueLevels(22, 8, 400), { low: 1408, target: 2816 })
|
||||
assert.deepEqual(queueLevels(7, 8, 400), { low: 448, target: 896 })
|
||||
})
|
||||
|
||||
test('few lanes: at least the configured batch', ({ assert }) => {
|
||||
assert.deepEqual(queueLevels(2, 8, 400), { low: 400, target: 800 })
|
||||
assert.deepEqual(queueLevels(0, 8, 400), { low: 400, target: 800 })
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,5 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { tailRanges, wildmatOf } from '#services/planner'
|
||||
import { tailMerges, tailRanges, wildmatOf } from '#services/planner'
|
||||
|
||||
const options = { rangeArticles: 1000, minArticles: 100, maxDelayMinutes: 15, priority: -1 }
|
||||
const now = new Date('2026-10-09T10:00:00Z')
|
||||
@@ -7,7 +7,7 @@ const minutesAgo = (m: number) => new Date(now.getTime() - m * 60_000)
|
||||
|
||||
test.group('coordinator / live tail', () => {
|
||||
test('a chunk of new articles, or a long enough wait, makes tail ranges', ({ assert }) => {
|
||||
const rows = tailRanges(
|
||||
const { rows, extended } = tailRanges(
|
||||
[
|
||||
// Enough new articles: planned at once, cut at rangeArticles.
|
||||
{ id: 1, plannedTo: 5000, plannedAt: minutesAgo(2), high: 7500 },
|
||||
@@ -32,6 +32,94 @@ test.group('coordinator / live tail', () => {
|
||||
[3, 101, 103, 101, -1],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(extended, [])
|
||||
})
|
||||
|
||||
test('an open last range grows up to rangeArticles before rows are added', ({ assert }) => {
|
||||
const { rows, extended } = tailRanges(
|
||||
[
|
||||
// Room in the open range for everything new.
|
||||
{
|
||||
id: 1,
|
||||
plannedTo: 5300,
|
||||
plannedAt: minutesAgo(20),
|
||||
high: 5350,
|
||||
open: { id: 11, start: 5001 },
|
||||
},
|
||||
// The open range fills up, the rest is cut as rows.
|
||||
{
|
||||
id: 2,
|
||||
plannedTo: 5300,
|
||||
plannedAt: minutesAgo(2),
|
||||
high: 7500,
|
||||
open: { id: 12, start: 5001 },
|
||||
},
|
||||
// An open range already full: rows only.
|
||||
{
|
||||
id: 3,
|
||||
plannedTo: 6000,
|
||||
plannedAt: minutesAgo(2),
|
||||
high: 6500,
|
||||
open: { id: 13, start: 5001 },
|
||||
},
|
||||
// Not open (taken, spooled, another priority): rows.
|
||||
{ id: 4, plannedTo: 5300, plannedAt: minutesAgo(20), high: 5350, open: null },
|
||||
],
|
||||
now,
|
||||
options
|
||||
)
|
||||
assert.deepEqual(
|
||||
extended.map((e) => [e.id, e.group_id, e.from, e.end_id]),
|
||||
[
|
||||
[11, 1, 5301, 5350],
|
||||
[12, 2, 5301, 6000],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(
|
||||
rows.map((r) => [r.group_id, r.start_id, r.end_id]),
|
||||
[
|
||||
[2, 6001, 7000],
|
||||
[2, 7001, 7500],
|
||||
[3, 6001, 6500],
|
||||
[4, 5301, 5350],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('adjacent ranges of a group and priority pack into rangeArticles; gaps and other priorities break a run', ({
|
||||
assert,
|
||||
}) => {
|
||||
const r = (id: number, group_id: number, start_id: number, end_id: number, priority = -3) => ({
|
||||
id,
|
||||
group_id,
|
||||
start_id,
|
||||
end_id,
|
||||
priority,
|
||||
})
|
||||
const merges = tailMerges(
|
||||
[
|
||||
// 1–300 + 301–600 + 601–900 fill 1000 with 901–1000; 1001–1100 starts the next run alone.
|
||||
r(1, 1, 1, 300),
|
||||
r(2, 1, 301, 600),
|
||||
r(3, 1, 601, 900),
|
||||
r(4, 1, 901, 1000),
|
||||
r(5, 1, 1001, 1100),
|
||||
// A gap, then two adjacent at another priority.
|
||||
r(6, 1, 1200, 1250),
|
||||
r(7, 1, 1251, 1300, -1),
|
||||
r(8, 1, 1301, 1400, -1),
|
||||
// Another group; one that would overflow stays alone.
|
||||
r(9, 2, 1, 900),
|
||||
r(10, 2, 901, 1100),
|
||||
r(11, 2, 1101, 1150),
|
||||
],
|
||||
1000
|
||||
)
|
||||
assert.deepEqual(merges, [
|
||||
{ id: 1, end_id: 1000, drop: [2, 3, 4] },
|
||||
{ id: 7, end_id: 1400, drop: [8] },
|
||||
{ id: 10, end_id: 1150, drop: [11] },
|
||||
])
|
||||
})
|
||||
|
||||
test('LIST ACTIVE patterns by the first two name parts', ({ assert }) => {
|
||||
|
||||
@@ -87,6 +87,38 @@ test.group('edge / RangeIndexer', () => {
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
|
||||
assert.lengthOf(batch.releases, 0)
|
||||
})
|
||||
|
||||
test("partialFiles: a file short of parts is a partial release, as the Rust window writes it; one article of a random multi-part file isn't", ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 200, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
|
||||
const name = '8c21f5c36450f9e2ccc6ff387d159d4d'
|
||||
indexer.add([
|
||||
// Parts 1, 3 and 5 of 6 (the others gone from the server), from a re-send's copy of part 3 too.
|
||||
...post(100, name, 6, { poster: 'Bob <bob@home.me>' }).filter((_, i) => i % 2 === 0),
|
||||
...post(150, name, 6, { poster: 'Bob <bob@home.me>' }).slice(2, 3),
|
||||
...post(160, 'Pq4ZmXc8VbN2LkJh6GfDs3', 4).slice(1, 2),
|
||||
], 201)
|
||||
indexer.finish()
|
||||
const batch = indexer.take()
|
||||
assert.lengthOf(batch.releases, 1)
|
||||
assert.include(batch.releases[0], { name, files: 1, segments: 3, partial: true, firstArticle: 100, lastArticle: 104 })
|
||||
assert.deepEqual(decodeChunk(batch.releases[0].payload)[0].segments.map((s) => s[0]), [1, 3, 5])
|
||||
assert.deepEqual(batch.fragments, [])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
|
||||
})
|
||||
|
||||
test('partialFiles: one article of a multi-part file under a scrambled name is per-article before the end, a partial release at it', ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 1_000_000, rules, idleChunks: 1, edgeArticles: 0, partialFiles: true })
|
||||
// Part 8 of 67, a random poster's: given up mid-range.
|
||||
indexer.add(post(100, 'nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr', 67, { poster: 'guAym5TqSlp SCTLKQ0R <8nGc3Rk@OrjrYqFkp.z>' }).slice(7, 8), 200)
|
||||
indexer.add([], 300)
|
||||
indexer.add([], 400)
|
||||
// The same shape still open when the block ends: the rest can be in the next one.
|
||||
indexer.add(post(999_990, 'bslXlwgPiqZadhRa', 5).slice(1, 2), 1_000_001)
|
||||
indexer.finish()
|
||||
const batch = indexer.take()
|
||||
assert.deepEqual(batch.releases.map((r) => [r.name, r.partial]), [['bslXlwgPiqZadhRa', true]])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
|
||||
})
|
||||
})
|
||||
|
||||
test.group('edge / taking over a range', () => {
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { classKey, describeClass, pendingClass } from '#lib/release/index'
|
||||
|
||||
test.group('release / pendingClass', () => {
|
||||
test('size bucket × name shape × partial, as names.rs pending_class', ({ assert }) => {
|
||||
// The same cases as the Rust test.
|
||||
assert.equal(pendingClass('c04212a389cf69edc54767355ab5285f', 5_000_000, false), 15)
|
||||
assert.equal(pendingClass('c04212a389cf69edc54767355ab5285f', 5_000_000, true), 16)
|
||||
assert.equal(pendingClass('Zx5AjNaK-mh7CFg6Hf8uu', 500_000_000, false), 3 * 14 + 5 * 2 + 1)
|
||||
assert.equal(pendingClass('1650777167', 100, false), 3)
|
||||
assert.equal(
|
||||
pendingClass('mrrtmzrzmi4dgmrzmyytqmlbztczbx', 2_000_000_000, false),
|
||||
4 * 14 + 2 * 2 + 1
|
||||
)
|
||||
assert.equal(pendingClass('PsNdnYiugfZ.rar', 20_000_000, false), 2 * 14 + 3 * 2 + 1)
|
||||
assert.equal(pendingClass('n5d1ueIf34X', 20_000_000, true), 2 * 14 + 4 * 2 + 2)
|
||||
assert.equal(pendingClass('sgkra vU7u2yuw9LDLCjDu04pOez', 999, false), 13)
|
||||
assert.equal(describeClass(16), '1-10M hex partial')
|
||||
assert.equal(describeClass(3 * 14 + 5 * 2 + 1), '100M-1G separators')
|
||||
assert.equal(classKey('nyuu', 'alt.binaries.boneless', 16), 'nyuu\talt.binaries.boneless\t16')
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,5 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fileStem, releaseIdentity, runOn, sizeAfterName, subjectRemainder } from '#lib/release/index'
|
||||
import { displayName, fileStem, releaseIdentity, runOn, sizeAfterName, subjectRemainder } from '#lib/release/index'
|
||||
import { looksLikeFilename, weakFileKey } from '#lib/headers/index'
|
||||
|
||||
test.group('release / fileStem', () => {
|
||||
@@ -604,3 +604,15 @@ test.group('release / par2 names', () => {
|
||||
assert.equal(id('[2/6] - "x.jpg" yEnc', 'x.jpg').key, id('[1/6] - "x.part01.rar" yEnc', 'x.part01.rar').key)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / displayName', () => {
|
||||
test('brackets the counters left unmatched leave the name, the key keeps them', ({ assert }) => {
|
||||
const subject = (i: number) => `Babe with Plump Bubble Butt Fucked and Facialized ${String(i).padStart(3, '0')}/477] - 0${176 + i}.jpg`
|
||||
const id = (i: number) => releaseIdentity({ poster: 'webmaster@hdbangvidz.com (Porn4You)', weak: false, subject: subject(i), name: `0${176 + i}.jpg` })!
|
||||
assert.equal(id(2).name, 'Babe with Plump Bubble Butt Fucked and Facialized')
|
||||
assert.equal(id(2).key, id(3).key)
|
||||
assert.equal(displayName('[Group] Show (2020)'), '[Group] Show (2020)')
|
||||
assert.equal(displayName('( Show [x] name'), 'Show [x] name')
|
||||
assert.equal(displayName(']'), ']')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,81 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { isUnreadableName, tokenKind } from '#lib/release/index'
|
||||
|
||||
test.group('release / isUnreadableName', () => {
|
||||
test('scrambles with separators, spaces, random extensions, packaging words and hex with an addon are unreadable', ({
|
||||
assert,
|
||||
}) => {
|
||||
for (const n of [
|
||||
'Zx5AjNaK-mh7CFg6Hf8uu.part01.rar',
|
||||
'yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU',
|
||||
'PsNdnYiugfZ.rar',
|
||||
'phfplC9y07U2yrj',
|
||||
'n5d1ueIf34X',
|
||||
'tJqr1tq7xK',
|
||||
'Description - Lt5nHjlo3RiuBb3C.par2',
|
||||
'bd5b4a59f3aa87114dcf75c0-addon.rar',
|
||||
'mrrtmzrzmi4dgmrzmyytqmlbztczbx',
|
||||
'AA-NaR16302830434zxP32108309f3Ab',
|
||||
'_QBs01_MqZYnYwM_iAyRquhc-',
|
||||
'sgkra vU7u2yuw9LDLCjDu04pOez',
|
||||
'Jk5Pud.zQdNj',
|
||||
'7uKsD_NRgT-G_Uc_p8',
|
||||
'NtpUgCmS_Xa1B2Pk',
|
||||
'{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}',
|
||||
'IkYovjrtZrItjQ.mkv',
|
||||
'ff75ce3f16c746c18b2d78cf.6c52f',
|
||||
'prvjr7hapd0i1ny14pz4',
|
||||
]) {
|
||||
assert.isTrue(isUnreadableName(n), n)
|
||||
}
|
||||
})
|
||||
|
||||
test('scene tags, CamelCase, codes, leet, acronyms and a hash beside words are readable', ({
|
||||
assert,
|
||||
}) => {
|
||||
for (const n of [
|
||||
'Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune',
|
||||
'Paul McCartney - Band on the Run',
|
||||
'ShesSoWild - Ester Ana 1080p',
|
||||
'MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS',
|
||||
'cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar',
|
||||
'Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing',
|
||||
'Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON',
|
||||
'Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr',
|
||||
'Eclipse_IDE_2023-12_R_win32_x86_64.zip',
|
||||
'PartsHPGL 1.06c',
|
||||
'Go4Customer Fax Machine v1.2b',
|
||||
'AdBeGone 1.2',
|
||||
'3DMark03 320',
|
||||
'COD4MW2REMASTERED',
|
||||
'Movie.2020.1080p-GRP [a3f9e2b1]',
|
||||
'EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps',
|
||||
'iPhone',
|
||||
'Some.Show.S01E01.1080p.mkv',
|
||||
'FastFilms 2.04',
|
||||
'BestCrypt NP v4.0',
|
||||
'EasyHTML 2.35 German',
|
||||
'SchiffsModell - September 2026',
|
||||
'CrossStitcher - Issue 440, October 2026',
|
||||
'Laura15Juli2026',
|
||||
'IHaveAWife - Isa Bella',
|
||||
'DickHDDaily - Santana Red',
|
||||
'1904-SmithsWork-Hasluck',
|
||||
]) {
|
||||
assert.isFalse(isUnreadableName(n), n)
|
||||
}
|
||||
})
|
||||
|
||||
test('token kinds', ({ assert }) => {
|
||||
assert.equal(tokenKind('Champions'), 'word')
|
||||
assert.equal(tokenKind('PRESCRiPTiON'), 'word')
|
||||
assert.equal(tokenKind('GarciaLive'), null)
|
||||
assert.equal(tokenKind('S02E10'), null)
|
||||
assert.equal(tokenKind('1080p'), null)
|
||||
assert.equal(tokenKind('addon'), null)
|
||||
assert.equal(tokenKind('NtpUgCmS'), 'random')
|
||||
assert.equal(tokenKind('tBWTuz'), 'random')
|
||||
assert.equal(tokenKind('Lt5nHjlo3RiuBb3C'), 'random')
|
||||
assert.equal(tokenKind('a3f9e2b1'), 'random')
|
||||
})
|
||||
})
|
||||
@@ -9,6 +9,18 @@ const file = (name: string, totalParts: number, firstMessageId: string, group =
|
||||
groups: [group],
|
||||
firstMessageId,
|
||||
poster,
|
||||
segments: totalParts,
|
||||
bytes: totalParts * 700_000,
|
||||
})
|
||||
/** A single archive of `segments` articles and `bytes` in all, posted to `groups` groups. */
|
||||
const archive = (name: string, poster: string, groups: number, segments: number, bytes: number, firstMessageId = NYUU) => ({
|
||||
name,
|
||||
totalParts: segments,
|
||||
groups: Array.from({ length: groups }, (_, i) => `alt.binaries.g${i}`),
|
||||
firstMessageId,
|
||||
poster,
|
||||
segments,
|
||||
bytes,
|
||||
})
|
||||
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
||||
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
||||
@@ -50,6 +62,79 @@ test.group('rules / verdicts', () => {
|
||||
assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU, 'alt.binaries.misc')]), { rule: 'garbage', key: 'nyuu alt.binaries.misc' })
|
||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU)]), { rule: null })
|
||||
})
|
||||
|
||||
test('stub archives: a settled single archive in 2+ groups with 250 kB articles, or 4+ groups, or a webmail poster', ({ assert }) => {
|
||||
// Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave; the article-size leg alone without webmail.
|
||||
assert.include(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com (Jahnunsinos007)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'AW' })
|
||||
assert.include(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'Yenc@power-post.org (Yenc-PP-A&A)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'A' })
|
||||
// Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone).
|
||||
assert.include(rules.judge('k', [archive('Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar', 'Yenc@power-post.org', 5, 2, 319_000)]), { rule: 'stub-archive', key: 'X' })
|
||||
// A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave, but only under 100 MB without the article-size leg.
|
||||
assert.include(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 100, 65_000_000)]), { rule: 'stub-archive', key: 'W' })
|
||||
assert.deepEqual(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 400, 260_000_000)]), { rule: null })
|
||||
// Not settled: a par2 can still join.
|
||||
assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)], false), { rule: null })
|
||||
})
|
||||
|
||||
test('stub archives: small, single-group, split and large hash-named archives are kept', ({ assert }) => {
|
||||
// An e-book: a one- or two-article release always has small articles (LOG.md (126)).
|
||||
assert.deepEqual(rules.judge('k', [archive('Maeve Binchy - Tara Road (epub).rar', 'kie@home.now (KiE)', 2, 2, 521_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 1, 48, 12_000_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'poster@example.org', 2, 100, 70_000_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.part01.rar', 'a@gmail.com', 2, 48, 12_000_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('cbcd32359460df1fa9b7d01d8720.rar', 'b648de0d1c0f7a9a77be273d@gmail.com', 2, 3163, 2_340_000_000)]), { rule: null })
|
||||
// ROT13-named volumes of passworded videos in a.b.erotica, 258 kB articles, two groups, 1-270 MB; and the 100 MB cap on every leg (LOG.md (128)).
|
||||
assert.deepEqual(rules.judge('k', [archive('Ghful.17.12.07.Qnvvfl.Fgbar.cneg3.rar', 'nospam@nospam.nl (EvH)', 2, 5, 1_147_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Ghful.16.07.05.Wbwb.Xvff.cneg1.rar', 'nospam@nospam.nl (EvH)', 2, 1051, 271_000_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Tinasoft EasyCafe 2.2.14 by OmerSalaj.rar', 'Seven7i@mail.com ({Seven7i})', 4, 463, 117_000_000)]), { rule: null })
|
||||
assert.include(rules.judge('k', [archive('Tipard DVD Ripper - 10 0 12 x86 x64 incl Patch.rar', 'Seven7i@mail.com ({Seven7i})', 4, 244, 62_000_000)]), { rule: 'stub-archive', key: 'AXW' })
|
||||
// A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)).
|
||||
assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl.zip', 'home@home.com (TexasDj)', 2, 13, 3_300_000), groups: ['alt.binaries.sounds.karaoke', 'alt.binaries.test'] }]), { rule: null })
|
||||
// A rar with its par2 is no single archive.
|
||||
assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'a@gmail.com', 2, 48, 12_000_000), file('Chat Watch 1.2.1.par2', 1, NYUU)]), { rule: null })
|
||||
const off = new RuleSet({ ...DEFAULT_RULES, dropStubArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
|
||||
assert.deepEqual(off.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)]), { rule: null })
|
||||
})
|
||||
|
||||
test('nonsense pieces: one article of 100 kB to 1 MB under a release-like name', ({ assert }) => {
|
||||
assert.include(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: 'nonsense-piece', key: 'nyuu' })
|
||||
// A 3 kB cue/log sidecar rar with a year in its name: under the floor (the tiny-archive rule's outside the music groups). A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule.
|
||||
assert.deepEqual(rules.judge('k', [{ ...archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000), groups: ['alt.binaries.sounds.lossless'] }]), { rule: null })
|
||||
assert.include(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: 'tiny-archive' })
|
||||
assert.deepEqual(rules.judge('k', [archive('Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar', 'fake@address.com (Psychopath)', 2, 1, 303_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 382_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar', 'saggir@domain.com', 1, 1, 113_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 2, 790_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
|
||||
const off = new RuleSet({ ...DEFAULT_RULES, dropNonsensePieces: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
|
||||
assert.deepEqual(off.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
|
||||
})
|
||||
|
||||
test('tiny archives: one article of 2-100 kB under a release name, spaces allowed', ({ assert }) => {
|
||||
// A RAR holding "<title>.exe": the 2015-17 flood in miniature (LOG.md (131)).
|
||||
assert.include(rules.judge('k', [archive('Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar', 'Yanc@power-post.org <Yanc@power-post.org>', 1, 1, 15_084)]), { rule: 'tiny-archive', key: 'nyuu' })
|
||||
assert.include(rules.judge('k', [archive('Cicero Magazin fuer politische Kultur Mai 05 2016.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 17_478)]), { rule: 'tiny-archive', key: 'nyuu' })
|
||||
// A scene name without a year, a tag or a group suffix: not release-like, so kept.
|
||||
assert.deepEqual(rules.judge('k', [archive('[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar', 'Yanc@power-post.org', 2, 1, 23_449)]), { rule: null })
|
||||
assert.include(rules.judge('k', [archive('Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 7_869)]), { rule: 'tiny-archive', key: 'nyuu' })
|
||||
// Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
|
||||
assert.deepEqual(rules.judge('k', [archive('The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar', 'henk@example.org', 1, 1, 38_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar', 'poster@example.org', 1, 1, 60_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Some Band - Album (2011) covers.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 38_000)]), { rule: null })
|
||||
// A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's; not settled.
|
||||
assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl (2017).zip', 'home@home.com', 1, 1, 60_000), groups: ['alt.binaries.sounds.karaoke'] }]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 2, 20_000)]), { rule: null })
|
||||
assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 1_500)]), { rule: null })
|
||||
assert.include(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 150_000)]), { rule: 'nonsense-piece' })
|
||||
assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)], false), { rule: null })
|
||||
const off = new RuleSet({ ...DEFAULT_RULES, dropTinyArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
|
||||
assert.deepEqual(off.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rules / upload tools', () => {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Why the pending namer's move fails: for sample pending releases (a JSON dump of rows, the rule set and the
|
||||
* priorities), the stored corpus chunks of their span through the TypeScript indexer, once with the rule set as
|
||||
* stored and once without its drop rules; prints whether the pending identity comes out and what does.
|
||||
*
|
||||
* node --import=@poppinss/ts-exec tools/namer_check.ts sample.json /corpus
|
||||
*/
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { identityKey } from '#lib/release/keys'
|
||||
|
||||
const [file, corpus] = process.argv.slice(2)
|
||||
const dump = JSON.parse(readFileSync(file, 'utf8'))
|
||||
const { version: _v, ...defaults } = DEFAULT_RULES
|
||||
const stored: Rules = { ...defaults, ...dump.rules, version: dump.version, priorities: dump.priorities, samplePercent: 0 }
|
||||
const lenient: Rules = { ...stored, droppedTools: [], droppedSignatures: [], droppedPosters: [], dropExecutables: false }
|
||||
|
||||
function text(row: any): string | null {
|
||||
const parts: string[] = []
|
||||
for (const r of row.ranges) {
|
||||
const dir = join(corpus, row.grp, `${r.start}-${r.end}`)
|
||||
let names: string[]
|
||||
try {
|
||||
names = readdirSync(dir).filter((n) => n.endsWith('.gz')).sort()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
for (const n of names) {
|
||||
const [, from, to] = n.slice(0, -3).split('-').map(Number)
|
||||
if (to >= row.first_article && from <= row.last_article) parts.push(unzipSync(readFileSync(join(dir, n))).toString('latin1'))
|
||||
}
|
||||
}
|
||||
return parts.length ? parts.join('') : null
|
||||
}
|
||||
|
||||
function run(row: any, overview: string, rules: Rules) {
|
||||
const ix = new RangeIndexer({ group: row.grp, start: row.first_article, end: row.last_article, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
|
||||
ix.addText(overview, row.last_article + 1)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.some((r) => identityKey(r.key).toString('hex') === row.identity)
|
||||
const near = batch.releases.filter((r) => r.firstArticle !== undefined && Math.abs(Number(r.firstArticle) - row.first_article) < 50)
|
||||
return { found, releases: batch.releases.length, near: near.map((r) => `${identityKey(r.key).toString('hex').slice(0, 12)} ${r.name} @${r.firstArticle}`), counts: batch.counts }
|
||||
}
|
||||
|
||||
for (const row of dump.rows) {
|
||||
const overview = text(row)
|
||||
console.log(`\n${row.grp} tries ${row.tries} ${row.first_article}–${row.last_article} "${row.name}" by ${row.poster}, ${row.segments} segments; identity ${row.identity.slice(0, 12)}`)
|
||||
if (!overview) {
|
||||
console.log(' no corpus text')
|
||||
continue
|
||||
}
|
||||
for (const [label, rules] of [['stored rules', stored], ['no drop rules', lenient]] as const) {
|
||||
const r = run(row, overview, rules)
|
||||
console.log(` ${label}: found ${r.found}; ${r.releases} releases; near: ${r.near.join(' | ') || '-'}; counts ${JSON.stringify(r.counts)}`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user