Compare commits
79
Commits
No files matched your search
@@ -59,35 +59,28 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Rust toolchain
|
||||
run: |
|
||||
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
# The edge worker (edge/, Rust): built first, since the e2e suite runs edge/target/release/edge-node against
|
||||
# the node API; then its own tests, and the Rust and TypeScript RangeIndexers compared on the committed
|
||||
# captures (edge/fixtures).
|
||||
- name: Build the edge worker
|
||||
run: cargo build --release --manifest-path edge/Cargo.toml --bin edge-node
|
||||
- name: Type check
|
||||
run: npx tsc --noEmit
|
||||
- name: Migrate
|
||||
run: node ace migration:run --force
|
||||
- name: Test
|
||||
run: node ace test
|
||||
|
||||
# The edge worker (edge/, Rust): its tests, and the Rust and TypeScript
|
||||
# RangeIndexers compared on the committed captures (edge/fixtures).
|
||||
edge:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Rust toolchain
|
||||
run: |
|
||||
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
- name: cargo test
|
||||
run: cargo test --release --manifest-path edge/Cargo.toml
|
||||
- name: Rust and TypeScript identical on the fixtures
|
||||
run: edge/check-fixtures.sh
|
||||
- name: The Rust indexer over the fixtures
|
||||
run: edge/bench-fixtures.sh
|
||||
|
||||
deploy:
|
||||
needs: [test, edge]
|
||||
needs: [test]
|
||||
if: github.ref == 'refs/heads/rebuild'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+11
-9
@@ -8,6 +8,15 @@
|
||||
# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
|
||||
ARG NODE_VERSION=24.16
|
||||
|
||||
# The Rust worker (before the app: its image carries edge-bench). Registry and target dir are build caches, so a rebuild only compiles what changed.
|
||||
FROM rust:1-bookworm AS edge-build
|
||||
WORKDIR /src
|
||||
COPY edge ./
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release --locked --bin edge-node --bin edge-bench \
|
||||
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
|
||||
|
||||
# The app, compiled, with production dependencies only.
|
||||
FROM node:${NODE_VERSION}-bookworm AS app-build
|
||||
WORKDIR /src
|
||||
@@ -22,19 +31,12 @@ FROM node:${NODE_VERSION}-bookworm-slim AS app
|
||||
ENV NODE_ENV=production
|
||||
WORKDIR /app
|
||||
COPY --from=app-build /src/build ./
|
||||
# The Rust indexer's bench: `node ace replay:corpus` runs a group's corpus through it (the replay harness).
|
||||
COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
|
||||
USER node
|
||||
# Every service picks its own command, e.g. `node ace coordinator:run`.
|
||||
CMD ["node", "ace", "list"]
|
||||
|
||||
# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed.
|
||||
FROM rust:1-bookworm AS edge-build
|
||||
WORKDIR /src
|
||||
COPY edge ./
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release --locked --bin edge-node --bin edge-bench \
|
||||
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
|
||||
|
||||
FROM debian:bookworm-slim AS edge
|
||||
COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
|
||||
# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
|
||||
|
||||
+595
-17
@@ -5,7 +5,8 @@ rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
|
||||
decisions D1–D42 are superseded; its domain model still holds, and its
|
||||
features not yet here are the phase 2 backlog below.
|
||||
|
||||
Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
|
||||
Branch `rebuild`; its successor is designed under *Voyager 3* below (branch
|
||||
`voyager-3`). Phase 1 finds releases from headers and makes their NZBs.
|
||||
Everything learned from articles (names from yEnc headers, par2, archive
|
||||
listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
|
||||
never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
|
||||
@@ -59,9 +60,21 @@ most workers on batcave, a few on poweredge-kantoor with
|
||||
`edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
|
||||
the cap is shared by the node's processes).
|
||||
|
||||
Every group is tiled from article 1 (2026-10-28): one "ancient" range,
|
||||
status `ancient`, covers 1 … low watermark − 1, what had expired on the
|
||||
server before the group was planned. Never leased, fetched or filled —
|
||||
nothing selects it — and the status panel reports it apart ("Ancient: …
|
||||
articles below the low watermark"); migration 6700 added it for the
|
||||
groups planned before, `Planner.historyRows` for groups planned since.
|
||||
|
||||
Also built (2026-10-09): the live tail (new articles planned every 2
|
||||
minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
|
||||
fetched before history); the catalog API, Newznab (`/api`) and the web
|
||||
fetched before history; since 2026-10-27 a round grows the group's open
|
||||
tail range, a planned one at the tail's priority nothing spooled or
|
||||
took, up to RANGE_ARTICLES before it adds rows, as a tail nothing leased
|
||||
had become a sliver a group a round, 127k rows of a median 14k articles;
|
||||
`ranges:merge-tail` packs what such rounds left); the catalog API,
|
||||
Newznab (`/api`) and the web
|
||||
interface, from voyager 1 (the current build), with categories from the
|
||||
name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
|
||||
`database/data/voyager1_signatures.tsv`).
|
||||
@@ -189,13 +202,22 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
number; merged when the neighbouring range is done, then written as a
|
||||
chunk. Never staged per segment.
|
||||
|
||||
8. **Settling**. A release is settled when the ranges covering its posting
|
||||
span are done and no chunk arrived for a day: completeness is computed
|
||||
then (expected vs found files and segments). Phase 2 picks its work from
|
||||
settled releases.
|
||||
8. **Settling**. A release is settled when nothing was added to it for 6
|
||||
hours (30 minutes once its last post is that old: history), and its
|
||||
completeness decided: complete when every file its [i/N] counter
|
||||
announced was found. Without a counter, by its par2 sets once their index
|
||||
is probed (`par2_completeness`): every file a set describes is there by
|
||||
name or by its first 16 KiB, or, renamed, as many files and bytes; decided
|
||||
again after a par2 merge and before the pruner drops a set's file list.
|
||||
Segments aren't compared. Phase 2 picks its work from settled releases.
|
||||
|
||||
9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
|
||||
search (names) and get. Web UI later.
|
||||
9. **NZB and API**. NZB built from a release's chunks, with a known
|
||||
password in its meta; Newznab search (names, inner names) and get; a web
|
||||
interface. Categories (Newznab's) by a trigger on releases (migration
|
||||
5200): from the name, else the inner name the prober found, else a spot's
|
||||
category (spot_category, set by the spot inspector), else the groups
|
||||
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
|
||||
in batches.
|
||||
|
||||
## Group list
|
||||
|
||||
@@ -221,6 +243,39 @@ range with the deployed build's edge-bench (`replay`, ~seconds a range) and
|
||||
reports suspects and the change since the previous build (`report`). A rule
|
||||
change is judged on the whole corpus before a reset.
|
||||
|
||||
Since 2026-10-23 also history fetched ahead of the indexing: while the
|
||||
catalog is over its NVMe cap, ranges marked `capture` (priority >= 0, not
|
||||
leased) are fetched into the corpus by idle loops, at most perGroup /
|
||||
perRange of a group at a time, to be indexed from disk once there's room
|
||||
(LOG.md 2026-10-23 (100)). Stored filtered (`CORPUS_FILTER`, default on):
|
||||
without the lines the edge worker only counts, decided per line as it does
|
||||
(a crosspost another indexed group owns, a subject that names no file part;
|
||||
~48% of headers), the chunk compressed again as zlib; beside it a `.drops`
|
||||
(gzip JSON: rule version, the owners' priorities, per rule the article
|
||||
numbers as differences, crossposts by owner) and `filtered` in range.json.
|
||||
The worker makes the same releases of a filtered chunk; rule_counts lack
|
||||
those two rules' counts. `corpus:filter` filters chunks stored raw. A
|
||||
corpus filtered this way no longer serves a sweep of those two rules.
|
||||
|
||||
Windows by posting time (2026-10-25; voyager 3's test data, LOG.md (104)):
|
||||
`ranges:dates` samples when each active group's articles were posted (one
|
||||
XOVER of 32 articles every 500M articles of the group, 16 … 1000 samples,
|
||||
the median Date of each slice, kept monotone; `group_dates`; over plain
|
||||
connections: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
output, which some small replies begin with, `TruncatedReply`, LOG.md
|
||||
2026-10-25 (105)) and estimates
|
||||
every range's posting date from the curve (`ranges.est_date`; the edges of
|
||||
a window are off by up to one sample interval, three weeks in boneless).
|
||||
`ranges:capture --from 2025-01-01 --to 2026-01-01` marks every active
|
||||
group's ranges posted in the window `capture`: planned ones not started at
|
||||
`--priority` (0), done ones as they are, both fetched into the corpus by the
|
||||
fill, which takes capture ranges in priority order, done ones too (they're
|
||||
fetched again, corpus only), and skips those complete (`ranges.stored_at`,
|
||||
set when it sees the range.json). A year of every group is the data to
|
||||
replay voyager 3's 30-day blocks on: marked 2026-10-25, 2016 (priority 0,
|
||||
30B articles, ~0.56 TB) and 2021 (priority 1, 87B, ~1.6 TB); a recent year
|
||||
is ~200B articles, ~3.6 TB.
|
||||
|
||||
## Phase 2
|
||||
|
||||
Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
|
||||
@@ -238,6 +293,21 @@ files the set describes, into the one with the most files (chunks,
|
||||
candidates, inspections, counts; the par2's shared name when none was
|
||||
readable); `release_merges` keeps what went where. Obfuscated uploads whose
|
||||
files were each a release of their own (`lone` candidates) become one.
|
||||
Readable files have no hash: a name pass (2026-10-23) joins the releases
|
||||
of the set's poster in its group posted within 12 hours whose files the
|
||||
set all names (no case or punctuation; sidecars, par2 and images aside),
|
||||
as `via = 'name'`: an album's tracks and its index, an index beside its
|
||||
differently spelled video. `releases:merge-names` for sets from before.
|
||||
Also merged: lone files whose inner names share a random stem (`inner`),
|
||||
and small sidecar-only releases into their upload (`sidecar`).
|
||||
It reads what's new an hour (`created_at`) at a time, its place kept after
|
||||
each, and merges only the sets held or matched by two releases or more; a
|
||||
file whose first 16 KiB and size more than 20 files share is a common one and
|
||||
matches nothing (LOG.md 2026-10-20 (85)).
|
||||
A set's file list is dropped once it can't merge anything more: complete (all
|
||||
its files in its holder), judged (every holder junk), or final (every holder
|
||||
inside a run of done ranges, those ranges' candidates probed, its hashes seen
|
||||
by the merger); `par2_sets` stays (LOG.md 2026-10-20 (87)).
|
||||
|
||||
**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
|
||||
voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
|
||||
@@ -252,7 +322,7 @@ with 68,032 releases with a password):
|
||||
| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
|
||||
| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
|
||||
| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
|
||||
| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed |
|
||||
| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
|
||||
| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
|
||||
| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
|
||||
|
||||
@@ -260,9 +330,11 @@ Live indexing (the tail) runs since 2026-10-09.
|
||||
|
||||
**Junk** (voyager 1's second filter, D42). After the header pass's rules
|
||||
(garbage signatures, executables, per-article and obfuscated subjects), the
|
||||
judge (`releases:judge`, service `judge`) judges releases hourly by what the
|
||||
prober, the merges and the spots found, in this order (the first rule
|
||||
wins): *dead* (every probed first article gone), *fake* (a film, episode
|
||||
judge (`releases:judge`, service `judge`) judges releases by what the
|
||||
prober, the merges and the spots found: every 2 minutes the releases whose
|
||||
facts changed (noted by triggers in `judge_dirty`), daily every release (for
|
||||
what time changes: a day after settling, ranges read beside it). In this
|
||||
order (the first rule wins): *dead* (every probed first article gone), *fake* (a film, episode
|
||||
or music name whose content is a program: archive entries all
|
||||
executables, or the file itself one by name or by its first bytes, sniffed
|
||||
`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
|
||||
@@ -312,6 +384,453 @@ processes, one NNTP connection each:
|
||||
`spot-text`, and the spot), not on the catalog row; the catalog only says
|
||||
`passworded`.
|
||||
|
||||
## Voyager 3: window indexing (design, 2026-10-25)
|
||||
|
||||
Branch `voyager-3` (a push to it only runs the tests; `rebuild` keeps
|
||||
running on homelab until the reset). Goal: the whole history (~1,220 B
|
||||
articles, 3% indexed on 2026-10-25) indexed and searchable on a 4 TB NVMe
|
||||
and the 15 TB HDD. The catalog costs 3.9 B of NVMe per article today
|
||||
(144 GB for 36.5 B), which puts the full history at ~4.7 TB; the target is
|
||||
≤ 2.5 B per article (~3 TB) with room for the WAL and bloat. Sources:
|
||||
LOG.md 2026-10-24 (102).
|
||||
|
||||
What is wrong today: a release is written as soon as one range sees it,
|
||||
updated by every later range that finds more parts, merged afterwards
|
||||
(fragments, par2, names) and settled by wall-clock quiet time, which means
|
||||
little in a backfill. That churn is most of the WAL (0.6 TB a day) and the
|
||||
20 k write IOPS bursts, and leaves bloat. Half the rows are single-article
|
||||
releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
||||
(102)).
|
||||
|
||||
1. **One cursor a group (amended 2026-10-26; was: blocks).** A group's
|
||||
history is still fetched as ranges of 2 M articles in chunks of 100 k
|
||||
(the spool, the corpus, the prefetcher stay) and leased as ranges, but
|
||||
a group's ranges are indexed in article order by one worker at a time,
|
||||
and the window below is *carried* from range to range: the next range
|
||||
finishes what the previous one left open. The first design leased
|
||||
blocks of 30 days of posting time, ran a group's blocks in parallel and
|
||||
joined their edges afterwards (a block merger). The user asked why
|
||||
(LOG.md 2026-10-25 (110)): downloading stays parallel through the
|
||||
prefetcher and a Rust thread parses faster than a connection delivers,
|
||||
so one worker a group loses nothing but the end of a reset, when the
|
||||
last big group runs alone. The merger, the block cutting and the
|
||||
`blocks` lease go. Costs: one seam a group where the history meets the
|
||||
tail (the releases in flight then end as two incomplete halves; history
|
||||
runs forward in one pass a group, priority only between groups), and
|
||||
the carried state must survive a restart (5 below). Measured on six
|
||||
consecutive ranges of a.b.ath (2016): one range is 15 hours of posting,
|
||||
so without the carry no release would finish inside a range (LOG.md
|
||||
2026-10-26 (111)).
|
||||
**The group lease (2026-10-26, the user's question in LOG.md (117)):**
|
||||
a worker thread leases a *group*, not a range, and asks the coordinator
|
||||
for the group's next range each time it finished one; the coordinator
|
||||
keeps one cursor a group (`group_leases`: the holder, its heartbeat,
|
||||
the next range in article order, the snapshot it stands at). The
|
||||
answer is the range (id, span, resume point), the rules version and the
|
||||
stream URL — nothing about where the chunks are: the stream serves a
|
||||
stored range's chunks from the corpus file and a live one's from the
|
||||
spool, so a worker needs no mount and no corpus layout, and a remote
|
||||
worker is the same binary with another URL. The overhead is a copy
|
||||
through the node API: the corpus is ~30 B an article compressed (2016;
|
||||
16–18 for 2021 and 2025), eight local workers at the bench's rate are
|
||||
~170 MB/s, which the HDD under the corpus bounds either way; a remote
|
||||
worker is bounded by its link (100 Mbit ≈ 400k articles/s). History
|
||||
therefore no longer waits for the corpus fill: a group's stored ranges
|
||||
are read from disk, an unstored one is fetched live and captured into
|
||||
the corpus on the way, and the prefetcher runs ahead of each held
|
||||
group's cursor (plus the capture work left). A group with no planned
|
||||
range left is the tail: the worker keeps the group and asks again
|
||||
every minute, the planner's tail ranges fall into its order. A lease
|
||||
whose holder went quiet for LEASE_TTL_SECONDS is taken over: the new
|
||||
holder downloads the group's snapshot (5) and redoes the ranges since
|
||||
it; the batch store's keys make the redo idempotent. The replay
|
||||
harness (8) stays for offline runs; the driver on homelab goes once the
|
||||
worker runs.
|
||||
|
||||
2. **The window.** The worker reads a block in article order and keeps its
|
||||
open files and releases in memory (counters, names, sizes; the segment
|
||||
lists spooled to a file per block on the worker's disk). The *frontier*
|
||||
is the median Date of the last 1,024 articles with a plausible date
|
||||
(within 7 days of the frontier; 1,024 such dates in a row that agree
|
||||
among themselves move it, the group having paused); Date headers alone
|
||||
aren't trusted (clock skew, bogus years). 1,024, not more: the median
|
||||
trails the newest article by half the ring, a day in a group of 500
|
||||
articles a day. A release is **finished** when the frontier is 24 h past
|
||||
its last part (`WINDOW_MS`): written once, with its completeness decided
|
||||
then (every announced file there and no file short of parts; par2
|
||||
decides later for those without a counter), never updated by the header
|
||||
pass again. Incomplete releases are kept (user, 2026-10-24). A fill
|
||||
posted after the window is a release of its own (the judge's `repeated`
|
||||
rule already knows reposts). At a lease's end (the tail's poll, a worker
|
||||
stopping; amended 2026-10-26, was: the block's end) the releases the
|
||||
window hasn't closed on are written **provisionally** (`settled_at`
|
||||
null, `finished: false`, flagged `edge`): a release being uploaded
|
||||
shows in the catalog, and the carried window finishes the same row
|
||||
later, since the release key is the identity, the group and the first
|
||||
article; the store's finish path adds the files not sent before. No
|
||||
merger: the carry does the joining. The settler goes: `settled_at` is
|
||||
the finish. The **tail** needs no path of its own: its lease ends leave
|
||||
such provisional releases, the next lease's continuation finishes them.
|
||||
One exception (2026-10-27): a provisional release that is complete
|
||||
(every file its counter announced found) and quiet for six hours is
|
||||
settled by the coordinator (`Settler.settleComplete`, every five
|
||||
minutes), as the finish would: no counted file can still come, and the
|
||||
continuation may be hours away or never (the first run's range leases
|
||||
left 109k provisional rows at their blocks' end edges, 606 of them
|
||||
complete, shown as still collecting; the user, release 10697).
|
||||
|
||||
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
||||
only by a number sequence, or share a directory-like prefix, posted
|
||||
within the window, become one release (`source = 'album'`): its name the
|
||||
shared template (or the prefix), its files kept by name so each stays
|
||||
findable; at least two files, else the single stays a release. This is
|
||||
for subjects that name no release (today's `post`/`stem` singles); a
|
||||
poster's ten different cracks in a day stay ten releases. The stake,
|
||||
from voyager 2's final analysis (LOG.md (112)): 77% of its 83.7 M
|
||||
releases had one file, 50% one segment (0.3% of the segments);
|
||||
a.b.pictures.erotica.porn alone was 27.7 M single-file releases of one
|
||||
poster. A counter total shared by many stems of one poster within the
|
||||
window is a batch counter, not a file count (Yenc-PP-A&A's [n/68067]):
|
||||
`file_total` null for them, not 68,067 incomplete singles.
|
||||
**Built 2026-10-26** (edge/src/indexer.rs, LOG.md (113)): a release the
|
||||
window closed on with one file named by its stem or post, the name with
|
||||
a number and no archive, volume, sidecar or random name, waits in the
|
||||
album of its poster and template (the name's digit runs as `#`, the
|
||||
extension apart: `#_#.jpg.data`) posted within WINDOW_MS of the album's
|
||||
first file, at most ALBUM_MAX_FILES (1,000) a album; the album closes a
|
||||
window after that span and is written as one release (`source: album`,
|
||||
no file total) named by the members' shared numbers (`1481439079025_#
|
||||
.jpg.data`), a lone member as the release it was. The batch store writes
|
||||
the members' names as `release_texts` kind 6 (`member`, the first 20).
|
||||
The directory-like prefix variant is not built (no case seen in the
|
||||
corpus samples). The batch counter: a total ≥ 100 under which one poster
|
||||
started 20+ releases within 100 batches gives none of them a file
|
||||
total. On six a.b.ath ranges of 2016: 46 albums of 790 files among
|
||||
2,099 chunks, most of them a subject parser miss (`[n/60] Shogun I
|
||||
(1980) Shogun I (1980).avi.012`: the counter text taken into the name,
|
||||
a volume each) that the album rule glues back together.
|
||||
|
||||
4. **A slimmer catalog.** `releases` loses its text copies: `poster_id` →
|
||||
`posters(id)` (the table gets an id; 33 B → 4), `group_set_id` →
|
||||
`group_sets` (the distinct group combinations: 139,927 in voyager 2's
|
||||
83.7 M releases, LOG.md (112); 85 B jsonb → 4),
|
||||
`release_key` and `identity_key` as 20-byte `bytea` (41 B each → 20),
|
||||
`source` a smallint. ~150 of ~430 B a row; the poster and key indexes
|
||||
shrink with them (~12 GB of 50 today); `releases_category_newest_idx`
|
||||
(5.3 GB, used once) goes, category browsing filters the newest index.
|
||||
Today's 85 GB would be ~55. Cold tables move to the HDD tablespace:
|
||||
`release_inspections` and `probe_candidates` (28 GB, written once, not
|
||||
read by search); `probe_cache` is there already.
|
||||
|
||||
5. **Memory and disk at the edge.** The busiest group posts ~50 M articles
|
||||
a day; at a file per ~50 articles that is ~1 M open files of ~100 B in
|
||||
memory per block, and their segment lists (~25 B each) in the block's
|
||||
spool file, read back when a release is finished. Per worker thread a
|
||||
few hundred MB at worst. The carried window (amended 2026-10-26) has to
|
||||
survive a restart (deploys restart the workers daily): its counters and
|
||||
names are small (kilobytes for most groups, ~100 MB for the busiest),
|
||||
the open files' segment lists are the bulk (~1.25 GB for a day of the
|
||||
busiest group). A snapshot at every 2 M range would be ~600 B/article
|
||||
of writes, so the state is snapshotted at a lease's end and on a timer,
|
||||
to a file on bulk, and a crash redoes the ranges since the last snapshot
|
||||
from the corpus or the prefetch cache: the deterministic keys make the
|
||||
redo idempotent. **The snapshot goes through the API** (2026-10-26):
|
||||
`PUT /node/groups/:id/snapshot` with the serialized window (the
|
||||
frontier, the open files and releases, the open albums, the upload
|
||||
numbers, the batch-counter table; gzip) and the article it stands at
|
||||
(the last batch's `next`), kept by the coordinator on bulk as
|
||||
`<snapshot dir>/<group id>`; `GET` gives it back to the next holder. A
|
||||
worker snapshots when it gives a group back (a stop) and every
|
||||
SNAPSHOT_MINUTES while it holds one, always right after a batch was
|
||||
stored, so the snapshot's article is a resume point. The replay
|
||||
harness (8) keeps a group's window in one process without one.
|
||||
**Built 2026-10-26** (LOG.md (117)): migration 6500 `group_leases`;
|
||||
`app/services/group_leases.ts` (lease by the most urgent planned
|
||||
range within LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY, the longest free
|
||||
first; takeover after LEASE_TTL_SECONDS, the ranges from the snapshot's
|
||||
article on planned again; next; heartbeat; release; the snapshot
|
||||
streamed to `SNAPSHOT_DIR/<group id>.gz`); the node API's
|
||||
`groups/lease`, `groups/:id/next|heartbeat|release|snapshot`; the
|
||||
range stream serving a stored range's chunks from the corpus; the
|
||||
prefetcher ahead of held groups' cursors, skipping chunks the corpus
|
||||
has; `RangeIndexer::for_group/begin_range/snapshot/restore` (bincode,
|
||||
the files a table, SNAPSHOT_VERSION 1); `edge-node` holding groups by
|
||||
default (`--ranges` keeps the old path). Tests: the lease order,
|
||||
takeover and redo, the endpoints with a corpus-served stream and the
|
||||
snapshot through the API, the snapshot restoring a window between two
|
||||
ranges (the two-range release one, the album joined). Deployed and
|
||||
started 2026-10-26 22:37 (LOG.md (118)). A chunk that doesn't inflate
|
||||
(a live reply cut short; the spool doesn't check) is reported after
|
||||
the batch before it went out and the range asked for again from that
|
||||
chunk: the window stays (the first run lost it, and the open batch).
|
||||
The lease bounds alone don't select a window: the history planner's
|
||||
oldest buckets are priority 0–2 too, so the first run also took 2026
|
||||
history at 0 (25k ranges in 40 groups, fetched live) while 1,739 of
|
||||
the 2016 corpus ranges sat at voyager 2's leftover priorities (done
|
||||
when the window was marked, planned again by the reset): hence
|
||||
`LEASE_CAPTURE_ONLY` (the lease, next and the prefetcher's run-ahead
|
||||
take capture ranges only) and the one-off repair of the priorities.
|
||||
The stream polls the corpus as well as the spool while it waits for a
|
||||
chunk: the corpus fill claims chunks of the same range ahead of the
|
||||
leased run (the prefetcher's claims are per range and chunk), stores
|
||||
them in the corpus only, and the run leaves them there; a stream that
|
||||
looked at the corpus once, seconds before that store, waited its whole
|
||||
hour on the spool and the worker lost the group and its open batch
|
||||
(three groups on the first morning). A failed stream now posts the
|
||||
open batch before the group goes back, like a stop does, so the
|
||||
window's snapshot survives it; only a batch the coordinator refused
|
||||
(lease lost, too large) still loses the window.
|
||||
**Two lanes a group (2026-10-27; migration 6600, LOG.md (119)).** A
|
||||
group's ranges in article order through one window meant a thread
|
||||
that leased a group for its tail range first waded through the
|
||||
group's history (older articles come first), and a held history
|
||||
group's tail waited behind days of its ranges; and the bounds that ran
|
||||
the 2016 window alone (0..0, capture only) shut the tail out entirely.
|
||||
Now `group_leases` has a row per group and *lane*: the `history` lane
|
||||
works the ranges the bounds hand out (the windows, the history), the
|
||||
`tail` lane the tail planner's (priority at or below the tail's,
|
||||
whatever the bounds), each with its own holder, window, snapshot
|
||||
(`<group>-tail.gz`) and takeover redo (the lane's ranges only, and
|
||||
only those the window did after the snapshot: leased, done since it,
|
||||
or read past it). The edge puts `--tail-threads` of its threads on the
|
||||
tail lane (EDGE_TAIL_THREADS; 1 of 2 or more by default, which is
|
||||
what homelab's 16 run with), the rest on history; the prefetcher runs ahead of every held
|
||||
cursor. The seam a group is the same one as before, once: where the
|
||||
history lane reaches the first range the tail lane did, it is caught
|
||||
up. **The lease order** within a lane and priority is a mix (the
|
||||
user, 2026-10-27): one lease starts from the group with the most
|
||||
planned articles, the next from the longest unheld one (never held
|
||||
first), alternating on a sequence (`group_lease_turns`), so a big
|
||||
group snapshotted at a stop doesn't wait behind hundreds of small
|
||||
never-held groups and a quiet group's tail is visited all the same.
|
||||
**`coordinator:status`** (`app/services/status.ts`, `--json`) prints
|
||||
the catalog's room against `CATALOG_MAX_NVME_GB`, the ranges by band
|
||||
(tail, raised, each capture window, the history by thousand), the
|
||||
last hour's indexed and stored articles, the spool, the tail backlog,
|
||||
the held groups with their snapshots' age and size, the snapshot
|
||||
files, and the phase-2 backlogs (provisional releases, probe
|
||||
candidates, the pending ladder, the judge's dirty marks). **End to
|
||||
end** (`tests/e2e`, suite `e2e`, CI builds the binary first): the Rust
|
||||
worker against the real node API, database, spool and corpus — the
|
||||
corpus-fill race, a failed stream and the restart that restores the
|
||||
snapshot, a restart whose snapshot file is gone, a takeover.
|
||||
|
||||
6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
|
||||
(30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
|
||||
those 73% are random 32-hex tokens, 27% `[PRiVATE]` hashes, ~0.3% posts
|
||||
with a readable name and no counter. The filter (crosspost, counterless)
|
||||
loses nothing voyager 3 would index; albums work from filtered chunks
|
||||
(the image dump's subjects carry "(1/1)"). Indexing without a counter at
|
||||
all (obfuscated uploads, ~31% of all lines) would need bodies, and the
|
||||
raw corpus (2,328 ranges) for a sweep.
|
||||
|
||||
7. **Where it lives.** The window, blocks and albums are in the Rust edge
|
||||
(`edge/`), the batch store, the planner and leases; the TypeScript
|
||||
RangeIndexer keeps the scanner (the corpus filter, tests) and drops its
|
||||
parity with the Rust worker for the window (the `check.ts` comparison
|
||||
covered the range path, which goes). New migrations rebuild the schema
|
||||
from a reset, no in-place conversion of the 81 M rows.
|
||||
|
||||
8. **Test before the reset.** Replay the corpus (the raw ranges and the
|
||||
captures, several B articles) with the new edge into a scratch database
|
||||
on bulk: NVMe bytes per article (the one number that decides whether
|
||||
4 TB is enough), the share of albums and incomplete releases, and a
|
||||
sample of releases against today's. Speed is measured after the reset,
|
||||
on the NVMe.
|
||||
|
||||
9. **The reset.** Tail first, then history by priority, read from the
|
||||
corpus where stored. Survives: `probe_cache` (HDD, by message-id),
|
||||
`spots`, `groups` and their priorities, `nntp_shares`, the corpus.
|
||||
Recomputed: releases, chunks, inspections, candidates, par2 data,
|
||||
verdicts, file hashes. Resets are paused (2026-10-12); this one waits
|
||||
for the user's go-ahead, and for the new NVMe.
|
||||
|
||||
### Schema (built 2026-10-25; migrations 6000 and 6100)
|
||||
|
||||
Migration 6000 *is* the reset: it truncates every table the indexing
|
||||
recomputes and refuses to run on a catalog with releases unless
|
||||
`VOYAGER3_RESET=1`, so merging `voyager-3` into `rebuild` is the go-ahead.
|
||||
Survives: `groups`, `ranges` (planned again from their start, with their
|
||||
priorities, `capture`, `est_date`, `stored_at`, `plain_chunks` and measured
|
||||
`posted_low/high`), the corpus, `probe_cache`, `spots` (unlinked from
|
||||
releases; the inspector matches them again), `nntp_shares`, `nodes`,
|
||||
`users`, the rule sets.
|
||||
|
||||
- `releases`: created anew, 8-byte columns first: `id`, `posted_at`,
|
||||
`last_posted_at`, `first_article`, `last_article`, `segments`, `bytes`,
|
||||
`settled_at` (the finish; null while a tail release is provisional),
|
||||
`inspected_at`, `first_seen_at`; `group_id`, `group_set_id`, `poster_id`,
|
||||
`file_total`, `files`, `chunks`, `category`; `source smallint`
|
||||
(`lib/release/source.ts`: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6
|
||||
album); `readable`, `has_par2`, `complete`, `passworded`, `sidecar_only`,
|
||||
`edge` (open at its block's end); `release_key` and `identity_key` as
|
||||
20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`,
|
||||
`last_seen_at`. Indexes: pkey, release_key unique, (identity_key,
|
||||
group_id), newest, (poster_id, newest), passworded, `(group_id,
|
||||
identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS
|
||||
NULL` for the provisional ones. No trigram indexes: search matches
|
||||
`release_texts`. The category trigger reads the groups from
|
||||
`group_sets`; the judge's dirty trigger watches `poster_id` instead of
|
||||
`poster`.
|
||||
- `pending_releases` (migrations 6300 and 6400, the HDD): the unnamed
|
||||
releases outside the catalog (decision f): `release_key` (primary),
|
||||
`identity_key`, `group_id`, `name` (the random stem), `poster` text, the
|
||||
posting span, `first_article`, `last_article`, counts, `file_total`,
|
||||
`partial`, `finished`, `tool`, `candidates` jsonb (the probe targets),
|
||||
`inspections` jsonb (what each probe showed), `named`, `status` (0
|
||||
waiting, 1 named and in the catalog, 2 discarded, 3 exhausted, 4 named
|
||||
awaiting the move, 9 taken), `tries`, `created_at`, `probed_at`; partial
|
||||
indexes on the waiting (by `created_at`) and the named (by `probed_at`).
|
||||
- `pending_signatures` (6400): per (tool, group, poster; poster '' for the
|
||||
tool and group together) `probed`, `named`, `dropped`, `updated_at`.
|
||||
- `release_texts(release_id, kind smallint, text varchar(255))` (6200),
|
||||
primary key on all three, one trigram index (fastupdate): every string
|
||||
search matches against, appended by whoever learns one and never an
|
||||
update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name
|
||||
(the release name, stored once more so one index serves everything),
|
||||
2 file (the prober's names inside an archive), 3 nfo (an NFO's title),
|
||||
4 par2 (a set's name), 5 media (a container's title); at most 20 per
|
||||
release and kind (`textsToStore`). A repeat insert is a no-op; the
|
||||
judge's dirty mark follows an insert or delete, so a rule can react to a
|
||||
name arriving later. Search: the trigram index yields release ids, joined
|
||||
to `releases` by primary key (never an OR across the two tables, which
|
||||
the planner cannot serve from indexes). `inner_name` stays on `releases`
|
||||
as the one display string. Facets to filter on (resolution, codec,
|
||||
language) are not texts: columns, or a `meta jsonb` with a
|
||||
`jsonb_path_ops` index once one is proven useful.
|
||||
- `posters(id serial, poster unique)` with its trigram index;
|
||||
`group_sets(id serial, groups jsonb unique)`, a sorted array of names.
|
||||
- `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume
|
||||
point), `posted_low/high`, `status`, `priority`, `node_id`, `attempts`,
|
||||
`rule_version`, `headers`, `releases`, `leased_at`, `heartbeat_at`,
|
||||
`done_at`, `error`; indexes todo (priority, id) and leased.
|
||||
`ranges.block_id` points at the block; the ranges' own lease columns stay
|
||||
until the leases move (then a migration drops them).
|
||||
- `release_chunks`, `rule_counts`, `rule_samples`: `range_id` → `block_id`.
|
||||
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
|
||||
`settled_at IS NOT NULL`), the settler, the fragment merger,
|
||||
`releases_category_newest_idx`.
|
||||
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
|
||||
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
|
||||
rule read all of `release_inspections` every round, 17 min on the HDD;
|
||||
the judge must read it by release or keep what it needs on the NVMe.
|
||||
|
||||
### Batch store (built 2026-10-25)
|
||||
|
||||
`app/services/batch_store.ts`: every release chunk of a batch is one
|
||||
release, inserted once. Its keys (`lib/release/keys.ts`): `identity_key`
|
||||
the identity's SHA-1 as 20 bytes; `release_key` the SHA-1 of the identity,
|
||||
the owner group and the release's first article, so it is deterministic
|
||||
(no numbering under a group lock, no catalog lookups before writing) and a
|
||||
repost, starting at another article, is a release of its own. Its poster
|
||||
and group set by id (`app/services/catalog_refs.ts`: insert what's new,
|
||||
read the ids, cache them per process; the tests forget the cache per
|
||||
test), its source as a code, its name into `release_texts` (kind 1). The
|
||||
chunk's own counts go in as the release's; `complete` is decided then from
|
||||
the counter. A finished release's probe candidates are claimable at once
|
||||
(`ready`, `unreadable`); a provisional tail release (`finished: false` on
|
||||
the chunk, `settled_at` null) is finished by a later chunk of the same
|
||||
release key, which adds the files not sent before (sums, one more chunk),
|
||||
sets `settled_at` and makes its candidates claimable; a finished release
|
||||
sent again is left as it is (`ON CONFLICT … WHERE settled_at IS NULL`).
|
||||
Gone: the join of a chunk into an earlier release of its key (the window
|
||||
keeps an upload together; the block merger joins what a block's end cut),
|
||||
the join across groups (FTD's spread uploads: a gap, to be handled by the
|
||||
block merger or the par2 merger across groups), the numbering of release
|
||||
keys, the group lock, `fragments` (ignored when sent; the Rust edge still
|
||||
sends them). The chunk type carries `finished` and `edge` (both optional;
|
||||
the Rust edge doesn't send them yet, so its releases are finished, not
|
||||
edges). Interim until blocks are leased: chunks, rule counts and samples
|
||||
are keyed by `ranges.block_id`, and a range without a block gets a block
|
||||
of its own in the store (`blockOf`).
|
||||
|
||||
Readers moved with it: search joins `posters` and `group_sets`, matches
|
||||
words in `release_texts` (`r.id IN (SELECT release_id … ILIKE)`), searches
|
||||
posters by id, filters `source` by code and returns the source name; the
|
||||
par2 merger, the sidecar folder and the judge's scraped rule use
|
||||
`poster_id` (`release_merges.from_key` holds the key as hex); the spot
|
||||
inspector matches by bytea identity keys and inserts its spot releases by
|
||||
id references with their title in `release_texts`; the spotter and the NZB
|
||||
service compare the source code. Tests insert fixtures through
|
||||
`tests/functional/helpers/releases.ts` (`insertRelease`: text keys, poster
|
||||
and groups by name, source by name). The unit and functional suites pass.
|
||||
|
||||
### The window in the Rust edge (built 2026-10-25)
|
||||
|
||||
`edge/src/indexer.rs`: the scanner and the file aggregator as before; then
|
||||
the window instead of the range path. `Frontier` keeps a ring of the last
|
||||
`FRONTIER_ARTICLES` (1,024) plausible dates (within `PLAUSIBLE_MS`, 7
|
||||
days, of the frontier; 1,024 implausible dates in a row that agree move
|
||||
it), recomputed once per chunk; established after 64 dated articles
|
||||
(`settle` at the block's end for smaller blocks); its first value is the
|
||||
block's start in posting time. Per chunk: completed files, then the files
|
||||
whose last part is before frontier − `WINDOW_MS` (24 h; undated ones idle
|
||||
for `idle` chunks), go into the open release of their key whose posting
|
||||
span is within `UPLOAD_GAP_MS` (3 h) of theirs, else start one (the
|
||||
upload number per key, distinct within a batch); releases whose last part
|
||||
is before frontier − `WINDOW_MS` are judged and written `finished`, as an
|
||||
`edge` when posted within `WINDOW_MS` of the block's start. At the end
|
||||
(`finish`) everything left is written: finished when the window closed on
|
||||
it, else `finished: false, edge: true`. A file short of parts stays in
|
||||
its release and sets `partial` (the store: `complete = false`). A lone
|
||||
article of a multi-part random file is still counted `per-article`, not
|
||||
written. Gone: fragments and the cut-off, `uploads_of` (online now),
|
||||
`check.ts` and the fixture comparison (the parity with the TypeScript
|
||||
indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
|
||||
(incomplete releases are kept). The wire carries `finished`, `edge` and
|
||||
`partial`; the store reads them (`partial` into `complete`). Memory: the
|
||||
window holds a day of a group's segments in memory (the busiest group's
|
||||
~50 M); the spool file per block of the design is not built yet.
|
||||
|
||||
### The replay harness (built 2026-10-26)
|
||||
|
||||
`node ace replay:corpus --group <name> [--dir CORPUS_DIR] [--from article]
|
||||
[--to article] [--limit ranges] [--batch-chunks 5] [--no-finish]` (commands/
|
||||
replay_corpus.ts; design 8): a group's complete corpus ranges in article
|
||||
order through one window in the Rust edge (`edge-bench --group <dir>
|
||||
--rules rules.json --wire -`: the window carried from range to range, the
|
||||
end finished after the last range; `edge/src/lease.rs` has the lease JSON
|
||||
the worker and the bench share) and the batch store into a scratch
|
||||
catalog, as the worker would store them: each range leased to a node named
|
||||
`replay`, its batches stored at its resume point (pulled from the bench's
|
||||
stdout as the store keeps up), the range finished when its last batch is
|
||||
in. It refuses a database not named `*_replay` or `*_scratch`.
|
||||
The rules are the scratch catalog's current set and group priorities:
|
||||
copy `groups` and `rule_sets` from the catalog for a faithful replay. The
|
||||
report: releases finished, provisional, edges; complete, incomplete,
|
||||
undecided; readable, with par2; files, segments as a share of the
|
||||
headers; bytes per header of the NVMe tables (releases, release_texts,
|
||||
posters, group_sets, rule counts and samples) and of those on
|
||||
CHUNK_TABLESPACE (release_chunks, probe_candidates, release_inspections).
|
||||
The app image carries `edge-bench` for it (Dockerfile); on homelab the
|
||||
scratch catalog is a second database (`usenet_replay`, default tablespace
|
||||
`nvme` like the catalog, the chunk tables on `pg_default`; migrated with
|
||||
the voyager-3 image built from the deploy cache's repo, `groups` and
|
||||
`rule_sets` copied over), the corpus mounted read-only.
|
||||
`deploy/replay-window.sh` runs a window there: each group whose window
|
||||
ranges are all stored is replayed once with the window's bounds as
|
||||
`--from`/`--to` (the corpus holds other windows of the same group), 4
|
||||
groups at a time, logs and `.done` markers in `.cache/replay/`, looping
|
||||
until the window is done (LOG.md 2026-10-26 (111) has the first local
|
||||
numbers, (112) the homelab run). **Deployed 2026-10-26** (LOG.md (116)):
|
||||
voyager 3 runs on homelab without its edge worker (`.cache/no-edge`:
|
||||
deploy.sh and the nightly resume leave it stopped, since range by range
|
||||
it would split releases at every edge with no merger left), and history
|
||||
comes from the corpus through this driver into the catalog itself
|
||||
(`DB=usenet_rebuild OUT=.cache/replay-catalog`, the harness allowed by
|
||||
`REPLAY_INTO_CATALOG=1`), the prober's ladder, the merger and the judge
|
||||
working on it as it fills; the scratch run's numbers (63 groups) were
|
||||
read before (LOG.md (115)). The tail waits for the window worker.
|
||||
|
||||
Next: the replay on homelab over the 2016 window (the NVMe bytes per
|
||||
header for real), the window worker (design 1 and 5: group leases, the
|
||||
stream serving stored chunks, the snapshot through the API; being built,
|
||||
LOG.md (117)), the prober writing `release_texts` kind 2, the spot/NZB
|
||||
match on pending keys (decision f), the tail's provisional writes at a
|
||||
range end (the window's open releases shown while they upload: the files
|
||||
sent provisionally marked, the finish sending the rest). Gone from the list: block
|
||||
leases, the planner and the block merger (design 1, amended).
|
||||
|
||||
## Decisions
|
||||
|
||||
a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
|
||||
@@ -321,8 +840,67 @@ c. Group list: editable, owner by priority number (above). **Accepted.**
|
||||
d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
|
||||
e. Headers through the coordinator's relay, with prefetch (part 1).
|
||||
**Accepted.**
|
||||
f. Random-named releases (no file with a readable name) keep no chunk in
|
||||
the header pass: their file keys and probe candidates only, until
|
||||
phase 2 names them (bodies) and fetches their segments again. The full
|
||||
history is ~3 TB with them, ~1.2–1.4 TB without (725 M of the 1.25 B
|
||||
kept files; LOG.md 2026-10-09). **Accepted** 2026-10-09, not built yet.
|
||||
f. **Nothing enters the catalog without a usable name** (amended
|
||||
2026-10-26, the user's direction; before: random-named releases keep
|
||||
no chunk but their rows). A release the window closed on whose files
|
||||
all have random names is no release yet: it goes to `pending_releases`
|
||||
(migration 6300, the HDD) with its keys, counts, article span, upload
|
||||
tool and probe targets, no row in `releases`, `release_texts` or
|
||||
`posters`, no chunk. The pending ladder works them **from least to most
|
||||
effort**: (0) no fetch: a signature verdict (tool and group, poster)
|
||||
learned from earlier probes, a spot or NZB that names the message-ids;
|
||||
(1) a par2 main file's first article (it names the files); (2) the first
|
||||
article of the first file (the yEnc name, an archive's listing in its
|
||||
first 16 KiB); (3) a 7z's tail; then nothing more. A probe that yields
|
||||
a usable name moves the release into the catalog, its segments read
|
||||
again from the corpus (the range stored) or the server; one that
|
||||
yields none counts against its signature, and a signature with
|
||||
≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
|
||||
2026-10-08 (5)) is **discarded** from then on, a sample of it still
|
||||
probed to notice a change. The reason, in the 2016 replay: one poster's
|
||||
hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
|
||||
groups) were 76% of the rows and three quarters of the NVMe bytes for
|
||||
11% of the segments and nothing anyone can find; the full history has
|
||||
725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
|
||||
edge sends an unnamed release with an empty payload and its tool, the
|
||||
batch store writes `pending_releases` (LOG.md (114)); the ladder in the
|
||||
prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
|
||||
the oldest waiting releases' next target (par2, archive, lone, 7z-tail
|
||||
in that order), at least PENDING_SHARE (25%) of each claim next to the
|
||||
catalog's own candidates, answered from `probe_cache` when the article
|
||||
was read before; `usableName` (lib/probe/inspect.ts) takes the largest
|
||||
named file of a par2 set or a listing, or a lone file's yEnc name, that
|
||||
is neither packaging nor random; a name sets status 4, no name the next
|
||||
target or status 3 (exhausted); each finished climb counts on
|
||||
`pending_signatures` for (tool, group, poster) and (tool, group), and a
|
||||
signature with PENDING_MIN_PROBES (100) climbs and at most
|
||||
PENDING_MAX_NAMED (0.5%) named is dropped: every five minutes the
|
||||
waiting releases of dropped signatures are discarded (status 2), the
|
||||
rules' `samplePercent` of them left to climb. The namer
|
||||
(`app/services/pending_namer.ts`, in the prober's process): a named
|
||||
release's article span read again from the corpus (the stored range's
|
||||
chunks) or by XOVER on a connection of its own, through the TypeScript
|
||||
indexer with `partialFiles` (a file given up short of parts stays a
|
||||
partial file of its release, as the Rust window writes it; without it
|
||||
the range path's fragment rule left nothing to find: by 2026-10-29
|
||||
about 52k of 210k named releases failed, all partial), written as a release under the pending key with `source`
|
||||
weak-stem, the name as `inner_name`, its chunk and its candidates (the
|
||||
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
|
||||
the password flag). Not built: the spot/NZB match on pending keys (step
|
||||
0 of the ladder beyond signatures), and a signature by poster alone
|
||||
across groups.
|
||||
g. The account's NNTP connections shared by priority (`nntp_shares`): the
|
||||
coordinator (headers) before the prober, each between its minimum and
|
||||
maximum, 5 kept for the spotter, the spot inspector, the plain
|
||||
connection and reconnects. A process grows only into connections the
|
||||
others closed, so the account never goes over its limit. Replaces the
|
||||
fixed split (38 + 8), under which the prober stayed at 8 while history
|
||||
was paused and the coordinator's connections idle (LOG.md 2026-10-18
|
||||
(75)). **Accepted** 2026-10-18.
|
||||
h. Idle NNTP connections closed: one with nothing to do for
|
||||
`NNTP_IDLE_CLOSE_SECONDS` (60) is closed and opened again with work
|
||||
(the coordinator's pool on its next sync, the plain one too; a dry
|
||||
prober lane; the spotter between rounds; the spot inspector with no
|
||||
spots). Eweka dropped the quiet ones and the next command failed
|
||||
("Spot inspection failed: NNTP connection closed"), and idle ones took
|
||||
the account's connections from downloads. **Accepted** 2026-10-22.
|
||||
@@ -91,6 +91,12 @@ export default defineConfig({
|
||||
name: 'functional',
|
||||
timeout: 30000,
|
||||
},
|
||||
// The Rust edge worker against the real node API (tests/e2e): needs edge/target/release/edge-node.
|
||||
{
|
||||
files: ['tests/e2e/**/*.spec.{ts,js}'],
|
||||
name: 'e2e',
|
||||
timeout: 120000,
|
||||
},
|
||||
],
|
||||
forceExit: false,
|
||||
},
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import { Readable } from 'node:stream'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import Leases, { LeaseLost } from '#services/leases'
|
||||
import GroupLeases, { LANES } from '#services/group_leases'
|
||||
import BatchStore, { StaleBatch } from '#services/batch_store'
|
||||
import Spool from '#services/spool'
|
||||
import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
/**
|
||||
* How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
|
||||
* coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
|
||||
* (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
|
||||
*/
|
||||
const CHUNK_WAIT_MS = 60 * 60 * 1000
|
||||
/** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
|
||||
const KEEPALIVE_MS = 20_000
|
||||
|
||||
@@ -30,18 +28,109 @@ export default class NodeController {
|
||||
return lease
|
||||
}
|
||||
|
||||
/** The range's chunks from its resume point, waiting for those still being fetched. */
|
||||
// Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
|
||||
// ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
|
||||
// `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
|
||||
|
||||
/** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
|
||||
private lane(ctx: HttpContext): Lane | null {
|
||||
const lane = ctx.request.qs().lane ?? 'history'
|
||||
if (LANES.includes(lane)) return lane
|
||||
ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
|
||||
return null
|
||||
}
|
||||
|
||||
async leaseGroup(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const lease = await GroupLeases.lease(this.node(ctx), lane)
|
||||
if (!lease) return ctx.response.noContent()
|
||||
return lease
|
||||
}
|
||||
|
||||
async nextRange(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!next) return ctx.response.noContent()
|
||||
return next
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async groupHeartbeat(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
async releaseGroup(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
/**
|
||||
* PUT /groups/:id/snapshot?next=N: the window's serialized state standing at article N, streamed to disk as sent
|
||||
* (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
|
||||
*/
|
||||
async putSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const next = Number(ctx.request.qs().next)
|
||||
if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
|
||||
const raw = ctx.request.raw()
|
||||
const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
|
||||
}
|
||||
|
||||
async getSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
|
||||
ctx.response.header('Content-Type', 'application/octet-stream')
|
||||
ctx.response.header('Content-Length', String(snapshot.size))
|
||||
ctx.response.header('X-Snapshot-Next', String(snapshot.next))
|
||||
return ctx.response.stream(snapshot.stream)
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The range's chunks from its resume point: a stored range's from the corpus (a worker needs no mount; REBUILD.md
|
||||
* design 1), else from the spool, waiting for those still being fetched.
|
||||
*/
|
||||
async stream(ctx: HttpContext) {
|
||||
const nodeId = this.node(ctx)
|
||||
const rangeId = Number(ctx.params.id)
|
||||
const range = await this.leased(ctx, rangeId, nodeId)
|
||||
if (!range) return
|
||||
const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
|
||||
const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
|
||||
const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
|
||||
// The chunk from the corpus or the spool, whichever has it. Both are polled while waiting: the corpus fill
|
||||
// claims a chunk ahead of the leased run (the prefetcher's claims are per range and chunk), stores it in the
|
||||
// corpus only, and the run then leaves it there — a stream that looked at the corpus once, before that
|
||||
// store, waited the whole hour on the spool (three groups on 2026-10-27, LOG.md (118)).
|
||||
const stored = async (span: ChunkSpan) =>
|
||||
corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
|
||||
async function* frames() {
|
||||
for (const span of spans) {
|
||||
const waitUntil = Date.now() + CHUNK_WAIT_MS
|
||||
const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
|
||||
let quietSince = Date.now()
|
||||
while (!(await Spool.has(rangeId, span))) {
|
||||
let data = await stored(span)
|
||||
while (!data) {
|
||||
if (await Spool.has(rangeId, span)) {
|
||||
data = await Spool.read(rangeId, span)
|
||||
break
|
||||
}
|
||||
if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
|
||||
if (Date.now() - quietSince >= KEEPALIVE_MS) {
|
||||
quietSince = Date.now()
|
||||
@@ -49,8 +138,9 @@ export default class NodeController {
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
await sleep(500)
|
||||
data = await stored(span)
|
||||
}
|
||||
yield encodeFrame({ ...span, data: await Spool.read(rangeId, span) })
|
||||
yield encodeFrame({ ...span, data })
|
||||
// Streaming keeps the lease alive.
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
@@ -75,7 +165,11 @@ export default class NodeController {
|
||||
}
|
||||
|
||||
async finish(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, (rangeId, nodeId) => Leases.finish(rangeId, nodeId))
|
||||
return this.lifecycle(ctx, async (rangeId, nodeId) => {
|
||||
const range = await Leases.leased(rangeId, nodeId)
|
||||
await Leases.finish(rangeId, nodeId)
|
||||
await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */
|
||||
|
||||
+276
-181
@@ -1,24 +1,57 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
|
||||
import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import CatalogRefs, { groupSetKey, posterText } from '#services/catalog_refs'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { SIDECAR } from '#services/sidecar_folder'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
import { uploadTool } from '#lib/rules/index'
|
||||
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
|
||||
|
||||
/**
|
||||
* A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
|
||||
* fold's candidates (releases.sidecar_only). Its file list decoded here, small as it is.
|
||||
*/
|
||||
function sidecarOnly(r: ReleaseChunk): boolean | null {
|
||||
if (r.files > 3 || r.bytes >= 1_000_000) return null
|
||||
const files = decodeChunk(r.payload)
|
||||
return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
|
||||
}
|
||||
|
||||
export class StaleBatch extends Error {}
|
||||
|
||||
export interface StoredBatch {
|
||||
/** Release chunks in the batch. */
|
||||
releases: number
|
||||
/** Releases the batch started. */
|
||||
newReleases: number
|
||||
/** Provisional (tail) releases the batch finished. */
|
||||
finalized: number
|
||||
candidates: number
|
||||
fragments: number
|
||||
/** Unnamed releases written to the pending store instead of the catalog (decision f). */
|
||||
pending: number
|
||||
}
|
||||
|
||||
/** A chunk's slot in its batch: the key and the upload the edge numbered it with (ProbeCandidate points at it). */
|
||||
const slot = (key: string, upload: number) => `${key}\n${upload}`
|
||||
|
||||
/**
|
||||
* Stores one batch of a leased range (REBUILD.md, parts 3–7), in one
|
||||
* transaction: release aggregates in the catalog, one chunk per release,
|
||||
* probe candidates, rule counts and samples, edge fragments, and the
|
||||
* range's resume point. A batch that doesn't start at the resume point was
|
||||
* Stores one batch of a leased range (REBUILD.md, "Voyager 3: window indexing", 2 and 4) in one transaction: each
|
||||
* release chunk is a whole release the edge's window finished, written once (its poster and group set by id, its
|
||||
* keys as bytea, its name into release_texts), with one chunk of its files, its probe candidates, and the batch's
|
||||
* rule counts and samples; then the range's resume point moves. A batch that doesn't start at the resume point was
|
||||
* sent twice or too late, and is refused (StaleBatch), so the sums stay right.
|
||||
*
|
||||
* A tail release comes twice: provisionally (finished: false, settled_at null, its candidates waiting) and then
|
||||
* finished, under the same release key (the identity, the group and its first article: lib/release/keys.ts), with
|
||||
* the files not sent before (a chunk's counts are its own; the finish adds them up); the finish is the one update a
|
||||
* release gets from the header pass. A finished release sent again is left as it is.
|
||||
*
|
||||
* Interim, until blocks are leased (the next step): chunks, counts and samples are keyed by the range's block
|
||||
* (ranges.block_id), and a range without one gets a block of its own here.
|
||||
*/
|
||||
class BatchStore {
|
||||
async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
|
||||
@@ -27,7 +60,9 @@ class BatchStore {
|
||||
// One batch of a range at a time, without holding the range's row for the whole transaction: the
|
||||
// prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
|
||||
// LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [rangeId])
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [
|
||||
rangeId,
|
||||
])
|
||||
const range = await trx.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
|
||||
@@ -35,35 +70,48 @@ class BatchStore {
|
||||
throw new StaleBatch(
|
||||
`Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
|
||||
)
|
||||
const blockId = await this.blockOf(trx, range)
|
||||
const result: StoredBatch = {
|
||||
releases: batch.releases.length,
|
||||
newReleases: 0,
|
||||
finalized: 0,
|
||||
candidates: 0,
|
||||
fragments: batch.fragments.length,
|
||||
pending: 0,
|
||||
}
|
||||
|
||||
// Nothing enters the catalog without a usable name (decision f): an unnamed release (no readable file name)
|
||||
// goes to the pending store with its probe targets, the named ones to the catalog.
|
||||
const named = batch.releases.filter((r) => r.readable)
|
||||
const unnamed = batch.releases.filter((r) => !r.readable)
|
||||
const written = await this.writeReleases(
|
||||
trx,
|
||||
rangeId,
|
||||
{ groupId: Number(range.group_id), blockId },
|
||||
batch.from,
|
||||
batch.releases,
|
||||
named,
|
||||
batch.candidates
|
||||
)
|
||||
result.newReleases = written.newReleases
|
||||
result.finalized = written.finalized
|
||||
result.candidates = written.candidates
|
||||
result.pending = await this.writePending(
|
||||
trx,
|
||||
Number(range.group_id),
|
||||
unnamed,
|
||||
batch.candidates
|
||||
)
|
||||
|
||||
if (batch.counts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (range_id, rule, key, articles, files)
|
||||
`INSERT INTO rule_counts (block_id, rule, key, articles, files)
|
||||
SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
|
||||
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
|
||||
[rangeId, JSON.stringify(batch.counts)]
|
||||
ON CONFLICT (block_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
|
||||
[blockId, JSON.stringify(batch.counts)]
|
||||
)
|
||||
}
|
||||
|
||||
for (const s of batch.samples) {
|
||||
await trx.insertQuery().table('rule_samples').insert({
|
||||
range_id: rangeId,
|
||||
block_id: blockId,
|
||||
rule: s.rule,
|
||||
rule_key: s.ruleKey,
|
||||
release_key: s.key,
|
||||
@@ -74,18 +122,6 @@ class BatchStore {
|
||||
})
|
||||
}
|
||||
|
||||
for (let i = 0; i < batch.fragments.length; i += 2000) {
|
||||
const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
|
||||
range_id: rangeId,
|
||||
file_key: f.fileKey,
|
||||
// Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
|
||||
article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
|
||||
segments: f.segments.length,
|
||||
fragment: JSON.stringify(f),
|
||||
}))
|
||||
await trx.insertQuery().table('fragments').multiInsert(rows)
|
||||
}
|
||||
|
||||
const moved = await trx
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
@@ -95,106 +131,138 @@ class BatchStore {
|
||||
.update({
|
||||
next_id: batch.next,
|
||||
headers: db.raw('headers + ?', [batch.headers]),
|
||||
// The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
|
||||
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
|
||||
// The range's posting time span: the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedLow ?? null,
|
||||
]),
|
||||
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedHigh ?? null,
|
||||
]),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1))
|
||||
throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* An upload spread over groups (FTD: par2 and part1 to a.b.music.misc,
|
||||
* part2 to a.b.sounds.music, each article in one group only; LOG.md
|
||||
* 2026-10-11 (9)): a chunk without a release in its own group joins one of
|
||||
* its key in another group whose window is within UPLOAD_GAP_MS, when none
|
||||
* of its file names are in that release yet. A post of the same files to
|
||||
* another group (other message-ids, the same names) stays a release of its
|
||||
* own, or the NZB would hold every file twice. Reads the candidate's file
|
||||
* names from its chunks; candidates are few.
|
||||
*/
|
||||
private async joinAcrossGroups(
|
||||
trx: TransactionClientContract,
|
||||
part: ReleaseChunk[],
|
||||
windows: { idx: number; key: string; first: number | null | undefined; last: number | null | undefined }[],
|
||||
groupId: number,
|
||||
joined: Map<number, number>
|
||||
) {
|
||||
const open = windows.filter((w) => !joined.has(w.idx) && w.first)
|
||||
if (open.length === 0) return
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT v.idx, r.id
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
|
||||
JOIN releases r ON r.identity_key = v.key AND r.group_id <> ?
|
||||
WHERE r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0)
|
||||
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
|
||||
[JSON.stringify(open), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
|
||||
)
|
||||
if (candidates.length === 0) return
|
||||
const ids: number[] = [...new Set<number>(candidates.map((c: { id: string }) => Number(c.id)))]
|
||||
const names = new Map<number, Set<string>>()
|
||||
const { rows: chunks } = await trx.rawQuery(`SELECT release_id, payload FROM release_chunks WHERE release_id = ANY(?)`, [ids])
|
||||
for (const c of chunks) {
|
||||
const set = names.get(Number(c.release_id)) ?? new Set<string>()
|
||||
for (const f of decodeChunk(c.payload)) set.add(f.name)
|
||||
names.set(Number(c.release_id), set)
|
||||
}
|
||||
for (const c of candidates as { idx: number; id: string }[]) {
|
||||
if (joined.has(c.idx)) continue
|
||||
const have = names.get(Number(c.id)) ?? new Set<string>()
|
||||
const incoming = decodeChunk(part[c.idx].payload).map((f) => f.name)
|
||||
if (incoming.every((name) => !have.has(name))) {
|
||||
joined.set(c.idx, Number(c.id))
|
||||
for (const name of incoming) have.add(name)
|
||||
names.set(Number(c.id), have)
|
||||
}
|
||||
}
|
||||
/** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
|
||||
async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
|
||||
if (range.block_id) return Number(range.block_id)
|
||||
const [block] = await trx
|
||||
.table('blocks')
|
||||
.insert({
|
||||
group_id: range.group_id,
|
||||
start_id: range.start_id,
|
||||
end_id: range.end_id,
|
||||
next_id: range.next_id,
|
||||
status: 'leased',
|
||||
priority: range.priority ?? 0,
|
||||
node_id: range.node_id,
|
||||
leased_at: range.leased_at ?? new Date(),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
.returning('id')
|
||||
await trx.from('ranges').where('id', range.id).update({ block_id: block.id })
|
||||
return Number(block.id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Release aggregates in the catalog, one write-once chunk per release, and
|
||||
* probe candidates (a batch's, or files the fragment merger completed).
|
||||
* A chunk joins the release of its key whose articles the range's group
|
||||
* owns and whose posting window is within UPLOAD_GAP_MS of its own, else
|
||||
* starts one: re-uploads are releases of their own (decision d). The
|
||||
* group is locked, so concurrent writers don't both start one.
|
||||
* The unnamed releases of a batch into pending_releases (migration 6300, the HDD): keys, counts, the upload
|
||||
* tool and the probe targets, no chunk. The pending ladder (REBUILD.md decision f) probes them cheapest first,
|
||||
* and a name moves one into the catalog; a provisional tail release sent again adds its counts up.
|
||||
*/
|
||||
async writePending(
|
||||
trx: TransactionClientContract,
|
||||
groupId: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
): Promise<number> {
|
||||
if (releases.length === 0) return 0
|
||||
const targets = new Map<string, { kind: string; messageId: string; file: string }[]>()
|
||||
for (const c of candidates) {
|
||||
const key = slot(c.releaseKey, c.upload ?? 0)
|
||||
const list = targets.get(key) ?? []
|
||||
list.push({ kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
targets.set(key, list)
|
||||
}
|
||||
let written = 0
|
||||
for (let i = 0; i < releases.length; i += 1000) {
|
||||
const rows = releases.slice(i, i + 1000).map((r) => {
|
||||
const probes = targets.get(slot(r.key, r.upload)) ?? []
|
||||
return {
|
||||
release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name.slice(0, TEXT_MAX),
|
||||
poster: r.poster || null,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
first_article: r.firstArticle,
|
||||
last_article: r.lastArticle,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
bytes: r.bytes,
|
||||
partial: r.partial === true,
|
||||
finished: r.finished !== false,
|
||||
file_total: r.fileTotal,
|
||||
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
|
||||
candidates: probes,
|
||||
}
|
||||
})
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
|
||||
files, segments, bytes, partial, finished, file_total, tool, candidates)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
|
||||
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
|
||||
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
|
||||
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
|
||||
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
|
||||
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END
|
||||
WHERE NOT pending_releases.finished
|
||||
RETURNING release_key`,
|
||||
[groupId, JSON.stringify(rows)]
|
||||
)
|
||||
written += done.length
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases of a batch into the catalog, one chunk each, and their probe candidates. Every chunk is one
|
||||
* release: inserted, or (a provisional tail release sent again) finished.
|
||||
*/
|
||||
async writeReleases(
|
||||
trx: TransactionClientContract,
|
||||
rangeId: number,
|
||||
where: { groupId: number; blockId: number },
|
||||
batchFrom: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
) {
|
||||
const out = { newReleases: 0, candidates: 0 }
|
||||
const out = { newReleases: 0, finalized: 0, candidates: 0 }
|
||||
if (releases.length === 0) return out
|
||||
/** Release ids by slot, for the chunks and candidates; a slot not here was a finished release sent again. */
|
||||
const ids = new Map<string, number>()
|
||||
const slot = (key: string, upload: number) => `${key}\n${upload}`
|
||||
const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
|
||||
// One writer per group at a time: a chunk only matches releases of its group, and a lock per key
|
||||
// ran out of shared memory (a picture group's batch has 50k keys; LOG.md 2026-10-09 (12)).
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('releases:' || ?::text, 0))`, [groupId])
|
||||
const finalized = new Set<number>()
|
||||
|
||||
const posters = await CatalogRefs.posterIds(trx, releases.map((r) => r.poster).filter(Boolean))
|
||||
const groupSets = await CatalogRefs.groupSetIds(
|
||||
trx,
|
||||
releases.map((r) => r.groups)
|
||||
)
|
||||
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
|
||||
for (let i = 0; i < sorted.length; i += 1000) {
|
||||
const part = sorted.slice(i, i + 1000)
|
||||
const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
|
||||
const { rows: matches } = await trx.rawQuery(
|
||||
`SELECT DISTINCT ON (v.idx) v.idx, r.id
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
|
||||
JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
|
||||
WHERE v.first IS NULL OR r.posted_at IS NULL
|
||||
OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
|
||||
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
|
||||
[JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
|
||||
)
|
||||
const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
|
||||
await this.joinAcrossGroups(trx, part, windows, groupId, joined)
|
||||
const row = (r: ReleaseChunk) => ({
|
||||
const rows = part.map((r, idx) => ({
|
||||
idx,
|
||||
release_key: releaseKey(r.key, where.groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceCode(r.source),
|
||||
poster_id: r.poster ? (posters.get(posterText(r.poster)) ?? null) : null,
|
||||
group_set_id: groupSets.get(groupSetKey(r.groups))!,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
file_total: r.fileTotal,
|
||||
@@ -205,98 +273,114 @@ class BatchStore {
|
||||
last_article: r.lastArticle,
|
||||
readable: r.readable,
|
||||
has_par2: r.hasPar2,
|
||||
})
|
||||
const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
|
||||
if (updates.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
|
||||
a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
|
||||
max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
|
||||
FROM v GROUP BY id)
|
||||
UPDATE releases r SET
|
||||
files = r.files + a.files,
|
||||
segments = r.segments + a.segments,
|
||||
bytes = r.bytes + a.bytes,
|
||||
chunks = r.chunks + a.chunks,
|
||||
posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
|
||||
last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
|
||||
file_total = coalesce(r.file_total, a.file_total),
|
||||
first_article = least(r.first_article, a.first_article),
|
||||
last_article = greatest(r.last_article, a.last_article),
|
||||
readable = r.readable OR a.readable,
|
||||
has_par2 = r.has_par2 OR a.has_par2,
|
||||
-- As it stands (the counter's files all found); settling makes it final.
|
||||
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
|
||||
last_seen_at = now()
|
||||
FROM a WHERE r.id = a.id`,
|
||||
[JSON.stringify(updates)]
|
||||
finished: r.finished !== false,
|
||||
edge: r.edge === true,
|
||||
partial: r.partial === true,
|
||||
sidecar_only: sidecarOnly(r),
|
||||
}))
|
||||
// Inserted, or a provisional release of the key finished (the chunk's files added up); a finished release
|
||||
// sent again (no row returned) is left as it is.
|
||||
const { rows: written } = await trx.rawQuery(
|
||||
`INSERT INTO releases (release_key, identity_key, group_id, group_set_id, poster_id, name, source, posted_at, last_posted_at, file_total, files, segments, bytes, chunks,
|
||||
first_article, last_article, readable, has_par2, complete, edge, sidecar_only, settled_at)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.group_set_id, v.poster_id, v.name, v.source,
|
||||
to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
|
||||
v.first_article, v.last_article, v.readable, v.has_par2,
|
||||
CASE WHEN v.partial THEN false WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END,
|
||||
v.edge, v.sidecar_only, CASE WHEN v.finished THEN now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, release_key text, identity_key text, name text, source int, poster_id int, group_set_id int, first bigint, last bigint,
|
||||
file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean,
|
||||
finished boolean, edge boolean, partial boolean, sidecar_only boolean)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = releases.files + excluded.files, segments = releases.segments + excluded.segments, bytes = releases.bytes + excluded.bytes,
|
||||
chunks = releases.chunks + CASE WHEN excluded.files > 0 THEN 1 ELSE 0 END,
|
||||
posted_at = least(releases.posted_at, excluded.posted_at), last_posted_at = greatest(releases.last_posted_at, excluded.last_posted_at),
|
||||
file_total = coalesce(excluded.file_total, releases.file_total),
|
||||
first_article = least(releases.first_article, excluded.first_article), last_article = greatest(releases.last_article, excluded.last_article),
|
||||
readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2,
|
||||
complete = CASE WHEN releases.complete IS FALSE AND releases.file_total IS NULL THEN false
|
||||
WHEN coalesce(excluded.file_total, releases.file_total) IS NULL THEN releases.complete
|
||||
ELSE releases.files + excluded.files >= coalesce(excluded.file_total, releases.file_total) END,
|
||||
edge = excluded.edge, sidecar_only = coalesce(releases.sidecar_only, excluded.sidecar_only), settled_at = excluded.settled_at
|
||||
WHERE releases.settled_at IS NULL
|
||||
RETURNING id, encode(release_key, 'hex') AS release_key, (xmax = 0) AS inserted, settled_at`,
|
||||
[where.groupId, JSON.stringify(rows)]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; inserted: boolean; settled: boolean }>(
|
||||
written.map(
|
||||
(w: { id: string; release_key: string; inserted: boolean; settled_at: Date | null }) => [
|
||||
w.release_key,
|
||||
{ id: Number(w.id), inserted: w.inserted, settled: w.settled_at !== null },
|
||||
]
|
||||
)
|
||||
part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
|
||||
}
|
||||
const fresh = part.filter((_, idx) => !joined.has(idx))
|
||||
if (fresh.length > 0) {
|
||||
// The first upload of a key keeps the key; later ones are key:2, key:3, …
|
||||
const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
|
||||
const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
|
||||
const numbered = (r: ReleaseChunk) => {
|
||||
const n = (taken.get(r.key) ?? 0) + 1
|
||||
taken.set(r.key, n)
|
||||
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
|
||||
)
|
||||
const texts: { release_id: number; kind: number; text: string }[] = []
|
||||
for (const row of rows) {
|
||||
const w = byKey.get(row.release_key)
|
||||
if (!w) continue
|
||||
const r = part[row.idx]
|
||||
ids.set(slot(r.key, r.upload), w.id)
|
||||
if (w.inserted) out.newReleases++
|
||||
else if (w.settled) {
|
||||
out.finalized++
|
||||
finalized.add(w.id)
|
||||
}
|
||||
// Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
|
||||
const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
|
||||
if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
|
||||
let inserts = fresh.map(numbered)
|
||||
// Writers of other groups number the same key at the same time (a spam post crossposted to 67
|
||||
// groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
|
||||
for (let attempt = 0; inserts.length > 0; attempt++) {
|
||||
if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
|
||||
const { rows } = await trx.rawQuery(
|
||||
`INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
|
||||
SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
|
||||
v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
|
||||
ON CONFLICT (release_key) DO NOTHING
|
||||
RETURNING id, release_key`,
|
||||
[groupId, JSON.stringify(inserts)]
|
||||
)
|
||||
const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
|
||||
const missed: typeof inserts = []
|
||||
for (const r of inserts) {
|
||||
const id = byReleaseKey.get(r.release_key)
|
||||
if (id === undefined) missed.push(r)
|
||||
else ids.set(slot(r.key, r.upload), id)
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.name, text: row.name.slice(0, TEXT_MAX) })
|
||||
// An album's members stay findable by their own names (the first TEXTS_PER_KIND of them).
|
||||
if (r.source === 'album' && r.files > 0) {
|
||||
for (const text of textsToStore(decodeChunk(r.payload).map((f) => f.name))) {
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.member, text })
|
||||
}
|
||||
out.newReleases += rows.length
|
||||
const chunks = new Map(fresh.map((r) => [slot(r.key, r.upload), r]))
|
||||
inserts = missed.map((r) => numbered(chunks.get(slot(r.key, r.upload))!))
|
||||
}
|
||||
}
|
||||
// The name (and an album's members), searchable (release_texts; a text sent again is a no-op).
|
||||
if (texts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_texts (release_id, kind, text)
|
||||
SELECT v.release_id, v.kind, v.text FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind int, text text) WHERE v.text <> ''
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[JSON.stringify(texts)]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// One write-once chunk per release.
|
||||
// One write-once chunk per release (a finish adds the files not sent before).
|
||||
for (let i = 0; i < sorted.length; i += 500) {
|
||||
const rows = sorted.slice(i, i + 500).map((r) => ({
|
||||
release_id: ids.get(slot(r.key, r.upload))!,
|
||||
range_id: rangeId,
|
||||
batch_from: batchFrom,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
payload: r.payload,
|
||||
}))
|
||||
const rows = sorted
|
||||
.slice(i, i + 500)
|
||||
.filter((r) => ids.has(slot(r.key, r.upload)) && r.files > 0)
|
||||
.map((r) => ({
|
||||
release_id: ids.get(slot(r.key, r.upload))!,
|
||||
block_id: where.blockId,
|
||||
batch_from: batchFrom,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
payload: r.payload,
|
||||
}))
|
||||
if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
|
||||
}
|
||||
|
||||
// A finished release's candidates are claimable at once; a provisional one's wait for its finish.
|
||||
const settled = new Set<number>()
|
||||
for (const r of sorted) {
|
||||
const id = ids.get(slot(r.key, r.upload))
|
||||
if (id !== undefined && r.finished !== false) settled.add(id)
|
||||
}
|
||||
const chunks = new Map(sorted.map((r) => [slot(r.key, r.upload), r]))
|
||||
const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
|
||||
for (let i = 0; i < kept.length; i += 2000) {
|
||||
const rows = kept
|
||||
.slice(i, i + 2000)
|
||||
.map((c) => ({
|
||||
release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
|
||||
const rows = kept.slice(i, i + 2000).map((c) => {
|
||||
const releaseId = ids.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
const chunk = chunks.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
return {
|
||||
release_id: releaseId,
|
||||
kind: c.kind,
|
||||
message_id: c.messageId,
|
||||
file: c.file,
|
||||
}))
|
||||
ready: settled.has(releaseId),
|
||||
unreadable: !chunk.readable,
|
||||
}
|
||||
})
|
||||
const added = await trx
|
||||
.knexQuery()
|
||||
.table('probe_candidates')
|
||||
@@ -305,6 +389,17 @@ class BatchStore {
|
||||
.ignore()
|
||||
.returning('id')
|
||||
out.candidates += added.length
|
||||
// A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
|
||||
const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
|
||||
if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
|
||||
}
|
||||
// The finish of a provisional release: its waiting candidates become claimable.
|
||||
if (finalized.size > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[[...finalized]]
|
||||
)
|
||||
}
|
||||
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import type db from '@adonisjs/lucid/services/db'
|
||||
|
||||
type Client = Pick<typeof db, 'rawQuery'>
|
||||
|
||||
/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
|
||||
export const POSTER_MAX = 200
|
||||
/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
|
||||
const CACHE_MAX = 200_000
|
||||
|
||||
const posterCache = new Map<string, number>()
|
||||
const groupSetCache = new Map<string, number>()
|
||||
|
||||
/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
|
||||
export function groupSetKey(groups: string[]): string {
|
||||
return JSON.stringify([...new Set(groups)].sort())
|
||||
}
|
||||
|
||||
/** A poster as posters stores it. */
|
||||
export function posterText(poster: string): string {
|
||||
return poster.slice(0, POSTER_MAX)
|
||||
}
|
||||
|
||||
async function ids(
|
||||
client: Client,
|
||||
cache: Map<string, number>,
|
||||
table: 'posters' | 'group_sets',
|
||||
column: 'poster' | 'groups',
|
||||
cast: 'text' | 'jsonb',
|
||||
values: string[]
|
||||
): Promise<Map<string, number>> {
|
||||
const out = new Map<string, number>()
|
||||
const missing: string[] = []
|
||||
for (const v of values) {
|
||||
const id = cache.get(v)
|
||||
if (id === undefined) missing.push(v)
|
||||
else out.set(v, id)
|
||||
}
|
||||
if (missing.length === 0) return out
|
||||
// Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
|
||||
await client.rawQuery(
|
||||
`INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
|
||||
[missing]
|
||||
)
|
||||
const { rows } = await client.rawQuery(
|
||||
`SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
|
||||
[missing]
|
||||
)
|
||||
if (cache.size + rows.length > CACHE_MAX) cache.clear()
|
||||
for (const r of rows) {
|
||||
// jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
|
||||
const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
|
||||
cache.set(key, Number(r.id))
|
||||
out.set(key, Number(r.id))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
|
||||
* values are inserted; known ones come from a cache per process (the batch store sees the same posters and
|
||||
* group sets batch after batch).
|
||||
*/
|
||||
export default {
|
||||
/** Ids by poster (posterText applied), for a batch of posters. */
|
||||
async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
|
||||
return ids(client, posterCache, 'posters', 'poster', 'text', [
|
||||
...new Set(posters.map(posterText)),
|
||||
])
|
||||
},
|
||||
|
||||
async posterId(client: Client, poster: string): Promise<number> {
|
||||
const found = await this.posterIds(client, [poster])
|
||||
return found.get(posterText(poster))!
|
||||
},
|
||||
|
||||
/** Ids by groupSetKey, for a batch of group lists. */
|
||||
async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
|
||||
return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
|
||||
...new Set(sets.map(groupSetKey)),
|
||||
])
|
||||
},
|
||||
|
||||
async groupSetId(client: Client, groups: string[]): Promise<number> {
|
||||
const found = await this.groupSetIds(client, [groups])
|
||||
return found.get(groupSetKey(groups))!
|
||||
},
|
||||
|
||||
/** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
|
||||
forget() {
|
||||
posterCache.clear()
|
||||
groupSetCache.clear()
|
||||
},
|
||||
}
|
||||
+45
-15
@@ -1,11 +1,24 @@
|
||||
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzip } from 'node:zlib'
|
||||
import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
|
||||
import { promisify } from 'node:util'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { filterOverviews } from '#lib/edge/index'
|
||||
import type { ChunkSpan } from '#services/spool'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
const deflate = promisify(deflateCb)
|
||||
const gzip = promisify(gzipCb)
|
||||
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
|
||||
const MAX_FILTER_BYTES = 256 * 1024 * 1024
|
||||
|
||||
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
|
||||
export interface CorpusFilter {
|
||||
ruleVersion: number
|
||||
priorities: Record<string, number>
|
||||
}
|
||||
/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
|
||||
export const FILTERED = ['crosspost', 'obfuscated']
|
||||
|
||||
export interface CorpusRange {
|
||||
group: string
|
||||
@@ -19,20 +32,23 @@ export interface CorpusRange {
|
||||
* every chunk is there, the layout edge-bench reads (edge/fixtures). The
|
||||
* grouping sweeps replay them with each build's rules instead of waiting for
|
||||
* a re-index; the prefetcher reads a stored chunk instead of fetching it
|
||||
* again. Off without CORPUS_DIR.
|
||||
* again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
|
||||
* (101)): the lines the edge worker only counts are left out, ~half of them,
|
||||
* and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
|
||||
* article numbers by rule; its range.json says `filtered`.
|
||||
*/
|
||||
class Corpus {
|
||||
get dir(): string | undefined {
|
||||
return coordinatorConfig.corpus.dir
|
||||
}
|
||||
|
||||
private rangeDir(range: CorpusRange) {
|
||||
return join(this.dir!, range.group, `${range.start}-${range.end}`)
|
||||
private rangeDir(range: CorpusRange, dir = this.dir!) {
|
||||
return join(dir, range.group, `${range.start}-${range.end}`)
|
||||
}
|
||||
|
||||
private path(range: CorpusRange, span: ChunkSpan) {
|
||||
private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
|
||||
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
|
||||
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
/** Whether every chunk of the range is stored (its range.json). */
|
||||
@@ -53,36 +69,50 @@ class Corpus {
|
||||
)
|
||||
}
|
||||
|
||||
/** A stored chunk, or null. */
|
||||
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
|
||||
if (!this.dir) return null
|
||||
return readFile(this.path(range, span)).catch(() => null)
|
||||
/** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
|
||||
async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
|
||||
if (!dir) return null
|
||||
return readFile(this.path(range, span, dir)).catch(() => null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores a chunk that inflates (a reply cut short isn't kept), and the
|
||||
* range.json once the range is complete (`total` chunks).
|
||||
* range.json once the range is complete (`total` chunks). With `filter`,
|
||||
* without the lines the edge worker drops on their own, compressed again
|
||||
* as the news server's (zlib), with its `.drops` written first.
|
||||
*/
|
||||
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
|
||||
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
|
||||
if (!this.dir) return
|
||||
let raw: Buffer
|
||||
try {
|
||||
await inflate(data)
|
||||
raw = await inflate(data)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const dir = this.rangeDir(range)
|
||||
const file = this.path(range, span)
|
||||
await mkdir(dir, { recursive: true })
|
||||
await writeFile(`${file}.part`, data)
|
||||
let stored = data
|
||||
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
|
||||
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
|
||||
if (filter && raw.length < MAX_FILTER_BYTES) {
|
||||
// Latin-1 both ways: every byte back as it was.
|
||||
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
|
||||
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
|
||||
await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
|
||||
await rename(`${file}.drops.part`, `${file}.drops`)
|
||||
}
|
||||
await writeFile(`${file}.part`, stored)
|
||||
await rename(`${file}.part`, file)
|
||||
const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
|
||||
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
|
||||
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
|
||||
}
|
||||
|
||||
/** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
|
||||
async removeChunk(range: CorpusRange, span: ChunkSpan) {
|
||||
if (!this.dir) return
|
||||
await rm(this.path(range, span), { force: true })
|
||||
await rm(`${this.path(range, span)}.drops`, { force: true })
|
||||
await rm(join(this.rangeDir(range), 'range.json'), { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { statfs } from 'node:fs/promises'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** How often free space is measured. */
|
||||
const CHECK_MS = 60_000
|
||||
/** Resumed only this much above the floor: not paused and resumed at every flush. */
|
||||
const MARGIN_GB = 1
|
||||
|
||||
/**
|
||||
* Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a
|
||||
* container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on.
|
||||
* The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md
|
||||
* 2026-10-18 (77)). Off when the floor is 0.
|
||||
*/
|
||||
export class DiskSpace {
|
||||
private checkedAt = 0
|
||||
private low = false
|
||||
|
||||
constructor(
|
||||
private what: string,
|
||||
private path: string,
|
||||
private floorGb: number,
|
||||
private free: (path: string) => Promise<number> = async (p) => {
|
||||
const s = await statfs(p)
|
||||
return s.bavail * s.bsize
|
||||
},
|
||||
private checkMs = CHECK_MS
|
||||
) {}
|
||||
|
||||
async short(): Promise<boolean> {
|
||||
if (!this.floorGb) return false
|
||||
if (Date.now() - this.checkedAt < this.checkMs) return this.low
|
||||
this.checkedAt = Date.now()
|
||||
const gb = (await this.free(this.path)) / 1e9
|
||||
const was = this.low
|
||||
this.low = gb < this.floorGb + (was ? MARGIN_GB : 0)
|
||||
if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`)
|
||||
return this.low
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { RuleSet } from '#lib/rules/index'
|
||||
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
|
||||
import type { AggregatedFile } from '#lib/files/index'
|
||||
import type { EdgeSegment } from '#lib/edge/index'
|
||||
|
||||
export interface MergeResult {
|
||||
files: number
|
||||
completed: number
|
||||
incomplete: number
|
||||
waiting: number
|
||||
/** This round reached the end of the file keys (the next one starts over). */
|
||||
wrapped: boolean
|
||||
}
|
||||
|
||||
/** File keys per round. */
|
||||
const BATCH = 20_000
|
||||
|
||||
/**
|
||||
* Files cut by a range's edge (REBUILD.md, fragments), put together once
|
||||
* the ranges around them are done: a complete file is judged by the rules
|
||||
* like any other and written as a release chunk (with probe candidates); a
|
||||
* file still missing parts while every neighbouring range is done is
|
||||
* incomplete, and dropped (counted on its ranges). Others wait.
|
||||
*/
|
||||
class FragmentMerger {
|
||||
/** Where the next round goes on: waiting files must not take every round's places. */
|
||||
private cursor = ''
|
||||
|
||||
async run(): Promise<MergeResult> {
|
||||
const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
|
||||
const { rows: keys } = await db.rawQuery(
|
||||
`SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
|
||||
WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
|
||||
[this.cursor, BATCH]
|
||||
)
|
||||
// At the end of the keys: the next round starts over.
|
||||
result.wrapped = keys.length < BATCH
|
||||
this.cursor = result.wrapped ? '' : keys.at(-1).file_key
|
||||
if (keys.length === 0) return result
|
||||
const rules = new RuleSet(await RuleSets.current())
|
||||
const rows = await db
|
||||
.from('fragments as f')
|
||||
.join('ranges as r', 'r.id', 'f.range_id')
|
||||
.whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
|
||||
.select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
|
||||
const byKey = new Map<string, typeof rows>()
|
||||
for (const row of rows) {
|
||||
const list = byKey.get(row.file_key)
|
||||
if (list) list.push(row)
|
||||
else byKey.set(row.file_key, [row])
|
||||
}
|
||||
|
||||
const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
|
||||
const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
|
||||
const neighbours = await this.neighbourStatus([...byKey.values()])
|
||||
for (const list of byKey.values()) {
|
||||
result.files++
|
||||
const file = mergeFragments(list.map((r) => r.fragment as Fragment))
|
||||
const ids = list.map((r) => Number(r.id))
|
||||
const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
|
||||
if (file) {
|
||||
completed.push({ file, rangeId, ids })
|
||||
continue
|
||||
}
|
||||
if (list.every((r) => r.status === 'done') && neighbours(list)) {
|
||||
incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
|
||||
} else {
|
||||
result.waiting++
|
||||
}
|
||||
}
|
||||
|
||||
// Completed files, grouped into their releases, judged and written like a batch's.
|
||||
const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
|
||||
for (const { file, rangeId } of completed) {
|
||||
const key = releaseKeyOf(file.segments[0])
|
||||
const entry = byRelease.get(key)
|
||||
if (entry) entry.files.push(file)
|
||||
else byRelease.set(key, { files: [file], rangeId })
|
||||
}
|
||||
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
||||
for (const [key, { files, rangeId }] of byRelease) {
|
||||
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
|
||||
if (verdict.rule !== null) continue
|
||||
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
||||
}
|
||||
await ChunkPartitions.ensure()
|
||||
await db.transaction(async (trx) => {
|
||||
// In key order (batches lock their group; the merger only ever waits on one batch at a time).
|
||||
releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
|
||||
for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
|
||||
const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
|
||||
for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
|
||||
for (const { rangeId, segments } of incomplete) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
|
||||
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
|
||||
[rangeId, segments]
|
||||
)
|
||||
}
|
||||
})
|
||||
result.completed = completed.length
|
||||
result.incomplete = incomplete.length
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* For each file's fragments: whether the ranges right before and after
|
||||
* their ranges (where a missing part could be) are done, read in one
|
||||
* query for the whole round (per file it was three, ~2,000 files a
|
||||
* minute). A neighbour not planned yet counts as not done (newer articles
|
||||
* come later), except before the group's oldest article.
|
||||
*/
|
||||
private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
|
||||
const span = (list: (typeof lists)[number]) => ({
|
||||
group: Number(list[0].group_id),
|
||||
first: Math.min(...list.map((r) => Number(r.start_id))),
|
||||
last: Math.max(...list.map((r) => Number(r.end_id))),
|
||||
})
|
||||
const spans = lists.map(span)
|
||||
const groups = [...new Set(spans.map((s) => s.group))]
|
||||
// The range holding the article before and the one after, each by the (group_id, start_id) index:
|
||||
// "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
|
||||
FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
|
||||
ORDER BY r.start_id DESC LIMIT 1) b ON true
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
|
||||
ORDER BY r.start_id DESC LIMIT 1) a ON true`,
|
||||
[spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
|
||||
)
|
||||
const byEnd = new Map<string, string>()
|
||||
const byStart = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
|
||||
if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
|
||||
}
|
||||
const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
|
||||
return (list: (typeof lists)[number]) => {
|
||||
const { group, first, last } = span(list)
|
||||
const before = byEnd.get(`${group}:${first - 1}`)
|
||||
const low = lows.get(group)
|
||||
const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
|
||||
return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new FragmentMerger()
|
||||
@@ -0,0 +1,138 @@
|
||||
import type { Overview, XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
|
||||
export interface DateSample {
|
||||
article: number
|
||||
postedAt: Date
|
||||
}
|
||||
|
||||
/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
|
||||
const OLDEST = Date.UTC(1995, 0, 1)
|
||||
const DAY = 86_400_000
|
||||
|
||||
/**
|
||||
* The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
|
||||
* `min` of them; null otherwise (an expired or empty slice).
|
||||
*/
|
||||
export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
|
||||
const latest = now.getTime() + DAY
|
||||
const times = overviews
|
||||
.map(([, fields]) => Date.parse(fields.date ?? ''))
|
||||
.filter((t) => t >= OLDEST && t <= latest)
|
||||
.sort((a, b) => a - b)
|
||||
if (times.length < min) return null
|
||||
return new Date(times[Math.floor(times.length / 2)])
|
||||
}
|
||||
|
||||
/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
|
||||
export function samplePoints(low: number, high: number, count: number, size: number): number[] {
|
||||
const last = Math.max(low, high - size + 1)
|
||||
if (count <= 1 || last <= low) return [low]
|
||||
const points = new Set<number>()
|
||||
for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
|
||||
return [...points].sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
/**
|
||||
* The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
|
||||
* (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
|
||||
* in number order, bar stragglers and old crossposts).
|
||||
*/
|
||||
export function monotone(samples: DateSample[]): DateSample[] {
|
||||
const sorted = [...samples].sort((a, b) => a.article - b.article)
|
||||
const kept = sorted.filter((s, i) => {
|
||||
const next = sorted.slice(i + 1, i + 3)
|
||||
return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
|
||||
})
|
||||
let floor = Number.NEGATIVE_INFINITY
|
||||
return kept.map((s) => {
|
||||
floor = Math.max(floor, s.postedAt.getTime())
|
||||
return { article: s.article, postedAt: new Date(floor) }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
|
||||
* it, the first or last sample's date beyond them; null without samples.
|
||||
*/
|
||||
export function estimateDate(samples: DateSample[], article: number): Date | null {
|
||||
if (samples.length === 0) return null
|
||||
if (article <= samples[0].article) return samples[0].postedAt
|
||||
const last = samples[samples.length - 1]
|
||||
if (article >= last.article) return last.postedAt
|
||||
let lo = 0
|
||||
let hi = samples.length - 1
|
||||
while (hi - lo > 1) {
|
||||
const mid = (lo + hi) >> 1
|
||||
if (samples[mid].article <= article) lo = mid
|
||||
else hi = mid
|
||||
}
|
||||
const a = samples[lo]
|
||||
const b = samples[hi]
|
||||
const f = (article - a.article) / (b.article - a.article)
|
||||
return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
|
||||
}
|
||||
|
||||
/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
|
||||
export function sampleCount(
|
||||
low: number,
|
||||
high: number,
|
||||
every: number,
|
||||
min = 16,
|
||||
max = 1000
|
||||
): number {
|
||||
return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
|
||||
}
|
||||
|
||||
/**
|
||||
* Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
|
||||
* with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
|
||||
* server's span.
|
||||
*/
|
||||
export async function sampleGroup(
|
||||
use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
|
||||
name: string,
|
||||
options: { every: number; size: number; maxFailures?: number },
|
||||
now = new Date()
|
||||
): Promise<{
|
||||
samples: DateSample[]
|
||||
low: number
|
||||
high: number
|
||||
commands: number
|
||||
failed: string[]
|
||||
}> {
|
||||
const info = await use((client) => client.group(name))
|
||||
const samples: DateSample[] = []
|
||||
const failed: string[] = []
|
||||
let commands = 1
|
||||
// Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
|
||||
// minutes; 16 groups never got through on one connection) costs that point, not the group.
|
||||
for (const point of samplePoints(
|
||||
info.low,
|
||||
info.high,
|
||||
sampleCount(info.low, info.high, options.every),
|
||||
options.size
|
||||
)) {
|
||||
if (failed.length >= (options.maxFailures ?? 3)) break
|
||||
try {
|
||||
const date = await use(async (client) => {
|
||||
await client.group(name)
|
||||
commands += 2
|
||||
const slice = medianDate(
|
||||
await client.xover(point, Math.min(point + options.size - 1, info.high)),
|
||||
now
|
||||
)
|
||||
if (slice) return slice
|
||||
commands++
|
||||
return medianDate(
|
||||
await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
|
||||
now
|
||||
)
|
||||
})
|
||||
if (date) samples.push({ article: point, postedAt: date })
|
||||
} catch (error: any) {
|
||||
failed.push(`${point}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { createReadStream, createWriteStream } from 'node:fs'
|
||||
import { mkdir, rename, stat } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import type { Readable } from 'node:stream'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { LeaseLost } from '#services/leases'
|
||||
import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
|
||||
|
||||
export const LANES: Lane[] = ['history', 'tail']
|
||||
|
||||
/**
|
||||
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
|
||||
* next range each time it finished one, so a group's ranges are indexed in article order through one carried
|
||||
* window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
|
||||
* works and the snapshot its window stands at.
|
||||
*
|
||||
* - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
|
||||
* LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
|
||||
* the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
|
||||
* group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
|
||||
* the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
|
||||
* reaches the tail lane's first range (its ranges are done: the history lane is caught up).
|
||||
* - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
|
||||
* mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
|
||||
* big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
|
||||
* is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
|
||||
* lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
|
||||
* (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
|
||||
* idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
|
||||
* - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
|
||||
* when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
|
||||
* planned more.
|
||||
* - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
|
||||
* `<group>-tail.gz`), given to the next holder of the lane.
|
||||
*/
|
||||
class GroupLeases {
|
||||
get dir(): string {
|
||||
return coordinatorConfig.snapshotDir
|
||||
}
|
||||
|
||||
/**
|
||||
* The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
|
||||
* above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
|
||||
*/
|
||||
private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
|
||||
if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
|
||||
const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
|
||||
return {
|
||||
sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
|
||||
bindings: [tail, min, max],
|
||||
}
|
||||
}
|
||||
|
||||
async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
|
||||
// The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
|
||||
if (lane === 'history' && (await CatalogSpace.full())) return null
|
||||
const rules = await RuleSets.current()
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
return db.transaction(async (trx) => {
|
||||
// One lease at a time: the choice reads several tables.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
// The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
|
||||
// last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
|
||||
// most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
|
||||
// the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
|
||||
// win).
|
||||
const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
|
||||
const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH todo AS (
|
||||
SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
|
||||
FROM ranges r
|
||||
WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
|
||||
AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
GROUP BY r.group_id
|
||||
), free AS (
|
||||
SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
|
||||
todo.priority, todo.articles
|
||||
FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
|
||||
LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
|
||||
WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
|
||||
), turns AS (
|
||||
SELECT free.*,
|
||||
least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
|
||||
2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
|
||||
FROM free
|
||||
)
|
||||
SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
|
||||
[ttl, ...todo.bindings, lane, ttl, first, second]
|
||||
)
|
||||
const row = rows[0]
|
||||
if (!row) return null
|
||||
const groupId = Number(row.id)
|
||||
// What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
|
||||
// after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
|
||||
if (row.snapshot_next !== null) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
|
||||
WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
|
||||
AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
|
||||
[
|
||||
row.snapshot_next,
|
||||
groupId,
|
||||
row.snapshot_next,
|
||||
...all.bindings,
|
||||
row.snapshot_at ?? new Date(0),
|
||||
row.snapshot_next,
|
||||
]
|
||||
)
|
||||
} else {
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
|
||||
[groupId, ...all.bindings]
|
||||
)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
|
||||
VALUES (?, ?, ?, now(), now(), NULL)
|
||||
ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
|
||||
[groupId, lane, nodeId]
|
||||
)
|
||||
await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
return {
|
||||
group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
|
||||
lane,
|
||||
rules,
|
||||
snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** The group's lease row of the lane, when this node holds it. */
|
||||
async held(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
const lease = await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.first()
|
||||
if (!lease || Number(lease.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
|
||||
return lease
|
||||
}
|
||||
|
||||
/**
|
||||
* The group's next range of the lane in article order (or the one this node has), leased to the node; null when
|
||||
* the lane is caught up.
|
||||
*/
|
||||
async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const rules = await RuleSets.current()
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
const mine = (
|
||||
await db.rawQuery(
|
||||
`SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
const row =
|
||||
mine ??
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
|
||||
WHERE r.id = (
|
||||
SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, groupId, ...todo.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
|
||||
if (!row) return null
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ heartbeat_at: new Date() })
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
}
|
||||
|
||||
/** The lane a range belongs to, by its priority. */
|
||||
laneOf(priority: number): Lane {
|
||||
return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
|
||||
}
|
||||
|
||||
/** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
|
||||
async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
|
||||
await db.rawQuery(
|
||||
`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
|
||||
[headers, groupId, lane]
|
||||
)
|
||||
}
|
||||
|
||||
/** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
|
||||
async release(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const all = this.laneFilter(lane, false)
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ node_id: null, range_id: null })
|
||||
}
|
||||
|
||||
snapshotPath(groupId: number, lane: Lane = 'history') {
|
||||
return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
|
||||
}
|
||||
|
||||
/**
|
||||
* The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
|
||||
* gigabytes) and written whole before it replaces the old one.
|
||||
*/
|
||||
async putSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
next: number,
|
||||
body: Readable,
|
||||
lane: Lane = 'history'
|
||||
) {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await mkdir(this.dir, { recursive: true })
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
await pipeline(body, createWriteStream(`${path}.part`))
|
||||
await rename(`${path}.part`, path)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
|
||||
}
|
||||
|
||||
/** The snapshot's stream and size, or null when there is none. */
|
||||
async getSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
lane: Lane = 'history'
|
||||
): Promise<{ stream: Readable; size: number; next: number } | null> {
|
||||
const lease = await this.held(groupId, nodeId, lane)
|
||||
if (lease.snapshot_next === null) return null
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
try {
|
||||
const { size } = await stat(path)
|
||||
return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new GroupLeases()
|
||||
+63
-9
@@ -41,7 +41,12 @@ class Leases {
|
||||
AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
|
||||
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
|
||||
[
|
||||
nodeId,
|
||||
rules.version,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
]
|
||||
)
|
||||
let { rows } = await take(true)
|
||||
if (rows.length === 0) ({ rows } = await take(false))
|
||||
@@ -49,12 +54,38 @@ class Leases {
|
||||
if (!row) return null
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
|
||||
return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
group: group.name,
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
rules,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ranges read to their end whose worker went quiet before finishing them: done. The lease query skips a range past
|
||||
* its end, so one whose finish never came stayed leased for good (a gap range since 2026-10-16 14:03, during the lock
|
||||
* pile-up; LOG.md 2026-10-17 (72)). Its batches are all stored; its spooled chunks go.
|
||||
*/
|
||||
async finishAbandoned(): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'done', done_at = now()
|
||||
WHERE status = 'leased' AND next_id > end_id AND heartbeat_at < now() - make_interval(secs => ?)
|
||||
RETURNING id`,
|
||||
[coordinatorConfig.leaseTtlSeconds]
|
||||
)
|
||||
for (const { id } of rows) await Spool.remove(Number(id))
|
||||
return rows.length
|
||||
}
|
||||
|
||||
async leased(rangeId: number, nodeId: number) {
|
||||
const range = await db.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
@@ -67,8 +98,14 @@ class Leases {
|
||||
/** The worker reached the range's end: done, and its spooled chunks go. */
|
||||
async finish(rangeId: number, nodeId: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
if (Number(range.next_id) <= Number(range.end_id))
|
||||
throw new LeaseLost(
|
||||
`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`
|
||||
)
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
await Spool.remove(rangeId)
|
||||
}
|
||||
|
||||
@@ -79,12 +116,19 @@ class Leases {
|
||||
*/
|
||||
async corrupt(rangeId: number, nodeId: number, from: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
|
||||
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find(
|
||||
(s) => s.from === from
|
||||
)
|
||||
if (span) {
|
||||
await Spool.removeChunk(rangeId, span)
|
||||
if (range.capture) {
|
||||
const group = await db.from('groups').where('id', range.group_id).select('name').first()
|
||||
await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
|
||||
await Corpus.removeChunk(
|
||||
{ group: group.name, start: Number(range.start_id), end: Number(range.end_id) },
|
||||
span
|
||||
)
|
||||
// Not complete in the corpus any more: the corpus fill fetches the chunk again.
|
||||
await db.from('ranges').where('id', rangeId).update({ stored_at: null })
|
||||
}
|
||||
}
|
||||
const plain: number[] = (range.plain_chunks ?? []).map(Number)
|
||||
@@ -92,11 +136,21 @@ class Leases {
|
||||
// refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
|
||||
if (plain.filter((p) => p === from).length >= 2) {
|
||||
// Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'failed', node_id: null, error: `chunk ${from} doesn't inflate, also fetched plain` })
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({
|
||||
status: 'failed',
|
||||
node_id: null,
|
||||
error: `chunk ${from} doesn't inflate, also fetched plain`,
|
||||
})
|
||||
return
|
||||
}
|
||||
// The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
|
||||
await db.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`, [from, rangeId])
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`,
|
||||
[from, rangeId]
|
||||
)
|
||||
}
|
||||
|
||||
/** A stopping worker gives its range back; the next one resumes it. */
|
||||
|
||||
+93
-13
@@ -2,12 +2,18 @@ import env from '#start/env'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import type { NntpShare } from '#services/nntp_shares'
|
||||
|
||||
/** Connections wanted beyond the most in use at once since the last sync (the planner's, a hedge's). */
|
||||
const SHARE_MARGIN = 2
|
||||
|
||||
/**
|
||||
* The coordinator's header connections (REBUILD.md, decision e: workers
|
||||
* have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
|
||||
* that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
|
||||
* replaced on the next use.
|
||||
* replaced on the next use. Sharing the account (share()), at most its
|
||||
* share: over it, connections close as they come free. One left free for
|
||||
* NNTP_IDLE_CLOSE_SECONDS is closed on the next sync.
|
||||
*/
|
||||
class NntpPool {
|
||||
private idle: XoverClient[] = []
|
||||
@@ -17,9 +23,43 @@ class NntpPool {
|
||||
private openedAt = new Map<XoverClient, number>()
|
||||
/** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
|
||||
private retireAt = new Map<XoverClient, number>()
|
||||
/** When an idle connection came free. */
|
||||
private idleSince = new Map<XoverClient, number>()
|
||||
private limit = coordinatorConfig.nntpConnections
|
||||
/** Since the last sync: the most connections in use at once, and whether work waited for one (starved()). */
|
||||
private peak = 0
|
||||
private short = false
|
||||
|
||||
get size() {
|
||||
return coordinatorConfig.nntpConnections
|
||||
return this.limit
|
||||
}
|
||||
|
||||
/** Work that could use another connection didn't get one (the prefetcher with all its loops running). */
|
||||
starved() {
|
||||
this.short = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the share what the pool could use and has open, and takes the share's grant as its size. After work waited:
|
||||
* twice its size (history going on gets its maximum in a few syncs; a tail round, done in seconds, doesn't take
|
||||
* the prober's connections for one); else the most in use at once since the last sync and a margin.
|
||||
*/
|
||||
async share(share: NntpShare) {
|
||||
const want = this.short || this.waiting.length > 0 ? Math.max(this.limit * 2, this.limit + SHARE_MARGIN) : this.peak + SHARE_MARGIN
|
||||
this.short = false
|
||||
this.peak = this.busy.size
|
||||
const granted = await share.sync(want, this.open)
|
||||
const was = this.limit
|
||||
this.limit = granted
|
||||
// Over the share: the idle ones close now, the busy ones once free (release()).
|
||||
while (this.open > this.limit && this.idle.length > 0) this.retire(this.idle.pop()!)
|
||||
this.closeIdle()
|
||||
// Under it with work waiting: connections for it.
|
||||
while (this.waiting.length > 0 && this.open < this.limit) {
|
||||
const next = this.waiting.shift()!
|
||||
this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
}
|
||||
return { was, granted }
|
||||
}
|
||||
|
||||
async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
@@ -44,6 +84,7 @@ class NntpPool {
|
||||
|
||||
private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.busy.add(client)
|
||||
this.peak = Math.max(this.peak, this.busy.size)
|
||||
let broken = false
|
||||
try {
|
||||
return await work(client)
|
||||
@@ -56,9 +97,28 @@ class NntpPool {
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the connections free for nntpIdleCloseMs, the plain one too. */
|
||||
private closeIdle() {
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs <= 0) return
|
||||
const before = Date.now() - idleMs
|
||||
for (const client of this.idle.filter((c) => (this.idleSince.get(c) ?? Infinity) <= before)) {
|
||||
this.idle.splice(this.idle.indexOf(client), 1)
|
||||
this.retire(client)
|
||||
}
|
||||
if (this.plain && this.plainPending === 0 && this.plainUsedAt <= before) {
|
||||
void this.plain.then((c) => c.close()).catch(() => {})
|
||||
this.plain = null
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<XoverClient> {
|
||||
// The one used last: the others stay free long enough to close.
|
||||
const ready = this.idle.pop()
|
||||
if (ready) return ready
|
||||
if (ready) {
|
||||
this.idleSince.delete(ready)
|
||||
return ready
|
||||
}
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
@@ -84,15 +144,12 @@ class NntpPool {
|
||||
}
|
||||
|
||||
private release(client: XoverClient, broken: boolean) {
|
||||
if (this.open > this.limit) {
|
||||
this.retire(client)
|
||||
return
|
||||
}
|
||||
if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
|
||||
this.open--
|
||||
this.openedAt.delete(client)
|
||||
this.retireAt.delete(client)
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing a failed NNTP connection: ${error.message}`)
|
||||
}
|
||||
this.retire(client)
|
||||
// Someone waiting gets a fresh connection.
|
||||
const next = this.waiting.shift()
|
||||
if (next) this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
@@ -100,12 +157,30 @@ class NntpPool {
|
||||
}
|
||||
const next = this.waiting.shift()
|
||||
if (next) next(client)
|
||||
else this.idle.push(client)
|
||||
else {
|
||||
this.idle.push(client)
|
||||
this.idleSince.set(client, Date.now())
|
||||
}
|
||||
}
|
||||
|
||||
private retire(client: XoverClient) {
|
||||
this.open--
|
||||
this.openedAt.delete(client)
|
||||
this.retireAt.delete(client)
|
||||
this.idleSince.delete(client)
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing an NNTP connection: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes every connection, also those in use: their commands fail. */
|
||||
private plain: Promise<XoverClient> | null = null
|
||||
private plainQueue: Promise<unknown> = Promise.resolve()
|
||||
/** Commands queued or running on the plain connection, and when the last one ended. */
|
||||
private plainPending = 0
|
||||
private plainUsedAt = 0
|
||||
|
||||
/**
|
||||
* One connection without XFEATURE COMPRESS GZIP, opened when first needed,
|
||||
@@ -113,6 +188,7 @@ class NntpPool {
|
||||
* cuts short (Leases.corrupt). Its replies are deflated here.
|
||||
*/
|
||||
async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.plainPending++
|
||||
const run = this.plainQueue.then(async () => {
|
||||
this.plain ??= XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
@@ -132,7 +208,10 @@ class NntpPool {
|
||||
throw error
|
||||
}
|
||||
})
|
||||
this.plainQueue = run.catch(() => {})
|
||||
this.plainQueue = run.catch(() => {}).finally(() => {
|
||||
this.plainPending--
|
||||
this.plainUsedAt = Date.now()
|
||||
})
|
||||
return run
|
||||
}
|
||||
|
||||
@@ -143,6 +222,7 @@ class NntpPool {
|
||||
this.busy.clear()
|
||||
this.openedAt.clear()
|
||||
this.retireAt.clear()
|
||||
this.idleSince.clear()
|
||||
this.open = 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/** A share whose process hasn't synced for this long is gone: its connections closed with it. */
|
||||
const STALE_SECONDS = 60
|
||||
/** Syncs one at a time (pg_advisory_xact_lock). */
|
||||
const LOCK = 2_026_101_875
|
||||
|
||||
export interface Share {
|
||||
name: string
|
||||
priority: number
|
||||
/** Kept whatever the others want (as far as the total goes, by priority). */
|
||||
min: number
|
||||
/** What it could use now. */
|
||||
want: number
|
||||
/** Connections it has open (or may have: a lane without one yet counts). */
|
||||
open: number
|
||||
}
|
||||
|
||||
/** What each share should have: its minimum first, then by priority what it wants, up to the total. */
|
||||
export function targets(total: number, shares: Share[]): Map<string, number> {
|
||||
const ordered = [...shares].sort((a, b) => b.priority - a.priority || a.name.localeCompare(b.name))
|
||||
const target = new Map<string, number>()
|
||||
let left = total
|
||||
for (const s of ordered) {
|
||||
const t = Math.min(s.min, left)
|
||||
target.set(s.name, t)
|
||||
left -= t
|
||||
}
|
||||
for (const s of ordered) {
|
||||
const more = Math.min(Math.max(s.want - target.get(s.name)!, 0), left)
|
||||
target.set(s.name, target.get(s.name)! + more)
|
||||
left -= more
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
/**
|
||||
* The connections `self` may have: its target, but no more than the others leave over. Another share holds what it
|
||||
* has open or was granted, whichever is more (it may open up to its grant until its next sync tells it otherwise),
|
||||
* so the account never goes over the total: a share grows only into what the others closed or gave up.
|
||||
*/
|
||||
export function grant(total: number, self: Share, others: (Share & { granted: number })[]): number {
|
||||
const target = targets(total, [self, ...others]).get(self.name)!
|
||||
const held = others.reduce((sum, o) => sum + Math.max(o.open, o.granted), 0)
|
||||
return Math.max(0, Math.min(target, total - held))
|
||||
}
|
||||
|
||||
/**
|
||||
* A process's share of the NNTP account's connections (LOG.md 2026-10-18 (75)): the coordinator and the prober had a
|
||||
* fixed split, and with history paused the coordinator's 38 stood idle while the prober's 8 were its limit. Each
|
||||
* syncs every nntpShares.everyMs (what it wants and has open) and gets what it may open; one wanting less closes
|
||||
* connections as they come free, and the other grows as those show up closed.
|
||||
*/
|
||||
export class NntpShare {
|
||||
granted = 0
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
private priority: number,
|
||||
readonly min: number,
|
||||
readonly max: number
|
||||
) {}
|
||||
|
||||
async sync(want: number, open: number): Promise<number> {
|
||||
const self: Share = { name: this.name, priority: this.priority, min: this.min, want: Math.min(Math.max(want, this.min), this.max), open }
|
||||
const total = coordinatorConfig.nntpShares.total
|
||||
this.granted = await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(?)`, [LOCK])
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT name, priority, min, want, open, granted FROM nntp_shares WHERE name <> ? AND heartbeat_at > now() - make_interval(secs => ?)`,
|
||||
[this.name, STALE_SECONDS]
|
||||
)
|
||||
const granted = grant(total, self, rows)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO nntp_shares (name, priority, min, want, open, granted, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, now())
|
||||
ON CONFLICT (name) DO UPDATE SET priority = excluded.priority, min = excluded.min, want = excluded.want,
|
||||
open = excluded.open, granted = excluded.granted, heartbeat_at = now()`,
|
||||
[this.name, this.priority, this.min, self.want, open, granted]
|
||||
)
|
||||
return granted
|
||||
})
|
||||
return this.granted
|
||||
}
|
||||
|
||||
/** Stopping: its connections are closed, the others needn't wait for the row to go stale. */
|
||||
async leave() {
|
||||
await db.from('nntp_shares').where('name', this.name).delete()
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,28 @@ import { gunzipSync } from 'node:zlib'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
|
||||
const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
|
||||
|
||||
/**
|
||||
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
|
||||
* chunk store: every chunk decoded, its files in name order. A file stored
|
||||
* twice (the same name and first article, e.g. a range read again) is taken
|
||||
* once.
|
||||
* once. A known password (release_passwords) goes into the NZB's meta, where
|
||||
* SABnzbd and NZBGet take it for unpacking.
|
||||
*/
|
||||
class NzbService {
|
||||
async files(releaseId: number): Promise<ChunkFile[]> {
|
||||
const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
/** The release's known passwords, the most trusted first. */
|
||||
async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
|
||||
const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
|
||||
const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
|
||||
return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
|
||||
}
|
||||
|
||||
async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
|
||||
const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
const files = new Map<string, ChunkFile>()
|
||||
for (const chunk of chunks) {
|
||||
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
|
||||
@@ -22,10 +34,11 @@ class NzbService {
|
||||
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
|
||||
if (!release) return null
|
||||
const password = (await this.passwords(releaseId))[0]?.password
|
||||
// A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
|
||||
if (release.source === 'spot') {
|
||||
if (Number(release.source) === SOURCES.spot) {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
|
||||
if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
|
||||
if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
|
||||
}
|
||||
const xml = buildNzb(
|
||||
(await this.files(releaseId)).map((f) => ({
|
||||
@@ -35,10 +48,18 @@ class NzbService {
|
||||
groups: f.groups,
|
||||
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
})),
|
||||
{ title: release.name }
|
||||
{ title: release.name, password }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
}
|
||||
|
||||
/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
|
||||
export function withPassword(xml: string, password: string | undefined): string {
|
||||
if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
|
||||
const meta = `<meta type="password">${password.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')}</meta>`
|
||||
if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
|
||||
return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
|
||||
}
|
||||
|
||||
export default new NzbService()
|
||||
@@ -0,0 +1,53 @@
|
||||
import NzbService from '#services/nzb_service'
|
||||
|
||||
/** A file a par2 set describes (par2_files). */
|
||||
export interface SetFile {
|
||||
name: string
|
||||
length: number
|
||||
hash16k: string
|
||||
}
|
||||
|
||||
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
||||
|
||||
/**
|
||||
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
||||
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
||||
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
||||
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
||||
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
||||
*/
|
||||
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
||||
const names = new Set(files.map((f) => base(f.name)))
|
||||
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
||||
const byName = set.filter((f) => names.has(base(f.name))).length
|
||||
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
||||
if (byName > 0) return false
|
||||
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
||||
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
||||
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
||||
* a transaction that moved chunks (a merge) reads them as moved.
|
||||
*/
|
||||
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const { rows: releases } = await client.rawQuery(
|
||||
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
let decided = 0
|
||||
for (const { id } of releases) {
|
||||
const { rows: set } = await client.rawQuery(
|
||||
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (set.length === 0) continue
|
||||
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
||||
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
||||
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
||||
decided++
|
||||
}
|
||||
return decided
|
||||
}
|
||||
+243
-60
@@ -2,8 +2,12 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { par2ReleaseName } from '#lib/release/index'
|
||||
import { looksRandom } from '#lib/release/random'
|
||||
import { SidecarFolder } from '#services/sidecar_folder'
|
||||
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
|
||||
import { randomPoster } from '#lib/release/identity'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||
@@ -15,21 +19,45 @@ const MAX_HOLDERS = 4
|
||||
/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
|
||||
const MAX_FILES_PER_SET_FILE = 3
|
||||
const SLACK_FILES = 10
|
||||
/**
|
||||
* A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
|
||||
* it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
|
||||
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
|
||||
*/
|
||||
const COMMON_FILE = 20
|
||||
/**
|
||||
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
|
||||
* NAME_CANDIDATES of them (a poster's uploads of a day).
|
||||
*/
|
||||
const NAME_SPREAD_MS = 12 * 3600 * 1000
|
||||
const NAME_CANDIDATES = 200
|
||||
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
|
||||
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
|
||||
/** Between rounds. */
|
||||
const ROUND_MS = 60_000
|
||||
/** A round looks back this far past the previous one's start (rows committed while it ran). */
|
||||
/** A window looks back this far past the previous one's end (rows committed after it was read). */
|
||||
const OVERLAP_MS = 60_000
|
||||
/**
|
||||
* A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
|
||||
* two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
|
||||
*/
|
||||
const WINDOW_MS = 60 * 60_000
|
||||
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
|
||||
const FOLD_EVERY_MS = 5 * 60_000
|
||||
/** Between par2_files prunes (complete and judged sets' file lists). */
|
||||
const PRUNE_EVERY_MS = 60 * 60_000
|
||||
/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
|
||||
/** A prune's share of the merger's time: the rest goes on in the next one. */
|
||||
const PRUNE_BUDGET_MS = 2 * 60_000
|
||||
/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
|
||||
const CURSOR = 'merger'
|
||||
|
||||
export interface MergeRound {
|
||||
sets: number
|
||||
merges: number
|
||||
releases: number
|
||||
/** The name pass: merges, releases in them. */
|
||||
nameMerges: number
|
||||
nameReleases: number
|
||||
/** The inner-name pass: stems checked, merges, releases in them. */
|
||||
stems: number
|
||||
stemMerges: number
|
||||
@@ -44,13 +72,33 @@ export interface MergeRound {
|
||||
*/
|
||||
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
|
||||
|
||||
/**
|
||||
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
|
||||
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
|
||||
*/
|
||||
export function fileKey(name: string): string {
|
||||
let n = name.replace(/^.*[\\/]/, '')
|
||||
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
|
||||
const read = Buffer.from(n, 'latin1').toString('utf8')
|
||||
if (!read.includes('\ufffd')) n = read
|
||||
}
|
||||
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
|
||||
const HOLDERS = `SELECT c.release_id FROM (
|
||||
SELECT l.release_id, count(*) OVER () AS holders
|
||||
FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
|
||||
) c WHERE c.holders <= ${COMMON_FILE}`
|
||||
|
||||
interface Member {
|
||||
id: number
|
||||
release_key: string
|
||||
group_id: number
|
||||
files: number
|
||||
posted_at: Date | null
|
||||
readable: boolean
|
||||
poster_id?: number | null
|
||||
/** The poster's text (joined from posters) where a query needs it. */
|
||||
poster?: string | null
|
||||
file_total?: number | null
|
||||
}
|
||||
@@ -95,11 +143,12 @@ export class Par2Merger {
|
||||
// First, at start too: a prune needn't wait for a long round.
|
||||
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
|
||||
prunedAt = Date.now()
|
||||
const pruned = await new Par2Pruner().round()
|
||||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
|
||||
const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
|
||||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
|
||||
}
|
||||
const round = await this.round()
|
||||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||||
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
|
||||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||||
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||||
} catch (error: any) {
|
||||
@@ -109,14 +158,18 @@ export class Par2Merger {
|
||||
}
|
||||
}
|
||||
|
||||
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
|
||||
/**
|
||||
* The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
|
||||
* lone inspection), a window at a time.
|
||||
*/
|
||||
async round(): Promise<MergeRound> {
|
||||
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
|
||||
// A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
|
||||
const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
|
||||
const end = new Date(now[0].at)
|
||||
if (!this.since) {
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
this.since = saved ? new Date(saved.at) : new Date(0)
|
||||
}
|
||||
const since = new Date(this.since.getTime() - OVERLAP_MS)
|
||||
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
|
||||
// par2 pass (hours when that starts over).
|
||||
const folded = await this.folder.round()
|
||||
@@ -130,57 +183,170 @@ export class Par2Merger {
|
||||
foldedAt = Date.now()
|
||||
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||||
}
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT set_id FROM par2_sets WHERE created_at >= ?
|
||||
UNION
|
||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
|
||||
[since, since]
|
||||
)
|
||||
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
for (const { set_id: setId } of rows) {
|
||||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
while (this.since < end) {
|
||||
if (this.stopped) return round
|
||||
await foldAgain()
|
||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.merges++
|
||||
round.releases += merged + 1
|
||||
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
|
||||
const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
|
||||
if (from) {
|
||||
const { touched, sets } = await this.setsToMerge(from, until)
|
||||
round.sets += touched
|
||||
for (const setId of sets) {
|
||||
if (this.stopped) return round
|
||||
await foldAgain()
|
||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.merges++
|
||||
round.releases += merged + 1
|
||||
}
|
||||
}
|
||||
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
|
||||
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
|
||||
// 2026-10-23 (97)).
|
||||
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
|
||||
for (const { set_id: setId } of found) {
|
||||
if (this.stopped) return round
|
||||
const merged = await this.mergeByName(setId.trim()).catch((error) => {
|
||||
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.nameMerges++
|
||||
round.nameReleases += merged + 1
|
||||
}
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
|
||||
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
[from, until]
|
||||
)
|
||||
for (const { stem } of stems) {
|
||||
if (this.stopped) return round
|
||||
if (!looksRandom(stem)) continue
|
||||
round.stems++
|
||||
const merged = await this.mergeStem(stem).catch((error) => {
|
||||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.stemMerges++
|
||||
round.stemReleases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only after a whole window: a restart halfway goes on from the window before.
|
||||
this.since = until
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, this.since]
|
||||
)
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i WHERE i.kind = 'lone' AND i.created_at >= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
[since]
|
||||
)
|
||||
for (const { stem } of stems) {
|
||||
if (this.stopped) return round
|
||||
if (!looksRandom(stem)) continue
|
||||
round.stems++
|
||||
const merged = await this.mergeStem(stem).catch((error) => {
|
||||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.stemMerges++
|
||||
round.stemReleases += merged + 1
|
||||
}
|
||||
}
|
||||
// Only after a whole round: a restart halfway goes on from the round before.
|
||||
this.since = now[0].at
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, this.since]
|
||||
)
|
||||
return round
|
||||
}
|
||||
|
||||
/** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
|
||||
private async firstNew(from: Date, end: Date): Promise<Date | null> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT least(
|
||||
(SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
|
||||
[from, from, from]
|
||||
)
|
||||
const at = rows[0].at ? new Date(rows[0].at) : null
|
||||
return at && at <= end ? at : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
|
||||
* change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
|
||||
* hour's 147k): mergeSet needn't look at them.
|
||||
*/
|
||||
private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH touched AS MATERIALIZED (
|
||||
SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
|
||||
UNION
|
||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
|
||||
WHERE h.created_at >= ? AND h.created_at <= ?
|
||||
)
|
||||
SELECT (SELECT count(*) FROM touched) AS touched, array(
|
||||
SELECT t.set_id FROM touched t
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM (
|
||||
SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
|
||||
UNION ALL
|
||||
SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
|
||||
) u HAVING count(DISTINCT u.release_id) >= 2)
|
||||
) AS sets`,
|
||||
[from, until, from, until]
|
||||
)
|
||||
return { touched: Number(rows[0].touched), sets: rows[0].sets }
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
|
||||
* and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
|
||||
* sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
|
||||
* again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
|
||||
*/
|
||||
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
|
||||
const { rows: all } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
|
||||
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
|
||||
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
|
||||
[setId]
|
||||
)
|
||||
const anchor: Member | undefined = all[0]
|
||||
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
|
||||
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
|
||||
const holders = all.filter(
|
||||
(h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
|
||||
)
|
||||
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||||
if (described.length === 0) return null
|
||||
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
|
||||
const at = new Date(anchor.posted_at).getTime()
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
|
||||
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
|
||||
AND r.files <= ?
|
||||
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
|
||||
// More files than the set describes, with its sidecars, can't all be named in it.
|
||||
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
|
||||
)
|
||||
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
|
||||
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
|
||||
for (const c of candidates) {
|
||||
const own = await NzbService.files(Number(c.id), trx)
|
||||
const content = own.filter((f) => !besideSet(f.name))
|
||||
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
|
||||
found.set(String(c.id), c)
|
||||
files += Number(c.files)
|
||||
}
|
||||
if (found.size < 2) return null
|
||||
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
|
||||
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
|
||||
return found
|
||||
}
|
||||
|
||||
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
|
||||
async mergeByName(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const found = await this.nameMatches(trx, setId)
|
||||
return found ? this.merge(trx, found, setId, 'name') : 0
|
||||
})
|
||||
}
|
||||
|
||||
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
|
||||
async mergeStem(stem: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
|
||||
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
|
||||
FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
|
||||
[stem]
|
||||
@@ -192,7 +358,7 @@ export class Par2Merger {
|
||||
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
|
||||
const near = (r: Member) =>
|
||||
r.group_id === anchor.group_id &&
|
||||
(r.poster ?? '') === (anchor.poster ?? '') &&
|
||||
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
|
||||
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
|
||||
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const found = new Map<string, Member>()
|
||||
@@ -210,7 +376,7 @@ export class Par2Merger {
|
||||
async mergeSet(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows: holders } = await trx.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
WHERE s.set_id = ? ORDER BY r.id`,
|
||||
[setId]
|
||||
)
|
||||
@@ -219,8 +385,8 @@ export class Par2Merger {
|
||||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||||
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const { rows: matches } = await trx.rawQuery(
|
||||
`SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
|
||||
FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
|
||||
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
|
||||
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
|
||||
WHERE f.set_id = ?`,
|
||||
[setId]
|
||||
)
|
||||
@@ -250,7 +416,7 @@ export class Par2Merger {
|
||||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||||
*/
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
|
||||
{
|
||||
// Locked in id order (the batch store and other rounds take the same rows).
|
||||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||||
@@ -262,12 +428,28 @@ export class Par2Merger {
|
||||
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
|
||||
SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||||
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||||
[target.id, key, via, others]
|
||||
)
|
||||
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
|
||||
// isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
|
||||
await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
|
||||
AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
|
||||
OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
|
||||
[others, target.id, others]
|
||||
)
|
||||
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
|
||||
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||||
}
|
||||
// An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[target.id]
|
||||
)
|
||||
// The members' 7z tails together: those below the highest volume go.
|
||||
await pruneSevenZipTails(trx, [target.id])
|
||||
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
|
||||
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
|
||||
await trx.rawQuery(
|
||||
@@ -282,25 +464,26 @@ export class Par2Merger {
|
||||
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
|
||||
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
|
||||
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
|
||||
first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
|
||||
first_seen_at = least(t.first_seen_at, s.first_seen_at),
|
||||
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
|
||||
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
|
||||
name = coalesce(nullif(?, ''), t.name)
|
||||
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
|
||||
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
|
||||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
|
||||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
|
||||
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
|
||||
max(file_total) AS file_total, max(inner_name) AS inner_name
|
||||
FROM releases WHERE id = ANY(?)) s
|
||||
WHERE t.id = ?`,
|
||||
[name ?? '', others, target.id]
|
||||
)
|
||||
// Completeness as the settler decides it, once settled.
|
||||
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
[target.id]
|
||||
)
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
||||
await decidePar2Completeness(trx, [target.id])
|
||||
return others.length
|
||||
}
|
||||
}
|
||||
|
||||
+75
-43
@@ -1,75 +1,107 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
||||
const SETTLED_MS = 2 * 86400_000
|
||||
/** Sets per statement. */
|
||||
const BATCH = 20_000
|
||||
/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
|
||||
/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
|
||||
const CURSOR = 'par2-prune'
|
||||
/** The merger's row in worker_cursors: the end of its last whole window. */
|
||||
const MERGER = 'merger'
|
||||
/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
|
||||
const MERGER_OVERLAP_MS = 60_000
|
||||
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
|
||||
const JUNK = ['dead', 'fake', 'repeated']
|
||||
/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
|
||||
const MERGE_KINDS = ['lone', 'par2']
|
||||
|
||||
export interface PruneRound {
|
||||
complete: number
|
||||
judged: number
|
||||
final: number
|
||||
rows: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
|
||||
* 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
|
||||
* done when
|
||||
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
|
||||
* matches file hashes to releases and names an unreadable upload once; it's done when
|
||||
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
|
||||
* set (all of it merged into one);
|
||||
* - judged: every release holding it has a junk verdict (dead, fake, repeated).
|
||||
* - judged: every release holding it has a junk verdict (dead, fake, repeated);
|
||||
* - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
|
||||
* it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
|
||||
* and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
|
||||
* describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
|
||||
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
|
||||
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
||||
* The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
|
||||
* of a run, its ranges' probes not done yet) is looked at again on the next pass.
|
||||
*/
|
||||
export class Par2Pruner {
|
||||
async round(): Promise<PruneRound> {
|
||||
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
let since: Date = saved ? new Date(saved.at) : new Date(0)
|
||||
const until = new Date(Date.now() - SETTLED_MS)
|
||||
/** Sets per batch (BATCH; smaller in the tests). */
|
||||
constructor(private batch = BATCH) {}
|
||||
|
||||
/**
|
||||
* Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
|
||||
* it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
|
||||
* 2026-10-17 (72)).
|
||||
*/
|
||||
async round(budgetMs = Infinity): Promise<PruneRound> {
|
||||
const deadline = Date.now() + budgetMs
|
||||
const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after: string = cursors[0]?.key ?? ''
|
||||
// Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
|
||||
// still to come. Hashes since the merger's last window: it hasn't matched them yet.
|
||||
const { rows: marks } = await db.rawQuery(
|
||||
`SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
|
||||
coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
|
||||
now() - make_interval(secs => ?) AS settled`,
|
||||
[MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
|
||||
)
|
||||
const { probed, merged, settled } = marks[0]
|
||||
for (;;) {
|
||||
const { rows: sets } = await db.rawQuery(
|
||||
`SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
|
||||
HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
|
||||
[since, until, BATCH]
|
||||
`WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
|
||||
recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
|
||||
unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
|
||||
SELECT b.set_id, CASE
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
|
||||
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||
WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
|
||||
WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
|
||||
r.settled_at IS NULL
|
||||
OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
|
||||
AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
|
||||
THEN 'final' END AS reason
|
||||
FROM batch b ORDER BY b.set_id`,
|
||||
[after, this.batch, probed, merged, settled, JUNK, settled]
|
||||
)
|
||||
if (sets.length === 0) break
|
||||
const ids = sets.map((s: { set_id: string }) => s.set_id)
|
||||
const { rows: done } = await db.rawQuery(
|
||||
`SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
|
||||
WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
|
||||
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||
WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
|
||||
[ids]
|
||||
)
|
||||
round.complete += done.length
|
||||
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
|
||||
since = sets.at(-1).at
|
||||
const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
|
||||
for (const reason of ['complete', 'judged', 'final'] as const) {
|
||||
const ids = done(reason)
|
||||
round[reason] += ids.length
|
||||
// The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
|
||||
if (reason !== 'judged' && ids.length > 0) {
|
||||
const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
|
||||
await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
|
||||
}
|
||||
round.rows += await this.drop(ids)
|
||||
}
|
||||
// Past the last set: the next batch starts the next pass.
|
||||
after = sets.length < this.batch ? '' : sets.at(-1).set_id
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, since]
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, after]
|
||||
)
|
||||
if (sets.length < BATCH) break
|
||||
if (after === '' || Date.now() > deadline) break
|
||||
}
|
||||
// Judged: no cursor (verdicts come and go); only sets with rows left.
|
||||
const { rows: judged } = await db.rawQuery(
|
||||
`SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
|
||||
WHERE v.rule = ANY(?)
|
||||
AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
|
||||
WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
|
||||
[JUNK, JUNK]
|
||||
)
|
||||
round.judged = judged.length
|
||||
const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
|
||||
for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
|
||||
return round
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { explored } from '#lib/rules/index'
|
||||
import { usableName, type Inspection } from '#lib/probe/index'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
|
||||
/** pending_releases.status (migration 6400). */
|
||||
export const PENDING = {
|
||||
waiting: 0,
|
||||
named: 1,
|
||||
discarded: 2,
|
||||
exhausted: 3,
|
||||
naming: 4,
|
||||
taken: 9,
|
||||
} as const
|
||||
|
||||
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
|
||||
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
|
||||
|
||||
export interface PendingTarget {
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
}
|
||||
|
||||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||||
export interface PendingProbe {
|
||||
/** pending_releases.release_key as hex. */
|
||||
releaseKey: string
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
tool: string
|
||||
groupId: number
|
||||
poster: string
|
||||
/** Targets probed before this one. */
|
||||
tries: number
|
||||
targets: number
|
||||
}
|
||||
|
||||
export type PendingOutcome = {
|
||||
probe: PendingProbe
|
||||
status: 'done' | 'missing' | 'error' | 'waiting'
|
||||
inspection?: Inspection
|
||||
}
|
||||
|
||||
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||||
const rank = (kind: string) => {
|
||||
const i = (EFFORT as readonly string[]).indexOf(kind)
|
||||
return i < 0 ? EFFORT.length : i
|
||||
}
|
||||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
|
||||
* signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
|
||||
*/
|
||||
class PendingLadder {
|
||||
/** A previous prober's claims back in the queue. */
|
||||
async release(): Promise<number> {
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ?`,
|
||||
[PENDING.waiting, PENDING.taken]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
}
|
||||
|
||||
/** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
|
||||
async claim(n: number): Promise<PendingProbe[]> {
|
||||
if (n <= 0) return []
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
|
||||
[PENDING.taken, PENDING.waiting, n]
|
||||
)
|
||||
return rows.map((r: any) => {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
|
||||
* end of the ladder (status 3); the finished climbs counted on their signatures.
|
||||
*/
|
||||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||||
if (outcomes.length === 0) return
|
||||
const rows = outcomes.map((o) => {
|
||||
const p = o.probe
|
||||
if (o.status === 'waiting') {
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries: p.tries,
|
||||
named: null,
|
||||
inspection: null,
|
||||
final: null,
|
||||
}
|
||||
}
|
||||
const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const tries = p.tries + 1
|
||||
const inspection = {
|
||||
kind: p.kind,
|
||||
status: o.status,
|
||||
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
|
||||
}
|
||||
if (name)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.naming,
|
||||
tries: 0,
|
||||
named: name,
|
||||
inspection,
|
||||
final: 'named',
|
||||
}
|
||||
if (tries >= p.targets)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.exhausted,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: 'exhausted',
|
||||
}
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: null,
|
||||
}
|
||||
})
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||||
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
|
||||
)
|
||||
// The signatures: the poster's own and the tool's and group's together.
|
||||
const counts = new Map<
|
||||
string,
|
||||
{ tool: string; group_id: number; poster: string; probed: number; named: number }
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const final = rows[i].final
|
||||
if (!final) continue
|
||||
for (const poster of [o.probe.poster, '']) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
poster,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
}
|
||||
entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
}
|
||||
if (counts.size === 0) return
|
||||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
|
||||
async discard(limit = 10_000): Promise<number> {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
|
||||
WHERE p.status = ? AND EXISTS (
|
||||
SELECT 1 FROM pending_signatures s
|
||||
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
|
||||
LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
)
|
||||
const keys = rows
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
if (keys.length === 0) return 0
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
|
||||
[PENDING.discarded, PENDING.waiting, keys]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
}
|
||||
}
|
||||
|
||||
export default new PendingLadder()
|
||||
@@ -0,0 +1,229 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import { unzip } from 'node:zlib'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
|
||||
import { identityKey } from '#lib/release/keys'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import Corpus from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { PENDING } from '#services/pending_ladder'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
|
||||
interface NamedRow {
|
||||
release_key: Buffer
|
||||
identity_key: Buffer
|
||||
group_id: number
|
||||
named: string
|
||||
first_article: number
|
||||
last_article: number
|
||||
file_total: number | null
|
||||
tries: number
|
||||
}
|
||||
|
||||
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
|
||||
const MAX_MOVES = 3
|
||||
|
||||
/**
|
||||
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
|
||||
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
|
||||
* connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
|
||||
* release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
|
||||
* candidates, which the prober answers from probe_cache. Runs inside the prober.
|
||||
*/
|
||||
export class PendingNamer {
|
||||
private client: XoverClient | null = null
|
||||
private usedAt = 0
|
||||
|
||||
constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
|
||||
|
||||
async run(stopped: () => boolean) {
|
||||
while (!stopped()) {
|
||||
let rows: NamedRow[] = []
|
||||
try {
|
||||
rows = await this.take(20)
|
||||
for (const row of rows) {
|
||||
if (stopped()) break
|
||||
try {
|
||||
await this.move(row)
|
||||
} catch (error: any) {
|
||||
logger.warn(
|
||||
`Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
|
||||
[row.release_key]
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Naming: ${error.message}`)
|
||||
}
|
||||
if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
}
|
||||
if (rows.length === 0)
|
||||
for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
|
||||
}
|
||||
this.client?.close()
|
||||
this.client = null
|
||||
}
|
||||
|
||||
/** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
|
||||
async take(n: number): Promise<NamedRow[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
|
||||
FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
|
||||
[PENDING.naming, MAX_MOVES, n]
|
||||
)
|
||||
return rows.map((r: any) => ({
|
||||
...r,
|
||||
group_id: Number(r.group_id),
|
||||
first_article: Number(r.first_article),
|
||||
last_article: Number(r.last_article),
|
||||
tries: Number(r.tries),
|
||||
}))
|
||||
}
|
||||
|
||||
/** One named release into the catalog. */
|
||||
async move(row: NamedRow): Promise<void> {
|
||||
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
|
||||
const range = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.where('start_id', '<=', row.first_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('id')
|
||||
.first()
|
||||
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
|
||||
const text = await this.overviewText(group.name, row)
|
||||
const rules = await RuleSets.current()
|
||||
const ix = new RangeIndexer({
|
||||
group: group.name,
|
||||
start: row.first_article,
|
||||
end: row.last_article,
|
||||
rules: { ...rules, samplePercent: 0 },
|
||||
idleChunks: 0,
|
||||
edgeArticles: 0,
|
||||
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
|
||||
partialFiles: true,
|
||||
})
|
||||
ix.addText(text, row.last_article + 1)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
|
||||
if (!found)
|
||||
throw new Error(
|
||||
`its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
|
||||
)
|
||||
// The catalog row under the pending key (the identity, the group, the first article the edge saw).
|
||||
const chunk: ReleaseChunk = {
|
||||
...found,
|
||||
name: row.named,
|
||||
source: 'weak-stem',
|
||||
readable: true,
|
||||
finished: true,
|
||||
edge: false,
|
||||
firstArticle: row.first_article,
|
||||
fileTotal: row.file_total ?? found.fileTotal,
|
||||
tool: undefined,
|
||||
}
|
||||
const candidates = batch.candidates
|
||||
.filter((c) => c.releaseKey === found.key)
|
||||
.map((c) => ({ ...c, upload: found.upload }))
|
||||
// The chunk's month partition, as the batch store makes sure of before its writes.
|
||||
await ChunkPartitions.ensure()
|
||||
await db.transaction(async (trx) => {
|
||||
const blockId = await BatchStore.blockOf(trx, range)
|
||||
await BatchStore.writeReleases(
|
||||
trx,
|
||||
{ groupId: row.group_id, blockId },
|
||||
row.first_article,
|
||||
[chunk],
|
||||
candidates
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
|
||||
[row.named.slice(0, 255), row.release_key]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
|
||||
[PENDING.named, row.release_key]
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
|
||||
private async overviewText(groupName: string, row: NamedRow): Promise<string> {
|
||||
if (this.corpusDir) {
|
||||
const stored = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.whereNotNull('stored_at')
|
||||
.where('start_id', '<=', row.last_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('start_id')
|
||||
const parts: string[] = []
|
||||
let covered = true
|
||||
for (const r of stored) {
|
||||
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
for (
|
||||
let from =
|
||||
range.start +
|
||||
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
|
||||
from <= Math.min(row.last_article, range.end);
|
||||
from += size
|
||||
) {
|
||||
const raw = await Corpus.read(
|
||||
range,
|
||||
{ from, to: Math.min(from + size - 1, range.end) },
|
||||
this.corpusDir
|
||||
)
|
||||
if (!raw) {
|
||||
covered = false
|
||||
break
|
||||
}
|
||||
parts.push((await inflate(raw)).toString('latin1'))
|
||||
}
|
||||
if (!covered) break
|
||||
}
|
||||
if (
|
||||
covered &&
|
||||
parts.length > 0 &&
|
||||
stored.length > 0 &&
|
||||
Number(stored[0].start_id) <= row.first_article &&
|
||||
Number(stored[stored.length - 1].end_id) >= row.last_article
|
||||
) {
|
||||
return parts.join('')
|
||||
}
|
||||
}
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: true,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
this.usedAt = Date.now()
|
||||
try {
|
||||
await this.client.group(groupName)
|
||||
// One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
|
||||
const raw = await this.client.xoverRaw(row.first_article, row.last_article)
|
||||
return (await inflate(raw)).toString('latin1')
|
||||
} catch (error) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
+180
-34
@@ -35,20 +35,8 @@ class Planner {
|
||||
|
||||
async planGroup(group: { id: number; name: string }): Promise<number> {
|
||||
const info = await NntpPool.use((client) => client.group(group.name))
|
||||
const size = coordinatorConfig.rangeArticles
|
||||
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
|
||||
const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
|
||||
const span = Math.max(1, info.high - info.low)
|
||||
const rows = []
|
||||
for (let start = from; start <= info.high; start += size) {
|
||||
rows.push({
|
||||
group_id: group.id,
|
||||
start_id: start,
|
||||
end_id: Math.min(start + size - 1, info.high),
|
||||
next_id: start,
|
||||
priority: Math.floor((1000 * (start - info.low)) / span),
|
||||
})
|
||||
}
|
||||
const rows = historyRows(group.id, info, Number(last?.end ?? 0), coordinatorConfig.rangeArticles)
|
||||
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
|
||||
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
|
||||
return rows.length
|
||||
@@ -75,32 +63,99 @@ class Planner {
|
||||
* numbers come from LIST ACTIVE, one command per name prefix (GROUP per
|
||||
* group took minutes for 1000 groups).
|
||||
*/
|
||||
async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
|
||||
async planTail(now = new Date()): Promise<TailPlanned> {
|
||||
const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
|
||||
if (groups.length === 0) return { groups: 0, ranges: 0 }
|
||||
const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
|
||||
const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
|
||||
if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
const { rows: ends } = await db.rawQuery(
|
||||
`SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
|
||||
(status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
|
||||
FROM ranges ORDER BY group_id, start_id DESC`,
|
||||
[coordinatorConfig.tail.priority]
|
||||
)
|
||||
const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
|
||||
ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
|
||||
)
|
||||
const highs = new Map<string, number>()
|
||||
await NntpPool.use(async (client) => {
|
||||
for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
|
||||
for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
|
||||
}
|
||||
})
|
||||
const rows = tailRanges(
|
||||
groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
|
||||
const plan = tailRanges(
|
||||
groups.map((g) => {
|
||||
const l = last.get(Number(g.id))
|
||||
return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
|
||||
}),
|
||||
now,
|
||||
{ rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
|
||||
)
|
||||
if (rows.length === 0) return { groups: 0, ranges: 0 }
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('ranges').multiInsert(rows)
|
||||
for (const id of new Set(rows.map((r) => r.group_id))) {
|
||||
const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
|
||||
await trx.from('groups').where('id', id).update({ high, planned_at: now })
|
||||
}
|
||||
})
|
||||
return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
|
||||
return this.applyTail(plan, now)
|
||||
}
|
||||
|
||||
/** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
|
||||
async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
|
||||
const rows = [...plan.rows]
|
||||
let extended = 0
|
||||
if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
await db.transaction(async (trx) => {
|
||||
for (const e of plan.extended) {
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
|
||||
[e.end_id, e.id, e.from - 1]
|
||||
)
|
||||
if (done.length > 0) extended++
|
||||
else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
|
||||
}
|
||||
if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
|
||||
const highs = new Map<number, number>()
|
||||
for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
|
||||
for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
|
||||
})
|
||||
return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
|
||||
* RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
|
||||
* (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
|
||||
*/
|
||||
async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT id, group_id, start_id, end_id, priority FROM ranges
|
||||
WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
|
||||
ORDER BY group_id, start_id`
|
||||
)
|
||||
const merges = tailMerges(
|
||||
candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
|
||||
coordinatorConfig.rangeArticles
|
||||
)
|
||||
const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
|
||||
if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
await db.transaction(async (trx) => {
|
||||
// The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
const { rows: gone } = await trx.rawQuery(
|
||||
`DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
|
||||
[merges.flatMap((m) => m.drop)]
|
||||
)
|
||||
if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
|
||||
const { rows: grown } = await trx.rawQuery(
|
||||
`UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
|
||||
WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
|
||||
[merges.map((m) => m.id), merges.map((m) => m.end_id)]
|
||||
)
|
||||
if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
|
||||
})
|
||||
return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
}
|
||||
}
|
||||
|
||||
export interface TailPlanned {
|
||||
groups: number
|
||||
/** Rows added. */
|
||||
ranges: number
|
||||
/** Open ranges grown. */
|
||||
extended: number
|
||||
}
|
||||
|
||||
/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
|
||||
@@ -117,28 +172,119 @@ export interface TailGroup {
|
||||
plannedAt: Date
|
||||
/** The server's newest article (null: not listed). */
|
||||
high: number | null
|
||||
/**
|
||||
* Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
|
||||
* nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
|
||||
*/
|
||||
open?: { id: number; start: number } | null
|
||||
}
|
||||
|
||||
export interface TailRow {
|
||||
group_id: number
|
||||
start_id: number
|
||||
end_id: number
|
||||
next_id: number
|
||||
priority: number
|
||||
}
|
||||
|
||||
export interface TailPlan {
|
||||
/** New ranges. */
|
||||
rows: TailRow[]
|
||||
/** Open ranges grown to end_id (from: the first new article, the old end + 1). */
|
||||
extended: { id: number; group_id: number; from: number; end_id: number }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The tail's new ranges: per group, the articles after its last range, in
|
||||
* ranges of at most `rangeArticles`, once there are `minArticles` of them or
|
||||
* the group's last planning is `maxDelayMinutes` ago.
|
||||
* The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
|
||||
* the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
|
||||
* tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
|
||||
*/
|
||||
export function tailRanges(
|
||||
groups: TailGroup[],
|
||||
now: Date,
|
||||
options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
|
||||
) {
|
||||
const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
|
||||
): TailPlan {
|
||||
const rows: TailRow[] = []
|
||||
const extended: TailPlan['extended'] = []
|
||||
for (const g of groups) {
|
||||
if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
|
||||
const waiting = g.high - g.plannedTo
|
||||
const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
|
||||
if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
|
||||
for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
|
||||
let start = g.plannedTo + 1
|
||||
if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
|
||||
const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
|
||||
extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
|
||||
start = end + 1
|
||||
}
|
||||
for (; start <= g.high; start += options.rangeArticles) {
|
||||
rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
|
||||
}
|
||||
}
|
||||
return { rows, extended }
|
||||
}
|
||||
|
||||
export interface TailMerge {
|
||||
/** The run's first range, grown to end_id. */
|
||||
id: number
|
||||
end_id: number
|
||||
/** The ranges it absorbed. */
|
||||
drop: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
|
||||
* `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
|
||||
*/
|
||||
export function tailMerges(
|
||||
ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
|
||||
rangeArticles: number
|
||||
): TailMerge[] {
|
||||
const merges: TailMerge[] = []
|
||||
let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
|
||||
const flush = () => {
|
||||
if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
|
||||
}
|
||||
for (const r of ranges) {
|
||||
if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
|
||||
cur.end_id = r.end_id
|
||||
cur.drop.push(r.id)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
|
||||
}
|
||||
flush()
|
||||
return merges
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows a group's history is cut into, from `lastEnd + 1` (0: the group has no ranges yet) up to the high
|
||||
* watermark: RANGE_ARTICLES slices at the position priority. A group planned for the first time also gets one
|
||||
* "ancient" range, articles 1 … low − 1: what had expired on the server before it was planned. Status
|
||||
* 'ancient', never leased or fetched; it keeps the tiling complete from article 1 (migration 6700 added it for
|
||||
* the groups planned before 2026-10-28).
|
||||
*/
|
||||
export function historyRows(
|
||||
groupId: number,
|
||||
info: { low: number; high: number },
|
||||
lastEnd: number,
|
||||
size: number
|
||||
): Record<string, unknown>[] {
|
||||
const rows: Record<string, unknown>[] = []
|
||||
if (lastEnd === 0 && info.low > 1)
|
||||
rows.push({ group_id: groupId, start_id: 1, end_id: info.low - 1, next_id: 1, priority: 0, status: 'ancient' })
|
||||
const from = Math.max(info.low, lastEnd + 1)
|
||||
const span = Math.max(1, info.high - info.low)
|
||||
for (let start = from; start <= info.high; start += size) {
|
||||
rows.push({
|
||||
group_id: groupId,
|
||||
start_id: start,
|
||||
end_id: Math.min(start + size - 1, info.high),
|
||||
next_id: start,
|
||||
priority: Math.floor((1000 * (start - info.low)) / span),
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
+197
-28
@@ -3,7 +3,8 @@ import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import Corpus, { type CorpusFilter } from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
|
||||
|
||||
@@ -72,7 +73,13 @@ export class Prefetcher {
|
||||
private writingNow = 0
|
||||
private sampler?: NodeJS.Timeout
|
||||
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
|
||||
private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
|
||||
private inFlight = new Set<{
|
||||
rangeId: number
|
||||
from: number
|
||||
to: number
|
||||
since: number
|
||||
client: XoverClient
|
||||
}>()
|
||||
/** Capture ranges found complete in the corpus (not looked at again). */
|
||||
private stored = new Set<number>()
|
||||
|
||||
@@ -131,6 +138,8 @@ export class Prefetcher {
|
||||
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
|
||||
.finally(() => this.running--)
|
||||
}
|
||||
// Work left with every loop running: the pool's share could be bigger.
|
||||
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
|
||||
}
|
||||
|
||||
stop() {
|
||||
@@ -145,6 +154,9 @@ export class Prefetcher {
|
||||
/** One connection's loop: chunks until there is nothing left to fetch. */
|
||||
private async work() {
|
||||
while (!this.stopped) {
|
||||
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
|
||||
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
|
||||
if (this.running > this.loops) return
|
||||
const next = this.take()
|
||||
if (!next) return
|
||||
const { target, span } = next
|
||||
@@ -157,10 +169,14 @@ export class Prefetcher {
|
||||
}
|
||||
const corpusRange = { group: target.group, start: target.start, end: target.end }
|
||||
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
|
||||
let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||||
let data =
|
||||
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||||
const stored = data !== null
|
||||
try {
|
||||
if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
|
||||
if (!data)
|
||||
data = target.plain.has(span.from)
|
||||
? await NntpPool.usePlain(fetch)
|
||||
: await this.fetchHedged(target.id, target.group, span)
|
||||
} finally {
|
||||
this.fetchingNow--
|
||||
}
|
||||
@@ -170,7 +186,8 @@ export class Prefetcher {
|
||||
this.stats.chunks++
|
||||
this.stats.articles += span.to - span.from + 1
|
||||
this.stats.fetchMs += Date.now() - fetchedAt
|
||||
if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
|
||||
if (target.capture)
|
||||
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
|
||||
}
|
||||
if (target.corpusOnly) {
|
||||
this.stats.captured++
|
||||
@@ -187,7 +204,8 @@ export class Prefetcher {
|
||||
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
|
||||
[target.id]
|
||||
)
|
||||
if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||||
if (Number(rows[0]?.spooled_chunks) >= target.total)
|
||||
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||||
} finally {
|
||||
this.writingNow--
|
||||
this.stats.writeMs += Date.now() - writtenAt
|
||||
@@ -209,7 +227,10 @@ export class Prefetcher {
|
||||
*/
|
||||
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
|
||||
let age: number | undefined
|
||||
const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
|
||||
const request = (
|
||||
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
|
||||
first: boolean
|
||||
): Request<Buffer> | null => {
|
||||
let client: XoverClient | undefined
|
||||
let cancelled = false
|
||||
const done = start(async (c) => {
|
||||
@@ -241,7 +262,9 @@ export class Prefetcher {
|
||||
data = await c.xoverRaw(span.from, span.to)
|
||||
} catch (error) {
|
||||
if (!late) throw error
|
||||
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
|
||||
throw new Error(
|
||||
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
|
||||
)
|
||||
} finally {
|
||||
clearInterval(watch)
|
||||
this.inFlight.delete(entry)
|
||||
@@ -284,7 +307,8 @@ export class Prefetcher {
|
||||
|
||||
/** The longest-running request: range, chunk, how long, how much came in. */
|
||||
private oldest() {
|
||||
let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||||
let oldest:
|
||||
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||||
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
|
||||
if (!oldest) return 'none in flight'
|
||||
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
|
||||
@@ -309,7 +333,12 @@ export class Prefetcher {
|
||||
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
|
||||
if (chunks > 0) {
|
||||
const ages = byAge
|
||||
.map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
|
||||
.map((b, i) =>
|
||||
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
|
||||
'– min',
|
||||
'+ min'
|
||||
)
|
||||
)
|
||||
.join(', ')
|
||||
logger.info(
|
||||
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
|
||||
@@ -335,13 +364,18 @@ export class Prefetcher {
|
||||
}
|
||||
|
||||
private peek(): boolean {
|
||||
return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
|
||||
return this.targets.some(
|
||||
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether the target is under its range's and its group's limit of connections. */
|
||||
private open(target: Target) {
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||||
return (
|
||||
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
|
||||
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||||
)
|
||||
}
|
||||
|
||||
private count(target: Target, by: number) {
|
||||
@@ -372,66 +406,201 @@ export class Prefetcher {
|
||||
this.sizedAt = Date.now()
|
||||
this.spoolBytes = await Spool.size()
|
||||
}
|
||||
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
|
||||
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
|
||||
const mine = (q: any) =>
|
||||
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
|
||||
const rows = (q: any) =>
|
||||
mine(
|
||||
q
|
||||
.from('ranges as r')
|
||||
.join('groups as g', 'g.id', 'r.group_id')
|
||||
.select(
|
||||
'r.id',
|
||||
'g.name as group',
|
||||
'r.start_id',
|
||||
'r.end_id',
|
||||
'r.next_id',
|
||||
'r.plain_chunks',
|
||||
'r.capture'
|
||||
)
|
||||
)
|
||||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||||
// The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
|
||||
// cursors (a group's history and tail lanes are two cursors).
|
||||
const held: { group: number; lane: string }[] = (
|
||||
await db
|
||||
.from('group_leases')
|
||||
.whereNotNull('node_id')
|
||||
.where(
|
||||
'heartbeat_at',
|
||||
'>=',
|
||||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||||
)
|
||||
.select('group_id', 'lane')
|
||||
).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
|
||||
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
|
||||
const ahead = Math.max(
|
||||
coordinatorConfig.spool.minAhead,
|
||||
coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
|
||||
)
|
||||
const started = await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', '>', 0)
|
||||
.whereNull('r.spooled_at')
|
||||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await mine(
|
||||
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
|
||||
).count('* as ready')
|
||||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||||
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
|
||||
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
|
||||
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
|
||||
const { perRange } = coordinatorConfig.spool
|
||||
const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||||
const short =
|
||||
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||||
const want = Math.max(ahead - Number(ready), short)
|
||||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||||
const full = await CatalogSpace.full()
|
||||
const fresh = room && want > 0
|
||||
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
|
||||
: []
|
||||
// Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
|
||||
// leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
|
||||
// at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
|
||||
// window worker running) nothing is fetched ahead for workers.
|
||||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||||
const tailPriority = coordinatorConfig.tail.priority
|
||||
const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
|
||||
const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
|
||||
const fresh =
|
||||
room && want > 0 && heldHistory.length + heldTail.length > 0
|
||||
? await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', 0)
|
||||
.whereRaw('r.next_id <= r.end_id')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((h: any) =>
|
||||
h
|
||||
.whereIn('r.group_id', heldHistory)
|
||||
.where('r.priority', '>', tailPriority)
|
||||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||||
.where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
|
||||
)
|
||||
.orWhere((t: any) =>
|
||||
t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
|
||||
)
|
||||
)
|
||||
.orderByRaw(
|
||||
'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
|
||||
[tailPriority]
|
||||
)
|
||||
.limit(want)
|
||||
: []
|
||||
const targets: Target[] = []
|
||||
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const total = Spool.chunksOf({ start, end }).length
|
||||
const missing: ChunkSpan[] = []
|
||||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||||
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||||
continue
|
||||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||||
missing.push(span)
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
|
||||
targets.push({
|
||||
id: Number(row.id),
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
capture: row.capture === true,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
|
||||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
|
||||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
|
||||
targets.push(target)
|
||||
return targets
|
||||
}
|
||||
|
||||
/** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
|
||||
private filterRules: { at: number; filter: CorpusFilter } | null = null
|
||||
|
||||
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
|
||||
private async corpusFilter(): Promise<CorpusFilter | undefined> {
|
||||
if (!coordinatorConfig.corpus.filter) return undefined
|
||||
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
|
||||
const rules = await RuleSets.current()
|
||||
this.filterRules = {
|
||||
at: Date.now(),
|
||||
filter: { ruleVersion: rules.version, priorities: rules.priorities },
|
||||
}
|
||||
}
|
||||
return this.filterRules.filter
|
||||
}
|
||||
|
||||
/**
|
||||
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
|
||||
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
|
||||
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
|
||||
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
|
||||
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
|
||||
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
|
||||
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
|
||||
* gets stored_at and isn't read again.
|
||||
*/
|
||||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||||
const { dir, ahead } = coordinatorConfig.corpus
|
||||
if (!dir || ahead <= 0) return []
|
||||
const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
const ofGroup =
|
||||
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||||
const groups = new Map<string, number>()
|
||||
const candidates = await rows(db.query())
|
||||
.where('r.capture', true)
|
||||
.whereNull('r.stored_at')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
|
||||
.orWhere('r.status', 'done')
|
||||
)
|
||||
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
|
||||
.limit(ahead * 16)
|
||||
const targets: Target[] = []
|
||||
for (const row of candidates) {
|
||||
if (targets.length >= ahead) break
|
||||
const id = Number(row.id)
|
||||
if (taken.has(id) || this.stored.has(id)) continue
|
||||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const range = { group: row.group, start, end }
|
||||
if (await Corpus.complete(range)) {
|
||||
this.stored.add(id)
|
||||
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
|
||||
continue
|
||||
}
|
||||
const chunks = Spool.chunksOf({ start, end })
|
||||
const missing: ChunkSpan[] = []
|
||||
for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
|
||||
for (const span of chunks)
|
||||
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
|
||||
missing.push(span)
|
||||
if (missing.length > 0) {
|
||||
targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
|
||||
targets.push({
|
||||
id,
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total: chunks.length,
|
||||
start,
|
||||
end,
|
||||
capture: true,
|
||||
corpusOnly: true,
|
||||
})
|
||||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
return targets
|
||||
|
||||
+222
-73
@@ -6,19 +6,27 @@ import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { YencError } from '#lib/yenc/index'
|
||||
import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
|
||||
import { PendingNamer } from '#services/pending_namer'
|
||||
|
||||
interface Candidate {
|
||||
id: number
|
||||
releaseId: number
|
||||
kind: string
|
||||
messageId: string
|
||||
/** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
|
||||
pending?: PendingProbe
|
||||
}
|
||||
|
||||
type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
|
||||
|
||||
/**
|
||||
* The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
|
||||
* then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
|
||||
*/
|
||||
const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'lone', unreadable: false },
|
||||
@@ -26,8 +34,35 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'archive', unreadable: true },
|
||||
{ kind: 'par2', unreadable: false },
|
||||
{ kind: 'archive', unreadable: false },
|
||||
{ kind: '7z-tail', unreadable: true },
|
||||
{ kind: '7z-tail', unreadable: false },
|
||||
]
|
||||
const FLUSH_MS = 2000
|
||||
/** Between sweeps that discard the waiting releases of dropped signatures. */
|
||||
const SWEEP_MS = 5 * 60_000
|
||||
|
||||
/**
|
||||
* JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
|
||||
* and the whole flush failed with it (LOG.md 2026-10-18 (78)).
|
||||
*/
|
||||
function jsonb(value: unknown): string {
|
||||
return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
|
||||
}
|
||||
|
||||
/** Rounds of every lane's pipeline still queued when the next claim starts. */
|
||||
const LOW_ROUNDS = 8
|
||||
|
||||
/**
|
||||
* The candidate queue's levels: the next claim starts below `low`, LOW_ROUNDS rounds of every lane's pipeline (at
|
||||
* ~0.2 s an article and a pipeline of 8, ~13 s of work), and fills it to `target`, twice that. A claim takes 1.5–4 s
|
||||
* and more (random slices of an 11 M-row table); refilling at half of four rounds left 3–6 s, and at 20–24 lanes the
|
||||
* lanes waited for candidates up to 19% of their time (LOG.md 2026-10-28 (120)). At least `minBatch` either way.
|
||||
*/
|
||||
export function queueLevels(lanes: number, pipeline: number, minBatch: number): { low: number; target: number } {
|
||||
const low = Math.max(minBatch, lanes * pipeline * LOW_ROUNDS)
|
||||
return { low, target: 2 * low }
|
||||
}
|
||||
|
||||
/** Slices a claim takes from random points of the queue. */
|
||||
const SLICES = 20
|
||||
const REPORT_MS = 60_000
|
||||
@@ -46,18 +81,31 @@ export class Prober {
|
||||
private refilling: Promise<void> | null = null
|
||||
private stopped = false
|
||||
private stats = Prober.freshStats()
|
||||
/** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
|
||||
private lanes = new Set<Promise<void>>()
|
||||
private running = 0
|
||||
/** The last lane to look found nothing to claim: the prober wants its minimum only. */
|
||||
private dry = false
|
||||
/** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */
|
||||
private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb)
|
||||
private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections)
|
||||
|
||||
/** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
|
||||
private static freshStats() {
|
||||
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
|
||||
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
|
||||
}
|
||||
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
|
||||
private namer = new PendingNamer()
|
||||
private sweptAt = 0
|
||||
|
||||
async run() {
|
||||
// A previous run's claims: one prober at a time.
|
||||
const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
|
||||
if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
|
||||
const connections = coordinatorConfig.probe.connections
|
||||
logger.info(`Probing with ${connections} connections.`)
|
||||
const released = await PendingLadder.release()
|
||||
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
|
||||
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
|
||||
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
|
||||
const flusher = (async () => {
|
||||
while (!this.stopped || this.outcomes.length > 0) {
|
||||
await sleep(FLUSH_MS)
|
||||
@@ -67,7 +115,27 @@ export class Prober {
|
||||
this.report()
|
||||
}
|
||||
})()
|
||||
await Promise.all(Array.from({ length: connections }, () => this.lane()))
|
||||
// Lanes as the share allows: the coordinator's history comes first (app/services/nntp_shares.ts).
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
const was = this.share.granted
|
||||
const granted = await this.share.sync(this.dry ? 0 : this.share.max, this.running)
|
||||
if (granted !== was) logger.info(`Probing: ${granted} connections (was ${was}).`)
|
||||
while (this.running < granted) {
|
||||
this.running++
|
||||
const lane: Promise<void> = this.lane()
|
||||
.catch((error) => logger.error(`Probing: a lane failed: ${error.message}`))
|
||||
.finally(() => this.lanes.delete(lane))
|
||||
this.lanes.add(lane)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Probing: NNTP share failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
|
||||
}
|
||||
await Promise.all(this.lanes)
|
||||
await namer
|
||||
await this.share.leave().catch(() => {})
|
||||
// What was claimed and not probed goes back.
|
||||
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
this.stopped = true
|
||||
@@ -78,68 +146,95 @@ export class Prober {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
/** One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure. */
|
||||
/**
|
||||
* One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure; done when
|
||||
* there are more lanes than the share (its connection closed, the coordinator takes it on its next sync).
|
||||
*/
|
||||
private async lane() {
|
||||
let client: XoverClient | null = null
|
||||
while (!this.stopped) {
|
||||
const batch: Candidate[] = []
|
||||
const waited = Date.now()
|
||||
while (batch.length < coordinatorConfig.probe.pipeline) {
|
||||
const candidate = await this.next()
|
||||
if (!candidate) break
|
||||
batch.push(candidate)
|
||||
}
|
||||
this.stats.waitMs += Date.now() - waited
|
||||
if (batch.length === 0) {
|
||||
this.stats.idleMs += 30_000
|
||||
await sleep(30_000)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const fetched = Date.now()
|
||||
const bodies = await client.bodies(batch.map((c) => c.messageId))
|
||||
this.stats.fetchMs += Date.now() - fetched
|
||||
const inspected = Date.now()
|
||||
for (const [i, candidate] of batch.entries()) {
|
||||
const body = bodies[i]
|
||||
this.stats.articles++
|
||||
if (body === null) {
|
||||
// Taken down or expired: final.
|
||||
this.outcomes.push({ candidate, status: 'missing' })
|
||||
continue
|
||||
let usedAt = Date.now()
|
||||
try {
|
||||
while (!this.stopped) {
|
||||
if (this.running > this.share.granted) break
|
||||
const batch: Candidate[] = []
|
||||
const waited = Date.now()
|
||||
while (batch.length < coordinatorConfig.probe.pipeline) {
|
||||
const candidate = await this.next()
|
||||
if (!candidate) break
|
||||
batch.push(candidate)
|
||||
}
|
||||
this.stats.waitMs += Date.now() - waited
|
||||
this.dry = batch.length === 0
|
||||
if (batch.length === 0) {
|
||||
// Nothing to probe for a while: the connection closes (nntpIdleCloseMs), and opens again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (client && idleMs > 0 && Date.now() - usedAt >= idleMs) {
|
||||
client.close()
|
||||
client = null
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
this.outcomes.push({ candidate, status: 'error' })
|
||||
this.stats.idleMs += 30_000
|
||||
await sleep(30_000)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const fetched = Date.now()
|
||||
const bodies = await client.bodies(batch.map((c) => c.messageId))
|
||||
this.stats.fetchMs += Date.now() - fetched
|
||||
const inspected = Date.now()
|
||||
for (const [i, candidate] of batch.entries()) {
|
||||
const body = bodies[i]
|
||||
this.stats.articles++
|
||||
if (body === null) {
|
||||
// Taken down or expired: final.
|
||||
this.outcomes.push({ candidate, status: 'missing' })
|
||||
continue
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
this.outcomes.push({ candidate, status: 'error' })
|
||||
}
|
||||
}
|
||||
this.stats.inspectMs += Date.now() - inspected
|
||||
usedAt = Date.now()
|
||||
} catch (error: any) {
|
||||
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
|
||||
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
client?.close()
|
||||
client = null
|
||||
await sleep(5000)
|
||||
}
|
||||
this.stats.inspectMs += Date.now() - inspected
|
||||
} catch (error: any) {
|
||||
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
|
||||
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
client?.close()
|
||||
client = null
|
||||
await sleep(5000)
|
||||
}
|
||||
} finally {
|
||||
client?.close()
|
||||
this.running--
|
||||
}
|
||||
client?.close()
|
||||
}
|
||||
|
||||
/** The queue's low-water mark and the level a claim fills it to (queueLevels). */
|
||||
private get levels() {
|
||||
return queueLevels(this.running, coordinatorConfig.probe.pipeline, coordinatorConfig.probe.batch)
|
||||
}
|
||||
|
||||
/** Claims to the queue's target; the claim after starts when it's below its low-water mark (queueLevels). */
|
||||
private get batch() {
|
||||
return this.levels.target
|
||||
}
|
||||
|
||||
private async next(): Promise<Candidate | undefined> {
|
||||
if (this.queue.length < coordinatorConfig.probe.batch / 2) {
|
||||
if (this.queue.length < this.levels.low) {
|
||||
this.refilling ??= this.claim().finally(() => (this.refilling = null))
|
||||
if (this.queue.length === 0) await this.refilling
|
||||
}
|
||||
@@ -162,13 +257,55 @@ export class Prober {
|
||||
}
|
||||
|
||||
private async claimTiers() {
|
||||
// The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
|
||||
if (await this.space.short()) return
|
||||
await this.claimCatalog()
|
||||
await this.claimPending()
|
||||
}
|
||||
|
||||
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
|
||||
private async claimPending() {
|
||||
if (Date.now() - this.sweptAt > SWEEP_MS) {
|
||||
this.sweptAt = Date.now()
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
}
|
||||
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
|
||||
const probes = await PendingLadder.claim(want)
|
||||
if (probes.length === 0) return
|
||||
const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
|
||||
probes.map((p) => p.messageId),
|
||||
PROBE_VERSION,
|
||||
])
|
||||
const known = new Map(cached.map((c: any) => [c.message_id, c]))
|
||||
for (const probe of probes) {
|
||||
const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
|
||||
const hit: any = known.get(probe.messageId)
|
||||
if (hit) {
|
||||
this.stats.cached++
|
||||
this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
|
||||
} else {
|
||||
this.queue.push(candidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async claimCatalog() {
|
||||
for (const { kind, unreadable } of TIERS) {
|
||||
const want = coordinatorConfig.probe.batch - this.queue.length
|
||||
const want = this.batch - this.queue.length
|
||||
if (want <= 0) return
|
||||
// The starting points chosen here: in the query Postgres drew random() once for all slices.
|
||||
// Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
|
||||
// Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting
|
||||
// entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)).
|
||||
const values = unreadable ? [true] : [false, true]
|
||||
const ends = values
|
||||
.map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`)
|
||||
.join(' UNION ALL ')
|
||||
const [{ lo, hi }] = (
|
||||
await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
|
||||
await db.rawQuery(
|
||||
`SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`,
|
||||
values.flatMap((v) => [kind, v])
|
||||
)
|
||||
).rows
|
||||
if (lo === null) continue
|
||||
const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
|
||||
@@ -203,22 +340,31 @@ export class Prober {
|
||||
|
||||
/** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
|
||||
private async flush() {
|
||||
const outcomes = this.outcomes.splice(0)
|
||||
if (outcomes.length === 0) return
|
||||
for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
const taken = this.outcomes.splice(0)
|
||||
if (taken.length === 0) return
|
||||
for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
// The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
|
||||
const pending = taken.filter((o) => o.candidate.pending)
|
||||
const outcomes = taken.filter((o) => !o.candidate.pending)
|
||||
this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
|
||||
[JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
|
||||
)
|
||||
if (outcomes.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
|
||||
[jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
|
||||
)
|
||||
}
|
||||
await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
|
||||
// Final answers read from the news server, kept by message-id for the next catalog (a reset).
|
||||
const fresh = outcomes.filter((o) => !o.cached && o.status !== 'waiting')
|
||||
// One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
|
||||
// row twice in one statement.
|
||||
const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
|
||||
if (fresh.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO probe_cache (message_id, status, result, version)
|
||||
SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
|
||||
ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
|
||||
[PROBE_VERSION, JSON.stringify(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
|
||||
[PROBE_VERSION, jsonb(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
|
||||
)
|
||||
}
|
||||
const done = outcomes.filter((o) => o.inspection)
|
||||
@@ -228,9 +374,11 @@ export class Prober {
|
||||
SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
|
||||
ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
|
||||
// A par2 set's file list goes to par2_files, not into every inspection.
|
||||
[JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
[jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
)
|
||||
await this.storeForMerge(trx, done)
|
||||
// A par2 index read: the release's completeness by its set, when it has no counter.
|
||||
await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
|
||||
// Each touched release from all its inspections (earlier rounds may have stored others).
|
||||
const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
|
||||
const { rows } = await trx.rawQuery(
|
||||
@@ -252,7 +400,7 @@ export class Prober {
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
|
||||
[JSON.stringify(updates)]
|
||||
[jsonb(updates)]
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -265,7 +413,7 @@ export class Prober {
|
||||
`INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
|
||||
SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
|
||||
ON CONFLICT (candidate_id) DO NOTHING`,
|
||||
[JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
|
||||
[jsonb(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
|
||||
)
|
||||
}
|
||||
for (const o of done) {
|
||||
@@ -275,7 +423,7 @@ export class Prober {
|
||||
`INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
|
||||
SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
|
||||
[set.setId, jsonb(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
|
||||
)
|
||||
await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
|
||||
}
|
||||
@@ -287,7 +435,8 @@ export class Prober {
|
||||
const { articles, bytes, done, missing, error } = this.stats
|
||||
logger.info(
|
||||
`Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
|
||||
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
|
||||
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
|
||||
`the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
|
||||
`Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
|
||||
`idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
|
||||
`claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
|
||||
* release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
|
||||
* range that isn't done or doesn't start where the one before ended.
|
||||
*/
|
||||
class RangeBlocks {
|
||||
async rebuild(): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`DELETE FROM range_blocks`)
|
||||
const result = await trx.rawQuery(
|
||||
`WITH r AS (
|
||||
SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
|
||||
min(start_id) OVER (PARTITION BY group_id) AS group_first,
|
||||
lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
|
||||
FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
|
||||
runs AS (
|
||||
SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
|
||||
OVER (PARTITION BY group_id ORDER BY start_id) AS run
|
||||
FROM r)
|
||||
INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
|
||||
SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
|
||||
FROM runs WHERE done GROUP BY group_id, run`
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new RangeBlocks()
|
||||
+184
-53
@@ -1,10 +1,19 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again. */
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
|
||||
export const JUDGE_VERSION = 1
|
||||
/** Between rounds: each rule is a scan of its table. */
|
||||
const ROUND_MS = 60 * 60 * 1000
|
||||
/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
|
||||
const ROUND_MS = 2 * 60 * 1000
|
||||
/**
|
||||
* A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
|
||||
* (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
|
||||
* after a merge (repeated).
|
||||
*/
|
||||
const FULL_MS = 24 * 60 * 60 * 1000
|
||||
/** Changed releases judged at a time. */
|
||||
const CHUNK = 50_000
|
||||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||||
const REPEATED_RELEASES = 20
|
||||
|
||||
@@ -24,10 +33,15 @@ export interface Rule {
|
||||
name: string
|
||||
/** Why, in a few words (the log, the web interface). */
|
||||
says: string
|
||||
/** SELECT of the release ids the rule judges. */
|
||||
sql: string
|
||||
/** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
|
||||
sql: (scoped: boolean) => string
|
||||
}
|
||||
|
||||
/** A join to the releases in scope (an incremental round), on the given release id column. */
|
||||
const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||||
* the header pass, the prober, the merges and the spots know by now.
|
||||
@@ -36,26 +50,33 @@ export const RULES: Rule[] = [
|
||||
{
|
||||
name: 'dead',
|
||||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||||
sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
|
||||
sql: (scoped) =>
|
||||
`SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
|
||||
WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
|
||||
},
|
||||
{
|
||||
name: 'fake',
|
||||
says: 'named as a film, episode or music, but the content is a program',
|
||||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||||
// an executable by name or by its first bytes, under a media name.
|
||||
sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
|
||||
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||||
UNION
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||||
},
|
||||
{
|
||||
name: 'repeated',
|
||||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||||
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
// Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
|
||||
// (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
|
||||
: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||||
},
|
||||
{
|
||||
@@ -64,21 +85,28 @@ export const RULES: Rule[] = [
|
||||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||||
sql: `SELECT r.id FROM releases r
|
||||
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||||
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
|
||||
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
|
||||
: `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
|
||||
},
|
||||
{
|
||||
name: 'locked',
|
||||
says: 'passworded, and no spot, NZB or title gives the password',
|
||||
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||
// Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
|
||||
sql: `SELECT r.id FROM releases r
|
||||
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||
AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
|
||||
AND r.settled_at IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||
@@ -86,15 +114,18 @@ export const RULES: Rule[] = [
|
||||
]
|
||||
|
||||
export interface JudgeRound {
|
||||
/** Per rule: releases it judges now, verdicts added, verdicts taken back. */
|
||||
/** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
|
||||
releases?: number
|
||||
/** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
|
||||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||||
}
|
||||
|
||||
/**
|
||||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
|
||||
* round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
|
||||
* matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
|
||||
* minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
|
||||
* daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
|
||||
* verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
*/
|
||||
export class ReleaseJudge {
|
||||
private stopped = false
|
||||
@@ -107,9 +138,13 @@ export class ReleaseJudge {
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
const round = await this.round()
|
||||
const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
|
||||
logger.info(`Judged: ${parts.join(', ')}.`)
|
||||
if (await this.fullDue()) {
|
||||
const round = await this.full()
|
||||
if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
|
||||
} else {
|
||||
const round = await this.changed()
|
||||
if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Judge round failed: ${error.message}`)
|
||||
}
|
||||
@@ -117,43 +152,139 @@ export class ReleaseJudge {
|
||||
}
|
||||
}
|
||||
|
||||
/** One round over every rule; with dry, only counts. */
|
||||
async round(dry = false): Promise<JudgeRound> {
|
||||
/** A full round is due: a day after the last, or the last was by another rule set. */
|
||||
async fullDue(): Promise<boolean> {
|
||||
const state = await db.from('judge_state').first()
|
||||
return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
|
||||
}
|
||||
|
||||
/**
|
||||
* Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
|
||||
* half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
|
||||
*/
|
||||
async full(dry = false): Promise<JudgeRound> {
|
||||
const started = new Date()
|
||||
const round: JudgeRound = { rules: {} }
|
||||
// Verdicts of an older rule set are judged again from scratch.
|
||||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||||
const judged = new Set<string>()
|
||||
await db.rawQuery(`TRUNCATE judge_full`)
|
||||
for (const rule of RULES) {
|
||||
if (this.stopped) break
|
||||
const { rows } = await db.rawQuery(rule.sql)
|
||||
const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
|
||||
for (const id of ids) judged.add(id)
|
||||
const stats = { matched: ids.length, added: 0, removed: 0 }
|
||||
if (!dry) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
for (let i = 0; i < ids.length; i += 50_000) {
|
||||
await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
|
||||
}
|
||||
const removed = await trx.rawQuery(
|
||||
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
|
||||
if (this.stopped) return round
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
|
||||
}
|
||||
if (dry || this.stopped) return round
|
||||
await db.rawQuery(`ANALYZE judge_full`)
|
||||
await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
|
||||
await db.rawQuery(
|
||||
`INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
|
||||
[started, JUDGE_VERSION]
|
||||
)
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
|
||||
* A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
|
||||
* already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
|
||||
* file in that many without them had its releases judged before.
|
||||
*/
|
||||
async changed(): Promise<JudgeRound> {
|
||||
const round: JudgeRound = { releases: 0, rules: {} }
|
||||
for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
|
||||
let more = true
|
||||
while (more && !this.stopped) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
|
||||
scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
|
||||
SELECT (SELECT count(*) FROM taken) AS taken`
|
||||
)
|
||||
const taken = Number(rows[0].taken)
|
||||
more = taken === CHUNK
|
||||
if (taken === 0) return
|
||||
// The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
|
||||
await trx.rawQuery(
|
||||
`CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_tipped
|
||||
SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
|
||||
FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT count(DISTINCT f.release_id) AS total,
|
||||
count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
|
||||
FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
|
||||
WHERE c.total >= ${REPEATED_RELEASES}`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
|
||||
WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
|
||||
ON CONFLICT DO NOTHING`
|
||||
)
|
||||
await trx.rawQuery(`ANALYZE judge_scope`)
|
||||
const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
|
||||
round.releases! += Number(scope[0].n)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
|
||||
for (const rule of RULES) {
|
||||
const { rowCount } = await trx.rawQuery(
|
||||
`INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
const added = await trx.rawQuery(
|
||||
`INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule`,
|
||||
[rule.name, JUDGE_VERSION]
|
||||
)
|
||||
stats.removed = removed.rowCount ?? 0
|
||||
stats.added = added.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
round.rules[rule.name] = stats
|
||||
round.rules[rule.name].matched += rowCount ?? 0
|
||||
}
|
||||
await trx.rawQuery(`ANALYZE judged`)
|
||||
await this.apply(trx, 'judged', true, round)
|
||||
})
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
|
||||
* more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
|
||||
* insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
|
||||
*/
|
||||
private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
|
||||
const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
|
||||
for (const { rule, n } of rows) {
|
||||
round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
|
||||
round.rules[rule][field] += Number(n)
|
||||
}
|
||||
}
|
||||
const { rows: removed } = await trx.rawQuery(
|
||||
`WITH gone AS (DELETE FROM release_verdicts v
|
||||
WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
|
||||
NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
|
||||
RETURNING v.rule)
|
||||
SELECT rule, count(*) AS n FROM gone GROUP BY 1`
|
||||
)
|
||||
tally(removed, 'removed')
|
||||
const { rows: added } = await trx.rawQuery(
|
||||
`WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
|
||||
WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule
|
||||
RETURNING rule)
|
||||
SELECT rule, count(*) AS n FROM written GROUP BY 1`,
|
||||
[JUDGE_VERSION]
|
||||
)
|
||||
tally(added, 'added')
|
||||
}
|
||||
}
|
||||
|
||||
/** "dead 1,157,018 (+1,136 −712), …" for the log. */
|
||||
function describe(round: JudgeRound): string {
|
||||
return Object.entries(round.rules)
|
||||
.map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
|
||||
.join(', ')
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { SOURCES, sourceName } from '#lib/release/source'
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
@@ -51,12 +52,14 @@ export interface ReleaseSummary {
|
||||
size: number
|
||||
readable: boolean
|
||||
complete: boolean | null
|
||||
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
|
||||
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
|
||||
final: boolean
|
||||
category: number
|
||||
settled: boolean
|
||||
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
||||
passworded: boolean | null
|
||||
/** A password is known (release_passwords): it's in the NZB. */
|
||||
passwordKnown: boolean
|
||||
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
||||
innerName: string | null
|
||||
}
|
||||
@@ -89,27 +92,32 @@ class SearchService {
|
||||
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
||||
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
||||
const bindings: unknown[] = []
|
||||
// Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
|
||||
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
|
||||
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike.
|
||||
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
||||
where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
|
||||
bindings.push(contains(word), contains(word))
|
||||
where.push('r.id IN (SELECT t.release_id FROM release_texts t WHERE t.text ILIKE ?)')
|
||||
bindings.push(contains(word))
|
||||
}
|
||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||
if (params.complete) where.push('r.complete')
|
||||
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
|
||||
// Incomplete only when final: a provisional tail release may yet get its missing files.
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
|
||||
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(params.source)
|
||||
const code = (SOURCES as Record<string, number>)[params.source]
|
||||
if (code === undefined) where.push('false')
|
||||
else {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(code)
|
||||
}
|
||||
}
|
||||
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
||||
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
||||
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
||||
let posters: string[] | null = null
|
||||
let posters: number[] | null = null
|
||||
if (params.poster) {
|
||||
const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||
posters = matched.map((m: { poster: string }) => m.poster)
|
||||
const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||
posters = matched.map((m: { id: string }) => Number(m.id))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
@@ -141,14 +149,14 @@ class SearchService {
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
||||
// Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
|
||||
// bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
|
||||
// Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
|
||||
// in the order asked; then merged.
|
||||
const source =
|
||||
posters === null
|
||||
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
||||
: {
|
||||
from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||
SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||
from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||
where: 'true',
|
||||
bindings: [posters] as unknown[],
|
||||
}
|
||||
@@ -156,15 +164,17 @@ class SearchService {
|
||||
posters === null
|
||||
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
||||
: db.rawQuery(
|
||||
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||
SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||
[posters, ...bindings]
|
||||
)
|
||||
const [{ rows }, { rows: counted }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
||||
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
`SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
||||
r.settled_at IS NOT NULL AS final
|
||||
FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
|
||||
WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...source.bindings, ...bindings, perPage, offset]
|
||||
),
|
||||
counting,
|
||||
@@ -179,9 +189,9 @@ class SearchService {
|
||||
releases: (rows as any[]).map((r): ReleaseSummary => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
@@ -192,6 +202,7 @@ class SearchService {
|
||||
category: r.category,
|
||||
settled: r.settled_at !== null,
|
||||
passworded: r.passworded,
|
||||
passwordKnown: r.password_known,
|
||||
innerName: r.inner_name,
|
||||
})),
|
||||
}
|
||||
@@ -199,19 +210,24 @@ class SearchService {
|
||||
|
||||
/** A release with its files, read from its chunks. */
|
||||
async release(id: number) {
|
||||
const r = await db.from('releases').where('id', id).first()
|
||||
const {
|
||||
rows: [r],
|
||||
} = await db.rawQuery(
|
||||
`SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (!r) return null
|
||||
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
|
||||
const [files, { rows: inspections }, passwords] = await Promise.all([
|
||||
NzbService.files(id),
|
||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
||||
NzbService.passwords(id),
|
||||
])
|
||||
return {
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
@@ -219,13 +235,14 @@ class SearchService {
|
||||
segments: Number(r.segments),
|
||||
readable: r.readable,
|
||||
complete: r.complete,
|
||||
final: finality[0].final,
|
||||
final: r.settled_at !== null,
|
||||
hasPar2: r.has_par2,
|
||||
category: r.category,
|
||||
firstSeenAt: r.first_seen_at,
|
||||
lastSeenAt: r.last_seen_at,
|
||||
settledAt: r.settled_at,
|
||||
passworded: r.passworded,
|
||||
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
|
||||
passwords,
|
||||
innerName: r.inner_name,
|
||||
inspectedAt: r.inspected_at,
|
||||
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
||||
@@ -287,9 +304,9 @@ class SearchService {
|
||||
// History by articles; the pass: the history ranges before those deferred behind them (priority
|
||||
// 2000 apart; e.g. groups added later that go first).
|
||||
const { rows: ranges } = await db.rawQuery(
|
||||
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
|
||||
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
|
||||
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
|
||||
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
|
||||
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
|
||||
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
|
||||
|
||||
+25
-36
@@ -1,46 +1,35 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
|
||||
const QUIET_HOURS = 6
|
||||
/**
|
||||
* History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
|
||||
* neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
|
||||
* Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
|
||||
* each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
|
||||
*/
|
||||
const QUIET_HISTORY_MINUTES = 30
|
||||
const BATCH = 50_000
|
||||
/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
|
||||
export const SETTLE_COMPLETE_HOURS = 6
|
||||
|
||||
/**
|
||||
* Settles releases (REBUILD.md, part 8): once nothing has been added for
|
||||
* QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
|
||||
* [i/N] counter announced was found, unknown without a counter (phase 2
|
||||
* reads par2 sets for those). Phase 2 picks its work from settled releases.
|
||||
* Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
|
||||
* lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
|
||||
* never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
|
||||
* hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
|
||||
* announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
|
||||
* settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
|
||||
* candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
|
||||
*/
|
||||
class Settler {
|
||||
/** Up to `batches` batches of BATCH (all there are without). */
|
||||
async run(batches = Infinity): Promise<number> {
|
||||
let settled = 0
|
||||
for (let batch = 0; batch < batches; batch++) {
|
||||
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH settled AS (
|
||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
|
||||
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
||||
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
||||
RETURNING id, readable, files, bytes, poster),
|
||||
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
|
||||
-- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
|
||||
queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
|
||||
ON CONFLICT DO NOTHING)
|
||||
SELECT count(*)::int AS n FROM settled`,
|
||||
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
|
||||
async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`UPDATE releases SET settled_at = now()
|
||||
WHERE settled_at IS NULL AND complete
|
||||
AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
|
||||
RETURNING id`,
|
||||
[quietHours, quietHours]
|
||||
)
|
||||
const n = rows[0].n
|
||||
settled += n
|
||||
if (n < BATCH) return settled
|
||||
}
|
||||
return settled
|
||||
if (rows.length > 0)
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[rows.map((r: { id: string }) => Number(r.id))]
|
||||
)
|
||||
return rows.length
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
|
||||
* segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
|
||||
* its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
|
||||
* it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
|
||||
* .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
|
||||
* waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
|
||||
* a piece that saw only part of it queued a segment that isn't its last.
|
||||
*/
|
||||
export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const result = await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates pc USING (
|
||||
SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
|
||||
FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
|
||||
FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
|
||||
) s
|
||||
WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
}
|
||||
@@ -6,7 +6,7 @@ const BATCH = 2000
|
||||
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
|
||||
const WINDOW_MINUTES = 10
|
||||
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
|
||||
const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||
export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||
|
||||
/** The words of a name worth comparing: 4+ letters or digits. */
|
||||
function words(text: string): Set<string> {
|
||||
@@ -53,29 +53,27 @@ export class SidecarFolder {
|
||||
|
||||
async round(): Promise<FoldRound> {
|
||||
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
|
||||
// A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
|
||||
// queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
|
||||
const { rows: queued } = await db.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
|
||||
FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||
`SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
|
||||
FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||
[BATCH]
|
||||
)
|
||||
const { rows: stale } = await db.rawQuery(
|
||||
`SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
|
||||
[BATCH]
|
||||
)
|
||||
const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
|
||||
const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
|
||||
for (const c of queued) {
|
||||
if (c.id === null) continue
|
||||
round.looked++
|
||||
done.push(Number(c.id))
|
||||
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
|
||||
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
|
||||
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
|
||||
round.sidecars++
|
||||
if (!c.posted_at) continue
|
||||
const { rows: siblings } = await db.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||
`SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
|
||||
AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
|
||||
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
|
||||
AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
|
||||
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
|
||||
)
|
||||
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
|
||||
if (!sibling) continue
|
||||
|
||||
@@ -9,6 +9,9 @@ import { decodeSpotBinary } from '#lib/spotnet/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/headers/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
|
||||
/** Spots claimed per round, newest first. */
|
||||
const BATCH = 20
|
||||
@@ -102,6 +105,12 @@ export class SpotInspector {
|
||||
this.client = undefined
|
||||
}
|
||||
if (!round || round.inspected === 0) {
|
||||
// Nothing to inspect for a while: closed (nntpIdleCloseMs), opened again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && IDLE_MS >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
@@ -245,8 +254,8 @@ export class SpotInspector {
|
||||
}
|
||||
if (wanted.length === 0) return 0
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
|
||||
for (const r of rows) {
|
||||
@@ -269,7 +278,7 @@ export class SpotInspector {
|
||||
await db.transaction(async (trx) => {
|
||||
// A release of the spot's own goes now that the header pass built its files' releases.
|
||||
if (spot.release_id) {
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
|
||||
if (own) {
|
||||
await trx.from('release_spots').where('release_id', own.id).delete()
|
||||
await trx.from('release_passwords').where('release_id', own.id).delete()
|
||||
@@ -288,6 +297,12 @@ export class SpotInspector {
|
||||
.ignore()
|
||||
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
|
||||
}
|
||||
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
|
||||
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
|
||||
spot.category,
|
||||
JSON.stringify(spot.subcats ?? {}),
|
||||
[...perRelease.keys()],
|
||||
])
|
||||
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
|
||||
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
|
||||
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
|
||||
@@ -323,16 +338,19 @@ export class SpotInspector {
|
||||
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
|
||||
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
|
||||
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
|
||||
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
|
||||
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
|
||||
const groupSetId = await CatalogRefs.groupSetId(db, groups)
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
release_key: `spot:${key}`,
|
||||
identity_key: key,
|
||||
release_key: releaseKey(key, groupId, null),
|
||||
identity_key: identityKey(key),
|
||||
group_id: groupId,
|
||||
group_set_id: groupSetId,
|
||||
poster_id: posterId,
|
||||
name: spot.title.slice(0, 1000),
|
||||
source: 'spot',
|
||||
poster: spot.poster,
|
||||
groups: JSON.stringify(groups),
|
||||
source: SOURCES.spot,
|
||||
posted_at: spot.posted_at,
|
||||
last_posted_at: spot.posted_at,
|
||||
files: files.length,
|
||||
@@ -341,6 +359,8 @@ export class SpotInspector {
|
||||
readable: true,
|
||||
passworded: spot.password ? true : null,
|
||||
settled_at: new Date(),
|
||||
// Kept when its title says nothing (the category trigger, migration 5200).
|
||||
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
|
||||
})
|
||||
.onConflict('release_key')
|
||||
.ignore()
|
||||
@@ -349,6 +369,8 @@ export class SpotInspector {
|
||||
const releaseId = Number(row.id)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
|
||||
// Its title, searchable (release_texts).
|
||||
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
|
||||
if (spot.password) {
|
||||
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { MissingArticle, XoverClient } from '#lib/nntp/index'
|
||||
@@ -60,6 +61,12 @@ export class Spotter {
|
||||
this.client = undefined
|
||||
}
|
||||
const wait = coordinatorConfig.spots.everySeconds * 1000
|
||||
// Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
@@ -177,7 +184,7 @@ export class Spotter {
|
||||
await trx.from('release_spots').where('spot_id', targetId).delete()
|
||||
await trx.from('release_passwords').where('spot_id', targetId).delete()
|
||||
// A release the spot made of its own goes with it.
|
||||
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete()
|
||||
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
|
||||
})
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { readdir, stat, statfs } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import type { Lane } from '#lib/edge/index'
|
||||
|
||||
export interface RangeBand {
|
||||
band: string
|
||||
planned: number
|
||||
plannedArticles: number
|
||||
leased: number
|
||||
done: number
|
||||
doneArticles: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
export interface HeldGroup {
|
||||
node: string
|
||||
group: string
|
||||
lane: Lane
|
||||
range: { start: number; end: number; next: number } | null
|
||||
heartbeatSeconds: number
|
||||
snapshotNext: number | null
|
||||
snapshotSeconds: number | null
|
||||
snapshotMb: number | null
|
||||
rangesDone: number
|
||||
headers: number
|
||||
}
|
||||
|
||||
export interface SnapshotFile {
|
||||
group: string
|
||||
lane: Lane
|
||||
mb: number
|
||||
hours: number
|
||||
}
|
||||
|
||||
export interface Status {
|
||||
at: string
|
||||
catalog: {
|
||||
tablespace: string
|
||||
gb: number
|
||||
capGb: number
|
||||
full: boolean
|
||||
freeGb: number
|
||||
}
|
||||
bands: RangeBand[]
|
||||
lastHour: {
|
||||
indexedArticles: number
|
||||
doneRanges: number
|
||||
storedArticles: number
|
||||
storedRanges: number
|
||||
}
|
||||
corpus: { pendingRanges: number; pendingArticles: number }
|
||||
spool: { ranges: number; chunks: number }
|
||||
tail: { groups: number; ranges: number; articles: number }
|
||||
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
|
||||
ancient: { groups: number; articles: number }
|
||||
held: HeldGroup[]
|
||||
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
|
||||
releases: { estimated: number; provisional: number }
|
||||
/** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
|
||||
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
|
||||
judge: { dirty: number }
|
||||
nodes: { name: string; lastSeenSeconds: number | null }[]
|
||||
}
|
||||
|
||||
const seconds = (since: Date | string | null) =>
|
||||
since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
|
||||
|
||||
/**
|
||||
* What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
|
||||
* NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
|
||||
* with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
|
||||
* filesystem; nothing is kept between calls.
|
||||
*/
|
||||
class StatusService {
|
||||
async collect(): Promise<Status> {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
const cap = coordinatorConfig.catalogMaxNvmeGb
|
||||
// The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
|
||||
// chunk partitions and the probe cache on the HDD, minutes on homelab.
|
||||
const { rows: space } = await db.rawQuery(
|
||||
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
|
||||
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
|
||||
)
|
||||
const tablespaceGb = Number(space[0].bytes) / 1e9
|
||||
const fs = await statfs('/').catch(() => null)
|
||||
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
|
||||
|
||||
// The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
|
||||
// capture too, at their history priorities), the history by its thousand.
|
||||
const { rows: bands } = await db.rawQuery(
|
||||
`SELECT CASE WHEN priority <= ? THEN 'tail'
|
||||
WHEN priority < 0 THEN 'raised'
|
||||
WHEN capture AND priority <= 2 THEN 'window ' || priority
|
||||
WHEN priority < 1000 THEN 'history 0-999'
|
||||
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
|
||||
count(*) FILTER (WHERE status = 'planned') AS planned,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
|
||||
count(*) FILTER (WHERE status = 'leased') AS leased,
|
||||
count(*) FILTER (WHERE status = 'done') AS done,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM ranges GROUP BY 1
|
||||
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
|
||||
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
|
||||
[tail, tail]
|
||||
)
|
||||
const { rows: hour } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
|
||||
count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
|
||||
count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
|
||||
count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
|
||||
count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
|
||||
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
|
||||
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
|
||||
FROM ranges`,
|
||||
[tail, tail, tail]
|
||||
)
|
||||
const h = hour[0]
|
||||
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const { rows: held } = await db.rawQuery(
|
||||
`SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
|
||||
r.start_id, r.end_id, r.next_id, l.group_id
|
||||
FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
|
||||
LEFT JOIN ranges r ON r.id = l.range_id
|
||||
WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
|
||||
ORDER BY l.lane, g.name`,
|
||||
[ttl]
|
||||
)
|
||||
const snapshotSizes = new Map<string, number>()
|
||||
const files: SnapshotFile[] = []
|
||||
const names = new Map<number, string>(
|
||||
(await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
|
||||
)
|
||||
for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
|
||||
const m = /^(\d+)(-tail)?\.gz$/.exec(file)
|
||||
if (!m) continue
|
||||
const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
|
||||
if (!s) continue
|
||||
const lane: Lane = m[2] ? 'tail' : 'history'
|
||||
snapshotSizes.set(`${m[1]}:${lane}`, s.size)
|
||||
files.push({
|
||||
group: names.get(Number(m[1])) ?? m[1],
|
||||
lane,
|
||||
mb: s.size / 1e6,
|
||||
hours: (Date.now() - s.mtimeMs) / 3600_000,
|
||||
})
|
||||
}
|
||||
files.sort((a, b) => b.mb - a.mb)
|
||||
|
||||
// The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
|
||||
// homelab); the rest are small partial indexes.
|
||||
const { rows: releases } = await db.rawQuery(
|
||||
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
|
||||
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
|
||||
(SELECT count(*) FROM judge_dirty) AS dirty`
|
||||
)
|
||||
const r = releases[0]
|
||||
const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
|
||||
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
catalog: {
|
||||
tablespace: space[0].tablespace,
|
||||
gb: tablespaceGb,
|
||||
capGb: cap,
|
||||
full: cap > 0 && tablespaceGb > cap,
|
||||
freeGb,
|
||||
},
|
||||
bands: bands.map((b: any) => ({
|
||||
band: b.band,
|
||||
planned: Number(b.planned),
|
||||
plannedArticles: Number(b.planned_articles),
|
||||
leased: Number(b.leased),
|
||||
done: Number(b.done),
|
||||
doneArticles: Number(b.done_articles),
|
||||
failed: Number(b.failed),
|
||||
})),
|
||||
lastHour: {
|
||||
indexedArticles: Number(h.indexed),
|
||||
doneRanges: Number(h.done_ranges),
|
||||
storedArticles: Number(h.stored),
|
||||
storedRanges: Number(h.stored_ranges),
|
||||
},
|
||||
corpus: {
|
||||
pendingRanges: Number(h.corpus_pending),
|
||||
pendingArticles: Number(h.corpus_pending_articles),
|
||||
},
|
||||
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
|
||||
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
|
||||
tail: {
|
||||
groups: Number(h.tail_groups),
|
||||
ranges: Number(h.tail_ranges),
|
||||
articles: Number(h.tail_articles),
|
||||
},
|
||||
held: held.map((l: any) => {
|
||||
const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
|
||||
return {
|
||||
node: l.node,
|
||||
group: l.group,
|
||||
lane: l.lane,
|
||||
range:
|
||||
l.start_id !== null
|
||||
? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
|
||||
: null,
|
||||
heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
|
||||
snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
|
||||
snapshotSeconds: seconds(l.snapshot_at),
|
||||
snapshotMb: size === undefined ? null : size / 1e6,
|
||||
rangesDone: Number(l.ranges_done),
|
||||
headers: Number(l.headers),
|
||||
}
|
||||
}),
|
||||
snapshots: {
|
||||
count: files.length,
|
||||
gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
|
||||
oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
|
||||
largest: files.slice(0, 5),
|
||||
},
|
||||
releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
|
||||
probes: {
|
||||
ready: Number(r.ready),
|
||||
waiting: Number(r.waiting),
|
||||
pendingWaiting: Number(r.pending_waiting),
|
||||
pendingNamed: Number(r.pending_named),
|
||||
},
|
||||
judge: { dirty: Number(r.dirty) },
|
||||
nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
|
||||
}
|
||||
}
|
||||
|
||||
/** The status as a text panel. */
|
||||
render(s: Status): string {
|
||||
const n = (v: number) => Math.round(v).toLocaleString('en-US')
|
||||
const b = (v: number) =>
|
||||
v >= 1e9
|
||||
? `${(v / 1e9).toFixed(2)} B`
|
||||
: v >= 1e6
|
||||
? `${(v / 1e6).toFixed(1)} M`
|
||||
: v >= 1e3
|
||||
? `${(v / 1e3).toFixed(0)} k`
|
||||
: String(v)
|
||||
const age = (sec: number | null) =>
|
||||
sec === null
|
||||
? '-'
|
||||
: sec < 90
|
||||
? `${sec} s`
|
||||
: sec < 5400
|
||||
? `${Math.round(sec / 60)} min`
|
||||
: `${(sec / 3600).toFixed(1)} h`
|
||||
const lines: string[] = []
|
||||
lines.push(`Status at ${s.at}`)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
|
||||
)
|
||||
lines.push(
|
||||
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
|
||||
)
|
||||
lines.push(
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
|
||||
)
|
||||
lines.push('')
|
||||
const table = (header: string[], rows: string[][]) => {
|
||||
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
|
||||
const line = (r: string[]) =>
|
||||
r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(' ')
|
||||
lines.push(line(header))
|
||||
for (const r of rows) lines.push(line(r))
|
||||
}
|
||||
lines.push('Ranges by band:')
|
||||
table(
|
||||
['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
|
||||
s.bands.map((x) => [
|
||||
x.band,
|
||||
n(x.planned),
|
||||
b(x.plannedArticles),
|
||||
n(x.leased),
|
||||
n(x.done),
|
||||
b(x.doneArticles),
|
||||
n(x.failed),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(`Held groups (${s.held.length}):`)
|
||||
if (s.held.length > 0)
|
||||
table(
|
||||
[
|
||||
'group',
|
||||
'lane',
|
||||
'node',
|
||||
'range',
|
||||
'at',
|
||||
'heartbeat',
|
||||
'snapshot',
|
||||
'size',
|
||||
'done',
|
||||
'headers',
|
||||
],
|
||||
s.held.map((x) => [
|
||||
x.group,
|
||||
x.lane,
|
||||
x.node,
|
||||
x.range ? `${x.range.start}-${x.range.end}` : '-',
|
||||
x.range ? String(x.range.next) : '-',
|
||||
age(x.heartbeatSeconds),
|
||||
x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
|
||||
x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
|
||||
n(x.rangesDone),
|
||||
b(x.headers),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
|
||||
)
|
||||
lines.push(
|
||||
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
|
||||
)
|
||||
lines.push(
|
||||
`Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
|
||||
)
|
||||
lines.push(
|
||||
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
export default new StatusService()
|
||||
+72
-57
@@ -3,45 +3,46 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { Prefetcher } from '#services/prefetcher'
|
||||
import Planner from '#services/planner'
|
||||
import FragmentMerger from '#services/fragment_merger'
|
||||
import Leases from '#services/leases'
|
||||
import Settler from '#services/settler'
|
||||
import RangeBlocks from '#services/range_blocks'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
const TICK_MS = 2000
|
||||
const PLAN_EVERY_MS = 60 * 60 * 1000
|
||||
const MERGE_EVERY_MS = 60 * 1000
|
||||
const MERGE_BUDGET_MS = 30 * 1000
|
||||
const STOP_WAIT_MS = 15 * 1000
|
||||
/**
|
||||
* Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
|
||||
* rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
|
||||
* (LOG.md 2026-10-16 (67)).
|
||||
*/
|
||||
const SETTLE_EVERY_MS = 5 * 60 * 1000
|
||||
const SETTLE_BATCHES = 4
|
||||
/** range_blocks (release_final) after ranges finish. */
|
||||
const BLOCKS_EVERY_MS = 5 * 60 * 1000
|
||||
/** Leases whose worker went quiet after reading everything are finished every few minutes. */
|
||||
const ABANDONED_EVERY_MS = 5 * 60 * 1000
|
||||
|
||||
/**
|
||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
|
||||
* merging fragments of finished ranges every minute, and every hour
|
||||
* planning ranges for new articles and settling quiet releases. The API (leases, stream,
|
||||
* batches) runs in the web server.
|
||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher, the tail's ranges every few minutes,
|
||||
* new groups' history every hour, and abandoned leases. The API (leases, stream, batches) runs in the web server.
|
||||
* Voyager 3: the settler and the fragment merger are gone (the window finishes a release once; the block merger
|
||||
* joins edge releases).
|
||||
*/
|
||||
export default class CoordinatorRun extends BaseCommand {
|
||||
static commandName = 'coordinator:run'
|
||||
static description = 'Prefetches ranges into the spool and plans new ones'
|
||||
static options: CommandOptions = { startApp: true, staysAlive: true }
|
||||
|
||||
@flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
|
||||
@flags.string({
|
||||
description:
|
||||
'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles',
|
||||
})
|
||||
declare shard?: string
|
||||
|
||||
async run() {
|
||||
const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
|
||||
if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
|
||||
if (!(count >= 1 && index >= 1 && index <= count))
|
||||
throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
|
||||
const prefetcher = new Prefetcher({ index: index - 1, count })
|
||||
const share = new NntpShare(
|
||||
count > 1 ? `coordinator ${index}/${count}` : 'coordinator',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority,
|
||||
coordinatorConfig.nntpMinConnections,
|
||||
coordinatorConfig.nntpConnections
|
||||
)
|
||||
const main = index === 1
|
||||
let stopping = false
|
||||
let stopped: () => void
|
||||
@@ -53,9 +54,33 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
prefetcher.stop()
|
||||
await done
|
||||
})
|
||||
this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
|
||||
// Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
|
||||
// of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
|
||||
this.logger.info(
|
||||
count > 1
|
||||
? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).`
|
||||
: 'Coordinator started.'
|
||||
)
|
||||
// The pool's size: its share of the account's connections, the prober the rest (app/services/nntp_shares.ts).
|
||||
// Before the first connection: until then it's NNTP_CONNECTIONS.
|
||||
const first = await NntpPool.share(share)
|
||||
this.logger.info(`NNTP share: ${first.granted} connections.`)
|
||||
const sharing = (async () => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
const { was, granted } = await NntpPool.share(share)
|
||||
if (granted !== was) this.logger.info(`NNTP share: ${granted} connections (was ${was}).`)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`NNTP share failed: ${error.message}`)
|
||||
}
|
||||
for (
|
||||
let waited = 0;
|
||||
waited < coordinatorConfig.nntpShares.everyMs && !stopping;
|
||||
waited += TICK_MS
|
||||
)
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// The prefetcher's ticks in a loop of their own: planning doesn't hold them (a merge round of up to 30 s once
|
||||
// left the connections idle once their targets were fetched).
|
||||
const prefetching = (async () => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
@@ -66,22 +91,31 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
|
||||
const blocks = (async () => {
|
||||
const abandoned = (async () => {
|
||||
while (main && !stopping) {
|
||||
try {
|
||||
await RangeBlocks.rebuild()
|
||||
const finished = await Leases.finishAbandoned()
|
||||
if (finished > 0)
|
||||
this.logger.info(
|
||||
`Finished ${finished} ranges read to their end whose worker went quiet.`
|
||||
)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Range blocks failed: ${error.message}`)
|
||||
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
|
||||
try {
|
||||
const settled = await Settler.settleComplete()
|
||||
if (settled > 0)
|
||||
this.logger.info(`Settled ${settled} complete provisional releases quiet for hours.`)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Settling complete releases failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// History for new groups hourly; the tail every few minutes.
|
||||
let plannedAt = 0
|
||||
let tailedAt = 0
|
||||
let mergedAt = 0
|
||||
let settledAt = 0
|
||||
while (!stopping) {
|
||||
try {
|
||||
if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
|
||||
@@ -89,46 +123,27 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
const { groups, ranges } = await Planner.planAll()
|
||||
if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
|
||||
}
|
||||
if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
|
||||
settledAt = Date.now()
|
||||
const settled = await Settler.run(SETTLE_BATCHES)
|
||||
if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
|
||||
}
|
||||
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
|
||||
tailedAt = Date.now()
|
||||
const { groups, ranges } = await Planner.planTail()
|
||||
if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
|
||||
}
|
||||
if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
|
||||
mergedAt = Date.now()
|
||||
// Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
|
||||
// (Going on while only waiting files were left kept Postgres busy all the time.)
|
||||
const totals = { completed: 0, incomplete: 0, waiting: 0 }
|
||||
let decidedThisPass = 0
|
||||
while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
|
||||
const merged = await FragmentMerger.run()
|
||||
totals.completed += merged.completed
|
||||
totals.incomplete += merged.incomplete
|
||||
totals.waiting = merged.waiting
|
||||
decidedThisPass += merged.completed + merged.incomplete
|
||||
if (merged.wrapped) {
|
||||
if (decidedThisPass === 0) break
|
||||
decidedThisPass = 0
|
||||
}
|
||||
}
|
||||
if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
|
||||
const { groups, ranges, extended } = await Planner.planTail()
|
||||
if (ranges + extended > 0)
|
||||
this.logger.info(
|
||||
`Tail: ${ranges} ranges added, ${extended} grown, for ${groups} groups.`
|
||||
)
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Coordinator round failed: ${error.message}`)
|
||||
}
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
await Promise.all([prefetching, blocks])
|
||||
await Promise.all([prefetching, abandoned, sharing])
|
||||
// The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
|
||||
const waitUntil = Date.now() + STOP_WAIT_MS
|
||||
while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
|
||||
if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
|
||||
if (prefetcher.active > 0)
|
||||
this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
|
||||
await NntpPool.close()
|
||||
await share.leave().catch(() => {})
|
||||
this.logger.info('Coordinator stopped.')
|
||||
stopped!()
|
||||
await this.terminate()
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import StatusService from '#services/status'
|
||||
|
||||
/**
|
||||
* What the indexer is doing (app/services/status.ts): the catalog's room, the ranges by band, the last hour's
|
||||
* rates, the held groups and their snapshots, the phase-2 backlogs. Reads only.
|
||||
*/
|
||||
export default class CoordinatorStatus extends BaseCommand {
|
||||
static commandName = 'coordinator:status'
|
||||
static description =
|
||||
'Prints what the indexer is doing: catalog room, ranges by band, rates, held groups, backlogs'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'JSON instead of the text panel' })
|
||||
declare json: boolean
|
||||
|
||||
async run() {
|
||||
const status = await StatusService.collect()
|
||||
this.logger.log(this.json ? JSON.stringify(status, null, 2) : StatusService.render(status))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { access, readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import Corpus, { FILTERED } from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/**
|
||||
* Filters corpus chunks stored as the news server sent them (before CORPUS_FILTER, 2026-10-23): each chunk without a
|
||||
* `.drops` of the planned capture ranges (--all: of every range in the corpus) written again as Corpus.write would.
|
||||
*/
|
||||
export default class CorpusFilterCommand extends BaseCommand {
|
||||
static commandName = 'corpus:filter'
|
||||
static description = 'Filters stored corpus chunks (planned capture ranges; --all: every range) as CORPUS_FILTER does'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'Every range in the corpus, also done ones (the grouping sweeps read them raw)' })
|
||||
declare all: boolean
|
||||
|
||||
async run() {
|
||||
const dir = coordinatorConfig.corpus.dir
|
||||
if (!dir) return this.logger.error('CORPUS_DIR is not set')
|
||||
const rules = await RuleSets.current()
|
||||
const filter = { ruleVersion: rules.version, priorities: rules.priorities }
|
||||
const query = db.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').where('r.capture', true).select('g.name as group', 'r.start_id', 'r.end_id')
|
||||
const ranges = await (this.all ? query : query.where('r.status', 'planned'))
|
||||
let chunks = 0
|
||||
let before = 0
|
||||
let after = 0
|
||||
for (const r of ranges) {
|
||||
const range = { group: r.group, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
const rangeDir = join(dir, range.group, `${range.start}-${range.end}`)
|
||||
const files = await readdir(rangeDir).catch(() => [] as string[])
|
||||
const gz = files.filter((f) => f.endsWith('.gz'))
|
||||
for (const name of gz) {
|
||||
if (files.includes(`${name}.drops`)) continue
|
||||
const m = /^\d{4}-(\d+)-(\d+)\.gz$/.exec(name)
|
||||
if (!m) continue
|
||||
const data = await readFile(join(rangeDir, name))
|
||||
await Corpus.write(range, { from: Number(m[1]), to: Number(m[2]) }, data, Number.POSITIVE_INFINITY, filter)
|
||||
before += data.length
|
||||
after += (await readFile(join(rangeDir, name))).length
|
||||
chunks++
|
||||
}
|
||||
// A complete range's range.json says it's filtered now (Corpus.write only writes it at the last chunk).
|
||||
const json = join(rangeDir, 'range.json')
|
||||
if (gz.length > 0 && (await access(json).then(() => true, () => false))) await writeFile(json, JSON.stringify({ ...range, filtered: FILTERED }))
|
||||
if (chunks > 0 && chunks % 1000 < gz.length) this.logger.info(`${chunks} chunks filtered: ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
|
||||
}
|
||||
this.logger.info(`Done: ${chunks} chunks of ${ranges.length} ranges filtered, ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
|
||||
}
|
||||
}
|
||||
@@ -108,7 +108,7 @@ export default class EdgeBench extends BaseCommand {
|
||||
let text = 0
|
||||
let peakRss = 0
|
||||
let peakDetail = ''
|
||||
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
|
||||
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, samples: 0 }
|
||||
const counts = new Map<string, number>()
|
||||
const keys = new Set<string>()
|
||||
let batches = 0
|
||||
@@ -164,7 +164,6 @@ export default class EdgeBench extends BaseCommand {
|
||||
out.payload += r.payload.length
|
||||
}
|
||||
out.candidates += batch.candidates.length
|
||||
out.fragments += batch.fragments.length
|
||||
out.samples += batch.samples.length
|
||||
for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
|
||||
const m = process.memoryUsage()
|
||||
@@ -182,7 +181,7 @@ export default class EdgeBench extends BaseCommand {
|
||||
this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
|
||||
this.logger.info(` inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
|
||||
this.logger.info(` ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
|
||||
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
|
||||
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.samples} samples`)
|
||||
for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(` dropped ${k}: ${v.toLocaleString('en')} articles`)
|
||||
if (this.store) this.logger.info(` stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import env from '#start/env'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/**
|
||||
* What the news server does with XOVER spans around an article that made a request hang (ranges:dates: one point
|
||||
* per group timed out after 2 minutes, again on a fresh connection; LOG.md 2026-10-25 (105)). Each probe on a
|
||||
* connection of its own with a short timeout: the span asked for, the reply, how many lines, how long. First the
|
||||
* span itself and wider ones around it, then halves of it down to single articles, then a span past `high`.
|
||||
*/
|
||||
export default class NntpProbe extends BaseCommand {
|
||||
static commandName = 'nntp:probe'
|
||||
static description =
|
||||
'Probes XOVER spans around an article that hangs, bisecting to the articles at fault'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@args.string({ description: 'Group' })
|
||||
declare group: string
|
||||
|
||||
@args.string({ description: 'First article of the span that hung' })
|
||||
declare article: string
|
||||
|
||||
@flags.number({ description: 'Articles in the span (default 32)' })
|
||||
declare size?: number
|
||||
|
||||
@flags.number({ description: 'Seconds without data before a probe is given up (default 20)' })
|
||||
declare timeout?: number
|
||||
|
||||
@flags.boolean({ description: 'Only the span itself and the wider ones, no bisection' })
|
||||
declare noBisect: boolean
|
||||
|
||||
@flags.boolean({ description: 'Without XFEATURE COMPRESS GZIP' })
|
||||
declare plain: boolean
|
||||
|
||||
@flags.string({
|
||||
description: 'Only spans from the article of these sizes, e.g. "8,16,24,32,48,64"',
|
||||
})
|
||||
declare sizes?: string
|
||||
|
||||
private timeoutMs = 20_000
|
||||
private info = { count: 0, low: 0, high: 0 }
|
||||
|
||||
async run() {
|
||||
this.timeoutMs = (this.timeout ?? 20) * 1000
|
||||
const a = Number(this.article)
|
||||
const size = this.size ?? 32
|
||||
const client = await this.connect()
|
||||
this.info = await client.group(this.group)
|
||||
client.close()
|
||||
this.logger.info(
|
||||
`${this.group}: ${this.info.count} articles, ${this.info.low}–${this.info.high}${this.plain ? '; plain' : '; compressed'}.`
|
||||
)
|
||||
const span = (from: number, to: number) =>
|
||||
[Math.max(this.info.low, from), Math.min(this.info.high, to)] as const
|
||||
if (this.sizes) {
|
||||
for (const n of this.sizes.split(',').map(Number)) await this.probe(...span(a, a + n - 1))
|
||||
return
|
||||
}
|
||||
this.logger.info('The span and wider ones:')
|
||||
await this.probe(...span(a, a + size - 1))
|
||||
await this.probe(...span(a, a + size * 32 - 1))
|
||||
await this.probe(...span(a - size * 32, a + size - 1))
|
||||
await this.probe(...span(a - 50_000, a + 49_999))
|
||||
this.logger.info('Past the end, and an empty span before it:')
|
||||
await this.probe(this.info.high + 100, this.info.high + 131)
|
||||
await this.probe(this.info.high + 1, this.info.high + 1)
|
||||
if (this.noBisect) return
|
||||
this.logger.info('Halves of the span:')
|
||||
await this.bisect(a, Math.min(this.info.high, a + size - 1))
|
||||
}
|
||||
|
||||
/** Halves of a span that hangs, down to single articles; a half that answers isn't split further. */
|
||||
private async bisect(from: number, to: number) {
|
||||
const hung = await this.probe(from, to)
|
||||
if (!hung || from === to) return
|
||||
const mid = Math.floor((from + to) / 2)
|
||||
await this.bisect(from, mid)
|
||||
await this.bisect(mid + 1, to)
|
||||
}
|
||||
|
||||
/** One XOVER on a fresh connection; true when it timed out. */
|
||||
private async probe(from: number, to: number): Promise<boolean> {
|
||||
const startedAt = Date.now()
|
||||
let client: XoverClient | null = null
|
||||
try {
|
||||
client = await this.connect()
|
||||
await client.group(this.group)
|
||||
const rows = await client.xover(from, to)
|
||||
const t = client.lastTiming
|
||||
this.logger.info(
|
||||
` XOVER ${from}-${to}: ${rows.length} lines in ${Date.now() - startedAt} ms (reply ${t.replyMs.toFixed(0)}, body ${t.bodyMs.toFixed(0)})${rows.length > 0 ? `; first ${rows[0][0]}, last ${rows[rows.length - 1][0]}` : ''}`
|
||||
)
|
||||
return false
|
||||
} catch (error: any) {
|
||||
const timedOut = /timed out/i.test(error.message)
|
||||
this.logger.warning(
|
||||
` XOVER ${from}-${to}: ${error.message} after ${Date.now() - startedAt} ms${client ? `, ${client.buffered} bytes buffered: ${JSON.stringify(client.bufferedHead().toString('latin1'))} (${client.bufferedHead().toString('hex')})` : ''}`
|
||||
)
|
||||
return timedOut
|
||||
} finally {
|
||||
client?.close()
|
||||
}
|
||||
}
|
||||
|
||||
private connect() {
|
||||
return XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: !this.plain,
|
||||
timeoutMs: this.timeoutMs,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
|
||||
/** The par2 prune (Par2Pruner) in one go: a whole pass from the first set, instead of the merger's 2 minutes an hour. */
|
||||
export default class Par2Prune extends BaseCommand {
|
||||
static commandName = 'par2:prune'
|
||||
static description = 'Drops the par2 file lists the merge can no longer use: one whole pass over every set, then exits'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
async run() {
|
||||
await db.from('worker_cursors').where('name', 'par2-prune').update({ key: '' })
|
||||
const round = await new Par2Pruner().round()
|
||||
this.logger.info(`Pruned ${round.rows} par2 file rows: ${round.complete} complete sets, ${round.judged} judged, ${round.final} final.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
|
||||
const BYTES_PER_ARTICLE = 18.5
|
||||
|
||||
/**
|
||||
* Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
|
||||
* corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
|
||||
* ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
|
||||
* fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
|
||||
* missing) and reported.
|
||||
*/
|
||||
export default class RangesCapture extends BaseCommand {
|
||||
static commandName = 'ranges:capture'
|
||||
static description =
|
||||
'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'First day of the window (ISO date)', required: true })
|
||||
declare from: string
|
||||
|
||||
@flags.string({ description: 'Day after the window (ISO date)', required: true })
|
||||
declare to: string
|
||||
|
||||
@flags.number({
|
||||
description: 'Priority for the planned ranges marked (default 0: before the history)',
|
||||
})
|
||||
declare priority?: number
|
||||
|
||||
@flags.boolean({ description: 'Only report what the window holds' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const from = new Date(this.from)
|
||||
const to = new Date(this.to)
|
||||
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
|
||||
return this.logger.error('--from and --to: ISO dates, from before to')
|
||||
const priority = this.priority ?? 0
|
||||
// A range covers est_date up to the next range's; the last range of a group up to now.
|
||||
const hits = `
|
||||
WITH spans AS (
|
||||
SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
|
||||
lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
|
||||
FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
|
||||
), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
|
||||
const { rows } = await db.rawQuery(
|
||||
`${hits}
|
||||
SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
|
||||
WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
|
||||
count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
|
||||
FROM hit GROUP BY 1 ORDER BY 1`,
|
||||
[to, from]
|
||||
)
|
||||
const fetch = rows
|
||||
.filter((r: any) => r.kind === 'planned' || r.kind === 'done')
|
||||
.reduce((sum: number, r: any) => sum + Number(r.articles), 0)
|
||||
this.logger.info(`${this.from} to ${this.to}:`)
|
||||
for (const r of rows)
|
||||
this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
|
||||
this.logger.info(
|
||||
`To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
|
||||
)
|
||||
if (this.dryRun) return
|
||||
const { rows: marked } = await db.rawQuery(
|
||||
`${hits}
|
||||
UPDATE ranges r SET capture = true,
|
||||
priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
|
||||
FROM hit WHERE r.id = hit.id AND NOT hit.capture
|
||||
AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
|
||||
RETURNING r.status`,
|
||||
[to, from, priority]
|
||||
)
|
||||
const planned = marked.filter((r: any) => r.status === 'planned').length
|
||||
this.logger.info(
|
||||
`Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
|
||||
|
||||
/**
|
||||
* When each active group's articles were posted (group_dates, sampled from the news server) and from that every
|
||||
* range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
|
||||
* posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
|
||||
* articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
|
||||
*/
|
||||
export default class RangesDates extends BaseCommand {
|
||||
static commandName = 'ranges:dates'
|
||||
static description =
|
||||
"Samples when each active group's articles were posted and estimates every range's date"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
|
||||
declare group?: string
|
||||
|
||||
@flags.number({ description: 'Articles between two samples of a group (default 500M)' })
|
||||
declare every?: number
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Connections, a share of the account beside the coordinator and the prober (default 6)',
|
||||
})
|
||||
declare connections?: number
|
||||
|
||||
@flags.boolean({ description: 'Only groups without samples yet' })
|
||||
declare missing: boolean
|
||||
|
||||
@flags.boolean({
|
||||
description: 'No sampling: estimate the ranges from the samples stored already',
|
||||
})
|
||||
declare estimateOnly: boolean
|
||||
|
||||
async run() {
|
||||
if (!this.estimateOnly) await this.sample()
|
||||
await this.estimate()
|
||||
await this.report()
|
||||
}
|
||||
|
||||
private groups(missing = this.missing) {
|
||||
const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
|
||||
if (this.group) q.where('name', 'like', this.group)
|
||||
if (missing)
|
||||
q.whereNotExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
return q
|
||||
}
|
||||
|
||||
private async sample() {
|
||||
const every = this.every ?? 500_000_000
|
||||
const connections = this.connections ?? 6
|
||||
const groups = await this.groups()
|
||||
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
|
||||
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
|
||||
// room for them), as many as asked for.
|
||||
const workers = Math.max(1, Math.min(connections, groups.length))
|
||||
const share = new NntpShare(
|
||||
'dates',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority + 1,
|
||||
workers,
|
||||
workers
|
||||
)
|
||||
let open = 0
|
||||
const sync = () =>
|
||||
share
|
||||
.sync(workers, open)
|
||||
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
|
||||
await sync()
|
||||
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
|
||||
this.logger.info(
|
||||
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
|
||||
)
|
||||
const startedAt = Date.now()
|
||||
let done = 0
|
||||
let commands = 0
|
||||
let samples = 0
|
||||
const queue = [...groups]
|
||||
const worker = async () => {
|
||||
let client: XoverClient | null = null
|
||||
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
|
||||
if (!client) {
|
||||
client = await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
open++
|
||||
}
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
client.close()
|
||||
client = null
|
||||
open--
|
||||
throw error
|
||||
}
|
||||
}
|
||||
for (let g = queue.shift(); g; g = queue.shift()) {
|
||||
try {
|
||||
const result = await sampleGroup(use, g.name, { every, size: 32 })
|
||||
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('group_dates').where('group_id', g.id).delete()
|
||||
if (result.samples.length > 0)
|
||||
await trx.table('group_dates').multiInsert(
|
||||
result.samples.map((s) => ({
|
||||
group_id: g.id,
|
||||
article: s.article,
|
||||
posted_at: s.postedAt,
|
||||
}))
|
||||
)
|
||||
})
|
||||
commands += result.commands
|
||||
samples += result.samples.length
|
||||
if (result.samples.length < 2)
|
||||
this.logger.warning(
|
||||
`${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
|
||||
)
|
||||
} catch (error: any) {
|
||||
this.logger.warning(`${g.name}: ${error.message}`)
|
||||
}
|
||||
done++
|
||||
if (done % 50 === 0)
|
||||
this.logger.info(
|
||||
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
if (client) {
|
||||
;(client as XoverClient).close()
|
||||
open--
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(Array.from({ length: workers }, worker))
|
||||
} finally {
|
||||
clearInterval(sharing)
|
||||
await share.sync(0, 0).catch(() => {})
|
||||
}
|
||||
this.logger.info(
|
||||
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
|
||||
/** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
|
||||
private async estimate() {
|
||||
const groups = await this.groups(false).whereExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
let updated = 0
|
||||
for (const g of groups) {
|
||||
const rows = await db
|
||||
.from('group_dates')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('article')
|
||||
.select('article', 'posted_at')
|
||||
const samples: DateSample[] = rows.map((r) => ({
|
||||
article: Number(r.article),
|
||||
postedAt: new Date(r.posted_at),
|
||||
}))
|
||||
const ranges = await db
|
||||
.from('ranges')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('start_id')
|
||||
.select('id', 'start_id')
|
||||
for (let i = 0; i < ranges.length; i += 1000) {
|
||||
const batch = ranges.slice(i, i + 1000)
|
||||
const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
|
||||
const bindings = batch.flatMap((r) => [
|
||||
Number(r.id),
|
||||
estimateDate(samples, Number(r.start_id))!.toISOString(),
|
||||
])
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
|
||||
bindings
|
||||
)
|
||||
updated += batch.length
|
||||
}
|
||||
}
|
||||
this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
|
||||
}
|
||||
|
||||
/** The planned history by posting year: what a window would take. */
|
||||
private async report() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
|
||||
round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
|
||||
FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
|
||||
const undatedRows = await db.rawQuery(
|
||||
`SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
|
||||
)
|
||||
const undated = undatedRows.rows[0].undated
|
||||
this.logger.info(
|
||||
`By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
|
||||
)
|
||||
for (const r of rows)
|
||||
this.logger.info(
|
||||
` ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import Planner from '#services/planner'
|
||||
|
||||
/**
|
||||
* Merges adjacent planned tail ranges (Planner.mergeTail): the tail planner added a sliver a group every round
|
||||
* while nothing leased the tail; now it grows the group's open range instead, and this packs what is there.
|
||||
*/
|
||||
export default class RangesMergeTail extends BaseCommand {
|
||||
static commandName = 'ranges:merge-tail'
|
||||
static description = 'Merges adjacent planned tail ranges into ranges of RANGE_ARTICLES'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'Only count what would be merged' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const { candidates, merged, dropped } = await Planner.mergeTail(this.dryRun)
|
||||
this.logger.info(
|
||||
`${candidates} planned tail ranges: ${dropped} ${this.dryRun ? 'would merge' : 'merged'} into ${merged} grown ranges (${candidates - dropped} left).`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,13 @@ export default class RangesPlan extends BaseCommand {
|
||||
|
||||
async run() {
|
||||
try {
|
||||
const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
|
||||
this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
|
||||
if (this.tail) {
|
||||
const { groups, ranges, extended } = await Planner.planTail()
|
||||
this.logger.info(`Tail: ${ranges} ranges added, ${extended} grown, over ${groups} groups.`)
|
||||
} else {
|
||||
const { groups, ranges } = await Planner.planAll()
|
||||
this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
|
||||
}
|
||||
} finally {
|
||||
await NntpPool.close()
|
||||
}
|
||||
|
||||
@@ -5,10 +5,10 @@ import { ReleaseJudge, RULES } from '#services/release_judge'
|
||||
/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
|
||||
export default class ReleasesJudge extends BaseCommand {
|
||||
static commandName = 'releases:judge'
|
||||
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable), hourly'
|
||||
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable): the changed ones every few minutes, all daily'
|
||||
static options: CommandOptions = { startApp: true, staysAlive: true }
|
||||
|
||||
@flags.boolean({ description: 'One round, then exit' })
|
||||
@flags.boolean({ description: 'One full round, then exit' })
|
||||
declare once: boolean
|
||||
|
||||
@flags.boolean({ description: 'With --once: count only, write no verdicts' })
|
||||
@@ -17,7 +17,7 @@ export default class ReleasesJudge extends BaseCommand {
|
||||
async run() {
|
||||
const judge = new ReleaseJudge()
|
||||
if (this.once) {
|
||||
const round = await judge.round(this.dry)
|
||||
const round = await judge.full(this.dry)
|
||||
for (const rule of RULES) {
|
||||
const r = round.rules[rule.name]
|
||||
if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Merger } from '#services/par2_merger'
|
||||
|
||||
/**
|
||||
* The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
|
||||
* first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
|
||||
*/
|
||||
export default class ReleasesMergeNames extends BaseCommand {
|
||||
static commandName = 'releases:merge-names'
|
||||
static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Sets to look at (newest first); all without' })
|
||||
declare sets: number
|
||||
|
||||
@flags.boolean({ description: 'Count and show, merge nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const merger = new Par2Merger()
|
||||
// Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
|
||||
// so microseconds survive the round trip.
|
||||
let before = ['9999-12-31', '']
|
||||
let looked = 0
|
||||
let merges = 0
|
||||
let releases = 0
|
||||
const examples: string[] = []
|
||||
while (this.sets === undefined || looked < this.sets) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT set_id, at::text AS at FROM (
|
||||
SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id) x
|
||||
WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
before
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
for (const { set_id: setId } of rows) {
|
||||
looked++
|
||||
if (this.dryRun) {
|
||||
const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
|
||||
if (found) {
|
||||
merges++
|
||||
releases += found.size
|
||||
if (examples.length < 15) {
|
||||
const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
|
||||
examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const merged = await merger.mergeByName(setId.trim())
|
||||
if (merged > 0) {
|
||||
merges++
|
||||
releases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
before = [rows.at(-1).at, rows.at(-1).set_id]
|
||||
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
for (const e of examples) this.logger.info(` ${e.slice(0, 300)}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases per batch. */
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Decides completeness by par2 set (app/services/par2_completeness.ts) for the releases that had their par2 read
|
||||
* before it was built (2026-10-23): every holder of a set whose file list is still there, by release id. Once.
|
||||
*/
|
||||
export default class ReleasesPar2Complete extends BaseCommand {
|
||||
static commandName = 'releases:par2-complete'
|
||||
static description = 'Decides completeness by their par2 sets for settled releases without a counter, then exits'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
async run() {
|
||||
let after = 0
|
||||
let looked = 0
|
||||
let decided = 0
|
||||
for (;;) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT s.release_id FROM par2_sets s
|
||||
WHERE s.release_id > ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
ORDER BY s.release_id LIMIT ?`,
|
||||
[after, BATCH]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
const ids = rows.map((r: { release_id: string }) => Number(r.release_id))
|
||||
decided += await decidePar2Completeness(db, ids)
|
||||
looked += ids.length
|
||||
after = ids.at(-1)!
|
||||
if (looked % 50_000 < BATCH) this.logger.info(`${looked} releases with a par2 set looked at, ${decided} decided (up to id ${after}).`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} releases with a par2 set looked at, ${decided} decided.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
|
||||
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||
const CURSOR = 'recategorize'
|
||||
|
||||
/**
|
||||
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
|
||||
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
|
||||
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
|
||||
* the dead versions reusable) and goes on from its cursor when run again.
|
||||
*/
|
||||
export default class ReleasesRecategorize extends BaseCommand {
|
||||
static commandName = 'releases:recategorize'
|
||||
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Release ids per batch' })
|
||||
declare batch: number
|
||||
|
||||
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
|
||||
declare toId: number
|
||||
|
||||
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
|
||||
declare floorGb: number
|
||||
|
||||
@flags.boolean({ description: 'Count what would change, write nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||
declare restart: boolean
|
||||
|
||||
async run() {
|
||||
const batch = this.batch ?? 20_000
|
||||
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
|
||||
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||
const changes = new Map<string, number>()
|
||||
let changed = 0
|
||||
let looked = 0
|
||||
const started = Date.now()
|
||||
while (after < hi) {
|
||||
while (!this.dryRun && (await space.short())) await sleep(60_000)
|
||||
const to = Math.min(after + batch, hi)
|
||||
const fresh = `release_category_for(name, inner_name, groups, NULL)`
|
||||
if (this.dryRun) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
|
||||
[after, to]
|
||||
)
|
||||
for (const r of rows) {
|
||||
looked += r.n
|
||||
if (r.was === r.now) continue
|
||||
changed += r.n
|
||||
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
|
||||
}
|
||||
} else {
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
|
||||
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
|
||||
[after, to]
|
||||
)
|
||||
changed += result.rowCount ?? 0
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, String(to)]
|
||||
)
|
||||
}
|
||||
after = to
|
||||
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
|
||||
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
|
||||
}
|
||||
}
|
||||
if (this.dryRun) {
|
||||
this.logger.info(`Would change ${changed} of ${looked}:`)
|
||||
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
|
||||
return
|
||||
}
|
||||
// The spots' categories for the linked releases whose names say nothing.
|
||||
const spots = await db.rawQuery(
|
||||
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
|
||||
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
|
||||
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
|
||||
)
|
||||
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
|
||||
/**
|
||||
* What lib/release's identity makes of releases already in the catalog: a
|
||||
@@ -27,7 +28,7 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
async run() {
|
||||
for (const name of this.group ?? []) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
|
||||
[name]
|
||||
)
|
||||
const ids = rows.map((r: { release_id: string }) => r.release_id)
|
||||
@@ -54,7 +55,7 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
}
|
||||
if (this.show?.length) return
|
||||
for (const source of ['subject', 'post', 'stem']) {
|
||||
const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
|
||||
const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
|
||||
let split = 0
|
||||
let pieces = 0
|
||||
const examples: string[] = []
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { access, readdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import Leases from '#services/leases'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { fromWire, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
/**
|
||||
* Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
|
||||
* Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)). With
|
||||
* REPLAY_INTO_CATALOG=1 any database but the test one: voyager 3 deployed, history replayed from the corpus into
|
||||
* the catalog itself (deploy/replay-window.sh, LOG.md (116)).
|
||||
*/
|
||||
const SCRATCH_DB = /_(replay|scratch)$/
|
||||
const TEST_DB = /_test$/
|
||||
const NODE_NAME = 'replay'
|
||||
|
||||
interface StoredRange {
|
||||
id: number
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
|
||||
* window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
|
||||
* scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
|
||||
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
|
||||
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
|
||||
*
|
||||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
|
||||
*
|
||||
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
|
||||
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
|
||||
*/
|
||||
export default class ReplayCorpus extends BaseCommand {
|
||||
static commandName = 'replay:corpus'
|
||||
static description =
|
||||
"Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'The group (its directory under the corpus)', required: true })
|
||||
declare group: string
|
||||
|
||||
@flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
|
||||
declare dir?: string
|
||||
|
||||
@flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
|
||||
declare from?: number
|
||||
|
||||
@flags.number({
|
||||
description: "Skip ranges starting after this article (one window of the group's history)",
|
||||
})
|
||||
declare to?: number
|
||||
|
||||
@flags.number({
|
||||
description: 'Chunks of 100k articles per batch, as the worker posts them',
|
||||
default: 5,
|
||||
})
|
||||
declare batchChunks: number
|
||||
|
||||
@flags.string({
|
||||
description:
|
||||
'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
|
||||
})
|
||||
declare bench?: string
|
||||
|
||||
@flags.boolean({
|
||||
description:
|
||||
'Leave the window open at the end (the last ranges provisional), to continue later',
|
||||
})
|
||||
declare noFinish: boolean
|
||||
|
||||
async run() {
|
||||
const database = db.connection().getReadClient().client.config.connection.database as string
|
||||
const intoCatalog = process.env.REPLAY_INTO_CATALOG === '1' && !TEST_DB.test(database)
|
||||
if (!SCRATCH_DB.test(database) && !intoCatalog) {
|
||||
this.logger.error(
|
||||
`The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}"; REPLAY_INTO_CATALOG=1 allows it)`
|
||||
)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const dir = this.dir ?? coordinatorConfig.corpus.dir
|
||||
if (!dir) {
|
||||
this.logger.error('No corpus directory: --dir or CORPUS_DIR')
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const bench = await this.benchBinary()
|
||||
if (!bench) {
|
||||
this.logger.error(
|
||||
'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
|
||||
)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const groupDir = join(dir, this.group)
|
||||
const stored = await this.storedRanges(groupDir)
|
||||
if (stored.length === 0) {
|
||||
this.logger.error(`${groupDir}: no complete range`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const nodeId = await this.nodeId()
|
||||
const groupId = await this.groupId()
|
||||
const ranges = await this.leaseRanges(groupId, nodeId, stored)
|
||||
const rules = await RuleSets.current()
|
||||
const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
|
||||
await writeFile(rulesPath, JSON.stringify(rules))
|
||||
this.logger.info(
|
||||
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
|
||||
)
|
||||
|
||||
// edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
|
||||
const args = [
|
||||
'--group',
|
||||
groupDir,
|
||||
'--rules',
|
||||
rulesPath,
|
||||
'--batch-chunks',
|
||||
String(this.batchChunks),
|
||||
'--wire',
|
||||
'-',
|
||||
]
|
||||
if (this.from) args.push('--from', String(this.from))
|
||||
if (this.to) args.push('--to', String(this.to))
|
||||
if (this.limit) args.push('--limit', String(this.limit))
|
||||
if (this.noFinish) args.push('--no-finish')
|
||||
const started = Date.now()
|
||||
const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
|
||||
const totals = {
|
||||
batches: 0,
|
||||
headers: 0,
|
||||
releases: 0,
|
||||
newReleases: 0,
|
||||
finalized: 0,
|
||||
candidates: 0,
|
||||
storeMs: 0,
|
||||
}
|
||||
let current: StoredRange | null = null
|
||||
const storeLine = async (line: string) => {
|
||||
const batch = fromWire(JSON.parse(line) as WireBatch)
|
||||
if (!current || batch.from < current.start || batch.from > current.end) {
|
||||
current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
|
||||
if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
|
||||
}
|
||||
const at = Date.now()
|
||||
const result = await BatchStore.store(current.id, nodeId, batch)
|
||||
totals.storeMs += Date.now() - at
|
||||
totals.batches++
|
||||
totals.headers += batch.headers
|
||||
totals.releases += result.releases
|
||||
totals.newReleases += result.newReleases
|
||||
totals.finalized += result.finalized
|
||||
totals.candidates += result.candidates
|
||||
if (batch.next > current.end) {
|
||||
await Leases.finish(current.id, nodeId)
|
||||
const seconds = (Date.now() - started) / 1000
|
||||
this.logger.info(
|
||||
` range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
|
||||
`(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
|
||||
)
|
||||
}
|
||||
}
|
||||
// Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
|
||||
let rest = ''
|
||||
let failure: Error | null = null
|
||||
try {
|
||||
for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
|
||||
rest += chunk.toString('utf8')
|
||||
let nl: number
|
||||
while ((nl = rest.indexOf('\n')) >= 0) {
|
||||
const line = rest.slice(0, nl)
|
||||
rest = rest.slice(nl + 1)
|
||||
if (line.length > 0) await storeLine(line)
|
||||
}
|
||||
}
|
||||
if (rest.trim().length > 0) await storeLine(rest)
|
||||
} catch (error: any) {
|
||||
failure = error
|
||||
child.kill('SIGTERM')
|
||||
}
|
||||
const code = await new Promise<number | null>((done) =>
|
||||
child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
|
||||
)
|
||||
if (failure) {
|
||||
this.logger.error(`Replay failed: ${failure.message}`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
if (code !== 0) {
|
||||
this.logger.error(`edge-bench exited with ${code}`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
await this.report(groupId, ranges, Date.now() - started)
|
||||
}
|
||||
|
||||
private async benchBinary(): Promise<string | null> {
|
||||
const candidates = [
|
||||
this.bench,
|
||||
process.env.EDGE_BENCH,
|
||||
resolve('edge/target/release/edge-bench'),
|
||||
'/usr/local/bin/edge-bench',
|
||||
].filter((p): p is string => !!p)
|
||||
for (const path of candidates) {
|
||||
if (
|
||||
await access(path).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
)
|
||||
return path
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
|
||||
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
|
||||
const names = await readdir(groupDir).catch(() => [] as string[])
|
||||
const ranges: { start: number; end: number }[] = []
|
||||
for (const name of names) {
|
||||
const m = /^(\d+)-(\d+)$/.exec(name)
|
||||
if (!m) continue
|
||||
const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
|
||||
if (!json) continue
|
||||
ranges.push({ start: Number(m[1]), end: Number(m[2]) })
|
||||
}
|
||||
ranges.sort((a, b) => a.start - b.start)
|
||||
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
|
||||
if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
|
||||
if (this.limit) chosen = chosen.slice(0, this.limit)
|
||||
return chosen
|
||||
}
|
||||
|
||||
private async nodeId(): Promise<number> {
|
||||
const row = await db.from('nodes').where('name', NODE_NAME).first()
|
||||
if (row) return Number(row.id)
|
||||
const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
|
||||
return Number(inserted.id)
|
||||
}
|
||||
|
||||
/** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
|
||||
private async groupId(): Promise<number> {
|
||||
const row = await db.from('groups').where('name', this.group).first()
|
||||
if (row) return Number(row.id)
|
||||
const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
|
||||
const [inserted] = await db
|
||||
.table('groups')
|
||||
.insert({ name: this.group, priority: Number(rows[0].next), active: true })
|
||||
.returning('id')
|
||||
this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
|
||||
return Number(inserted.id)
|
||||
}
|
||||
|
||||
/** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
|
||||
private async leaseRanges(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
stored: { start: number; end: number }[]
|
||||
): Promise<StoredRange[]> {
|
||||
const out: StoredRange[] = []
|
||||
const now = new Date()
|
||||
for (const r of stored) {
|
||||
const lease = {
|
||||
status: 'leased',
|
||||
node_id: nodeId,
|
||||
next_id: r.start,
|
||||
headers: 0,
|
||||
posted_low: null,
|
||||
posted_high: null,
|
||||
leased_at: now,
|
||||
heartbeat_at: now,
|
||||
done_at: null,
|
||||
error: null,
|
||||
capture: true,
|
||||
}
|
||||
const existing = await db
|
||||
.from('ranges')
|
||||
.where('group_id', groupId)
|
||||
.where('start_id', r.start)
|
||||
.first()
|
||||
if (existing) {
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', existing.id)
|
||||
.update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
|
||||
out.push({ id: Number(existing.id), start: r.start, end: r.end })
|
||||
continue
|
||||
}
|
||||
const [inserted] = await db
|
||||
.table('ranges')
|
||||
.insert({
|
||||
...lease,
|
||||
group_id: groupId,
|
||||
start_id: r.start,
|
||||
end_id: r.end,
|
||||
priority: 0,
|
||||
attempts: 1,
|
||||
})
|
||||
.returning('id')
|
||||
out.push({ id: Number(inserted.id), start: r.start, end: r.end })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** What the replay left in the catalog (this group's releases; the sizes are the whole catalog's), per article. */
|
||||
private async report(groupId: number, ranges: StoredRange[], elapsedMs: number) {
|
||||
const ids = ranges.map((r) => r.id)
|
||||
const { rows: counts } = await db.rawQuery(
|
||||
`SELECT count(*)::bigint AS releases,
|
||||
count(settled_at)::bigint AS finished,
|
||||
count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
|
||||
count(*) FILTER (WHERE edge)::bigint AS edges,
|
||||
count(*) FILTER (WHERE complete)::bigint AS complete,
|
||||
count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
|
||||
count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
|
||||
count(*) FILTER (WHERE readable)::bigint AS readable,
|
||||
count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
|
||||
coalesce(sum(files), 0)::bigint AS files,
|
||||
coalesce(sum(segments), 0)::bigint AS segments,
|
||||
coalesce(sum(bytes), 0)::bigint AS bytes
|
||||
FROM releases WHERE group_id = ?`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: sources } = await db.rawQuery(
|
||||
`SELECT source, count(*)::bigint AS n FROM releases WHERE group_id = ? GROUP BY source ORDER BY source`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: pending } = await db.rawQuery(
|
||||
`SELECT count(*)::bigint AS n, coalesce(sum(segments), 0)::bigint AS segments, count(*) FILTER (WHERE jsonb_array_length(candidates) = 0)::bigint AS unprobable
|
||||
FROM pending_releases WHERE group_id = ?`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: headerRows } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
|
||||
[ids]
|
||||
)
|
||||
const headers = Number(headerRows[0].headers)
|
||||
const { rows: sizes } = await db.rawQuery(
|
||||
`SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
|
||||
sum(pg_total_relation_size(c.oid))::bigint AS bytes
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'r'
|
||||
AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks', 'pending_releases')
|
||||
OR c.relname LIKE 'release_chunks%')
|
||||
GROUP BY 1 ORDER BY 2 DESC`
|
||||
)
|
||||
// On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
|
||||
const hdd = new Set([
|
||||
'release_chunks',
|
||||
'probe_candidates',
|
||||
'release_inspections',
|
||||
'pending_releases',
|
||||
])
|
||||
let nvme = 0
|
||||
let other = 0
|
||||
const c = counts[0]
|
||||
const fmt = (n: unknown) => Number(n).toLocaleString('en')
|
||||
this.logger.info(
|
||||
`Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
|
||||
)
|
||||
this.logger.info(
|
||||
`Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
|
||||
`complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
|
||||
`${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
|
||||
)
|
||||
this.logger.info(
|
||||
`By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
|
||||
)
|
||||
this.logger.info(
|
||||
`Pending (unnamed, outside the catalog): ${fmt(pending[0].n)} releases, ${fmt(pending[0].segments)} segments, ${fmt(pending[0].unprobable)} without a probe target.`
|
||||
)
|
||||
for (const s of sizes) {
|
||||
const bytes = Number(s.bytes)
|
||||
if (hdd.has(s.name)) other += bytes
|
||||
else nvme += bytes
|
||||
this.logger.info(
|
||||
` ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
|
||||
)
|
||||
}
|
||||
this.logger.info(
|
||||
`NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header (the whole catalog's sizes over this group's headers: only the first group's are its own).`
|
||||
)
|
||||
}
|
||||
}
|
||||
+61
-2
@@ -31,8 +31,37 @@ export default {
|
||||
},
|
||||
/** A leased range without a heartbeat for this long goes back to the queue. */
|
||||
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
||||
/** NNTP connections the coordinator fetches headers with. */
|
||||
/**
|
||||
* The ranges group leases hand out (REBUILD.md design 1), by priority: the tail is negative, the capture windows
|
||||
* 0–2 (2016, 2021, 2025), the history by its position in the group (0 oldest … 1000 newest: its first buckets
|
||||
* share 0–2 with the windows) or 4000+. Bounds so a deployment can run one window first, with
|
||||
* LEASE_CAPTURE_ONLY so the history's 0 isn't taken along (homelab 2026-10-26: 0..0 and capture only, the 2016
|
||||
* window; LOG.md (118)); unbounded and any range by default.
|
||||
*/
|
||||
leasePriorityMin: env.get('LEASE_MIN_PRIORITY', -1_000_000),
|
||||
leasePriorityMax: env.get('LEASE_MAX_PRIORITY', 1_000_000),
|
||||
leaseCaptureOnly: env.get('LEASE_CAPTURE_ONLY', false),
|
||||
/**
|
||||
* How long a range's stream waits for a chunk the prefetcher hasn't spooled yet (app/controllers/node_controller.ts).
|
||||
* Long: a chunk that keeps coming in isn't given up (only a stalled one is), and some groups' chunks take 20+
|
||||
* minutes (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
|
||||
*/
|
||||
streamChunkWaitMs: env.get('STREAM_CHUNK_WAIT_MINUTES', 60) * 60 * 1000,
|
||||
/** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
|
||||
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
|
||||
nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
|
||||
/**
|
||||
* The account's connections, shared by the coordinator and the prober by priority (app/services/nntp_shares.ts):
|
||||
* all of them less the reserved ones (the spotter's and the spot inspector's 1 each, the coordinator's plain one,
|
||||
* and 2 for a hedged request's or a reconnect's new connection while the old one closes).
|
||||
*/
|
||||
nntpShares: {
|
||||
total: env.get('NNTP_ACCOUNT_CONNECTIONS', 50) - env.get('NNTP_RESERVED_CONNECTIONS', 5),
|
||||
everyMs: 15_000,
|
||||
/** Headers before probes: history takes what it can use, the prober the rest. */
|
||||
coordinatorPriority: 2,
|
||||
proberPriority: 1,
|
||||
},
|
||||
/**
|
||||
* A reply with no data for this long fails, and its connection is replaced.
|
||||
* Eweka sometimes stops answering on long-lived connections (all 40 quiet,
|
||||
@@ -68,17 +97,36 @@ export default {
|
||||
* under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
|
||||
*/
|
||||
nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
|
||||
/**
|
||||
* A connection with nothing to do for this long is closed, and opened again
|
||||
* when there's work. 0: kept. Eweka drops connections that stay quiet, and
|
||||
* the next command on one failed ("Spot inspection failed: NNTP connection
|
||||
* closed"); idle ones also take the account's connections from downloads.
|
||||
*/
|
||||
nntpIdleCloseMs: env.get('NNTP_IDLE_CLOSE_SECONDS', 60) * 1000,
|
||||
probe: {
|
||||
/**
|
||||
* NNTP connections the prober reads articles with, besides the
|
||||
* At most (its share, nntpShares): NNTP connections the prober reads articles with, besides the
|
||||
* coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
|
||||
* couple stay free for downloads. ~7 MB/s each (~10 articles/s).
|
||||
*/
|
||||
connections: env.get('PROBE_CONNECTIONS', 6),
|
||||
/** Of them, always (its share's minimum; nntpShares). */
|
||||
minConnections: env.get('PROBE_MIN_CONNECTIONS', 2),
|
||||
/** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */
|
||||
minFreeGb: env.get('PROBE_MIN_FREE_GB', 0),
|
||||
/** Candidates claimed at a time. */
|
||||
batch: 400,
|
||||
/** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
|
||||
pipeline: env.get('PROBE_PIPELINE', 8),
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f; app/services/pending_ladder.ts): the share of each claim taken
|
||||
* from pending_releases at least (the catalog's own candidates come first, the rest of the claim is theirs).
|
||||
*/
|
||||
pendingShare: env.get('PENDING_SHARE', 0.25),
|
||||
/** A signature (tool, group, poster) with this many climbs and at most pendingMaxNamed of them named is dropped. */
|
||||
pendingMinProbes: env.get('PENDING_MIN_PROBES', 100),
|
||||
pendingMaxNamed: env.get('PENDING_MAX_NAMED', 0.005),
|
||||
},
|
||||
/**
|
||||
* The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
|
||||
@@ -107,7 +155,18 @@ export default {
|
||||
* ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
|
||||
*/
|
||||
ahead: env.get('CORPUS_AHEAD', 16),
|
||||
/**
|
||||
* Stored without the lines the edge worker drops on their own (crossposts another indexed group owns, obfuscated
|
||||
* subjects; ~48% of the indexed ranges' headers), what was left out in each chunk's `.drops`: twice the history
|
||||
* in the same space (LOG.md 2026-10-23 (101)). Off: as the news server sent them.
|
||||
*/
|
||||
filter: env.get('CORPUS_FILTER', true),
|
||||
},
|
||||
/**
|
||||
* The group windows' snapshots (REBUILD.md, "Voyager 3: window indexing", 5; app/services/group_leases.ts): a
|
||||
* file a group on bulk, next to the spool by default.
|
||||
*/
|
||||
snapshotDir: env.get('SNAPSHOT_DIR', `${env.get('SPOOL_DIR', app.tmpPath('spool'))}/snapshots`),
|
||||
spool: {
|
||||
dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
|
||||
/** Ranges kept fetched ahead, per active lease. */
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* releases.sidecar_only: set by the batch store when it inserts a small release (at most 3 files, under 1 MB) whose
|
||||
* files are all sidecars (.nfo, .sfv, .nzb, …). The settler queues only those for the sidecar fold: it queued every
|
||||
* small release, 13.5M (single images mostly), and the fold's queue scans held the merger back (LOG.md 2026-10-17 (72)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS sidecar_only boolean`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS sidecar_only`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* worker_cursors.key: a tie-breaker after `at`. The par2 pruner's cursor was a timestamp alone, read back to the
|
||||
* millisecond: 20,000 sets of one prober flush share created_at 2026-10-15 14:38:39.883984, and every round took the
|
||||
* same batch again (LOG.md 2026-10-17 (73)). Now (at to the microsecond, set_id).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE worker_cursors ADD COLUMN IF NOT EXISTS key text`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE worker_cursors DROP COLUMN IF EXISTS key`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* nntp_shares: the account's connections shared by the processes that read articles (app/services/nntp_shares.ts).
|
||||
* Each process's row says what it could use and has open, and what it may open; a row without a heartbeat for a
|
||||
* minute is a process gone (its connections closed with it). The split was fixed per process: with history paused
|
||||
* the coordinator's 38 stood idle while the prober's 8 were the limit (LOG.md 2026-10-18 (75)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE IF NOT EXISTS nntp_shares (
|
||||
name text PRIMARY KEY,
|
||||
priority integer NOT NULL,
|
||||
min integer NOT NULL,
|
||||
want integer NOT NULL,
|
||||
open integer NOT NULL,
|
||||
granted integer NOT NULL,
|
||||
heartbeat_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS nntp_shares`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* probe_cache's primary key in the catalog's tablespace (default_tablespace: nvme on homelab); the table stays on the
|
||||
* HDD. Each claim looks its candidates up by message_id (`message_id = ANY(…)`): with the index on the HDD that took
|
||||
* 1.27 s a claim, 48 s of each minute, the prober's lanes waiting on claims (LOG.md 2026-10-18 (76)). ~97% aren't in it,
|
||||
* so only the index is read. Homelab moved it by hand (prober stopped); a no-op where it's there already, or without a
|
||||
* default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE c.relname = 'probe_cache_pkey' AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||
[target]
|
||||
)
|
||||
if (rows.length > 0) await db.rawQuery(`ALTER INDEX probe_cache_pkey SET TABLESPACE ${target}`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Was: release_inspections' rows on the HDD (pg_default), for the NVMe's room during the probe backlog (LOG.md
|
||||
* 2026-10-18 (79)). The judge's fake rule reads all of it every round: on the HDD that took 17–18 min, was ~3
|
||||
* (LOG.md 2026-10-19 (81)). Migration 4800 moves it back; nothing here now.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The junk judge's locked rule by an index of the passworded releases (1.2M of 74M), not a scan of all of releases
|
||||
* every round (30 s; LOG.md 2026-10-19 (80)). And the posters of the spot inspector's own releases (source `spot`),
|
||||
* which it didn't add to posters: poster search, and the judge's scraped rule by posters, missed them.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_passworded_idx ON releases (id) WHERE passworded`)
|
||||
this.schema.raw(
|
||||
`INSERT INTO posters (poster) SELECT DISTINCT left(s.poster, 200) FROM spots s
|
||||
JOIN releases r ON r.id = s.release_id AND r.source = 'spot' WHERE s.poster IS NOT NULL ON CONFLICT DO NOTHING`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_passworded_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* release_inspections back in the catalog's tablespace (default_tablespace: nvme on homelab), where migration 4600 had
|
||||
* moved it to the HDD: the judge's fake rule reads all of it every round, 17–18 min on the HDD, ~3 on the NVMe (LOG.md
|
||||
* 2026-10-19 (81)). Homelab moved it by hand (prober, merger and judge stopped); a no-op where it's there already, or
|
||||
* without a default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE c.relname = 'release_inspections' AND c.relkind = 'r' AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||
[target]
|
||||
)
|
||||
if (rows.length > 0) await db.rawQuery(`ALTER TABLE release_inspections SET TABLESPACE ${target}`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The junk judge by what changed (LOG.md 2026-10-19 (83)): a full round over every release took 35–38 min of each
|
||||
* hour, longer with every inspection the prober added. Triggers note each release whose facts a rule reads changed
|
||||
* (judge_dirty: its probes' statuses, inspections, file hashes, passwords, spots, par2 sets, merges, its own judged
|
||||
* columns); the judge judges those every few minutes, and every release in a daily full round (judge_state: when the
|
||||
* last one was, by which rule set), its matches gathered in judge_full. judge_dirty is unlogged: what a crash loses
|
||||
* the next full round judges.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_dirty (release_id bigint NOT NULL)`)
|
||||
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_full (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL)`)
|
||||
this.schema.raw(
|
||||
`CREATE TABLE IF NOT EXISTS judge_state (id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1), full_at timestamptz NOT NULL, version integer NOT NULL)`
|
||||
)
|
||||
// The verdicts are an hourly full round's: the next full one a day from now.
|
||||
this.schema.raw(`INSERT INTO judge_state (full_at, version) VALUES (now(), 1) ON CONFLICT DO NOTHING`)
|
||||
|
||||
// Rows of a table with release_id: the releases of the rows inserted, deleted, or updated (both, when moved).
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_by_release_id() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('INSERT', 'UPDATE') THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
||||
END IF;
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM old_rows WHERE release_id IS NOT NULL;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
// probe_candidates: only a status that counts for the dead rule, or a candidate moved by a merge (not the claims'
|
||||
// 'taken', the settler's ready).
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_probes() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
INSERT INTO judge_dirty (release_id)
|
||||
SELECT DISTINCT unnest(ARRAY[n.release_id, o.release_id]) FROM new_rows n JOIN old_rows o ON o.id = n.id
|
||||
WHERE n.release_id IS DISTINCT FROM o.release_id
|
||||
OR (n.status IS DISTINCT FROM o.status AND (n.status IN ('done', 'missing', 'error') OR o.status IN ('done', 'missing', 'error')));
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_merges() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT into_id FROM new_rows;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_releases() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT id FROM new_rows;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT id FROM old_rows;
|
||||
ELSE
|
||||
INSERT INTO judge_dirty (release_id) VALUES (NEW.id);
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
|
||||
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets']) {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON ${table}`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON ${table}`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON ${table}`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON ${table} REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON ${table} REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON ${table} REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
}
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON probe_candidates`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON probe_candidates`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON probe_candidates REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_probes()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON probe_candidates REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON release_merges`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_merges REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_merges()`
|
||||
)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON releases`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON releases`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON releases`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
// Per row, and only when a column a rule reads changed: the tail and the settler update releases all the time.
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
|
||||
WHEN (OLD.name IS DISTINCT FROM NEW.name OR OLD.poster IS DISTINCT FROM NEW.poster OR OLD.source IS DISTINCT FROM NEW.source
|
||||
OR OLD.file_total IS DISTINCT FROM NEW.file_total OR OLD.passworded IS DISTINCT FROM NEW.passworded
|
||||
OR OLD.settled_at IS DISTINCT FROM NEW.settled_at OR OLD.readable IS DISTINCT FROM NEW.readable
|
||||
OR OLD.inner_name IS DISTINCT FROM NEW.inner_name OR OLD.group_id IS DISTINCT FROM NEW.group_id
|
||||
OR OLD.first_article IS DISTINCT FROM NEW.first_article OR OLD.last_article IS DISTINCT FROM NEW.last_article
|
||||
OR OLD.posted_at IS DISTINCT FROM NEW.posted_at OR OLD.last_posted_at IS DISTINCT FROM NEW.last_posted_at
|
||||
OR OLD.complete IS DISTINCT FROM NEW.complete)
|
||||
EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets', 'probe_candidates', 'release_merges', 'releases']) {
|
||||
for (const event of ['insert', 'update', 'delete']) this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_${event} ON ${table}`)
|
||||
}
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS judge_dirty_by_release_id(), judge_dirty_probes(), judge_dirty_merges(), judge_dirty_releases()`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS judge_dirty, judge_full, judge_state`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* When a group's articles were posted, sampled from the news server (ranges:dates; app/services/group_dates.ts), and
|
||||
* from those each range's estimated posting date, so ranges can be chosen by era: ranges:capture --from --to marks a
|
||||
* posting-time window of every group for the corpus (voyager 3's test data; REBUILD.md, corpus). `stored_at`: the
|
||||
* range is complete in the corpus (the prefetcher saw its range.json), so the corpus fill skips it without looking.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE IF NOT EXISTS group_dates (
|
||||
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
|
||||
article bigint NOT NULL,
|
||||
posted_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (group_id, article)
|
||||
)`)
|
||||
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS est_date timestamptz`)
|
||||
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS stored_at timestamptz`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS stored_at`)
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS est_date`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS group_dates`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* file_hashes by (hash16k, size, release_id), in place of (hash16k): how many releases have a file, from the index
|
||||
* alone. The judge counts that for every file of each changed release (repeated); with the row fetched for size and
|
||||
* release_id per entry, a file posted in 100k releases cost 100k heap reads every round (24–32 s a chunk; LOG.md
|
||||
* 2026-10-19 (84)). The merger and pruner join on (hash16k, size) too.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_size_release_idx ON file_hashes (hash16k, size, release_id)`)
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_idx`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_idx ON file_hashes (hash16k)`)
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_size_release_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Probe candidates added to a release after it settled (a later range of its upload, a merge into it) stayed not
|
||||
* ready: the settler marks a release's candidates only when it settles it (LOG.md 2026-10-20 (87)). 93,642 waited
|
||||
* since 2026-10-13 (17k lone, 31k par2, 45k archive). The batch store and the merge mark them now; these are made
|
||||
* claimable once.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.status = 'waiting' AND NOT pc.ready AND r.id = pc.release_id AND r.settled_at IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release's category from more than its name (REBUILD.md, part 9; LOG.md 2026-10-23 (95)). Migration 1200 made it
|
||||
* a generated column of release_category(name), and knex took the `?` in its regexes for bindings: the function held
|
||||
* `blu-$1ray`, `web-$2dl`, `h\.$326[45]`, so BluRay, WEB-DL and H.264 names without a resolution were Misc. Now:
|
||||
* - release_category(name), fixed (no `?` in any regex here: `{0,1}`) and with XXX, audiobooks, console and PC games,
|
||||
* and TV codes the old one missed (`S01e07[`, `S04E01E02`);
|
||||
* - the inner name the prober found (an obfuscated release's real name) when the name says nothing;
|
||||
* - else a category given before (a spot's, below), else the groups' (erotica, sounds, ebooks, …);
|
||||
* - a spot's category (spot_category: its head category and subcategories), set by the spot inspector.
|
||||
* `category` becomes a plain column (DROP EXPRESSION keeps the values, no rewrite) set by a trigger; existing rows are
|
||||
* recategorized by `releases:recategorize` in batches, not here (a rewrite of the 70 GB table).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- XXX first: its names carry years, resolutions and episode-like codes too.
|
||||
WHEN name ~* '(^|[^a-z0-9])xxx([^a-z0-9]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 6045
|
||||
WHEN name ~* '720p|1080p|x26[45]|h\.{0,1}26[45]|hevc' THEN 6040
|
||||
WHEN name ~* 'imageset|picset|(^|[._ -])pics([._ -]|$)' THEN 6060 ELSE 6070 END
|
||||
-- TV: S01E02 (also S01e02[, S04E01E02), 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[^a-z0-9])(s[0-9]{1,2}[ ._-]{0,1}e[0-9]{1,3}|[0-9]{1,2}x[0-9]{2})([^0-9]|$)' OR name ~* '(^|[._ -])s[0-9]{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)[0-9]{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-{0,1}ray|web-{0,1}dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.{0,1}26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-{0,1}ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: audiobooks, lossless, mp3, or scene music names (-WEB-2026-GROUP).
|
||||
WHEN name ~* '(audiobook|audioboek|luisterboek|h(oe|ö)rbuch)' THEN 3030
|
||||
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)[0-9]{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN name ~* '(^|[._ -])(epub|ebook|e-book|mobi|azw3)([._ -]|$)' THEN 7020
|
||||
-- Games: console tags, PC scene groups and repackers.
|
||||
WHEN name ~* '(^|[._ -])(nsw|ps4|ps5|xboxone|xbox360|wiiu|3ds|nds)([._ -]|$)' THEN 1000
|
||||
WHEN name ~* '-(codex|plaza|skidrow|reloaded|flt|doge|tenoke|rune|empress|razor1911|tinyiso|darksiders|simplex)$' OR name ~* '(fitgirl|dodi|elamigos)[._ -]{0,1}repack' THEN 4050
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_group_category(groups jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
WHEN g ~* '(erotica|xxx|porn|erotic|nude|naked)' THEN CASE WHEN g ~* '(pictures|pics|images|photos)' THEN 6060 ELSE 6070 END
|
||||
WHEN g ~* 'anime' THEN 5070
|
||||
WHEN g ~* 'audiobook' THEN 3030
|
||||
WHEN g ~* '(sounds|mp3|music)' THEN CASE WHEN g ~* '(lossless|flac)' THEN 3040 ELSE 3010 END
|
||||
WHEN g ~* 'comics' THEN 7030
|
||||
WHEN g ~* 'e-{0,1}books{0,1}' THEN 7020
|
||||
WHEN g ~* '(games|nintendo|xbox|playstation|wii)' THEN 1000
|
||||
WHEN g ~* '(^|[. ])(tv|hdtv|teevee|tvseries)([. ]|$)' THEN 5000
|
||||
WHEN g ~* '(^|[. ])(movies|dvd|bluray|dvdr)([. ]|$)' THEN 2000
|
||||
ELSE 8010
|
||||
END
|
||||
FROM (SELECT coalesce(string_agg(value, ' '), '') AS g FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(groups) = 'array' THEN groups ELSE '[]' END)) x
|
||||
$$`)
|
||||
// The name's, else the inner name's, else one given before (a spot's), else the groups'.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION release_category_for(name text, inner_name text, groups jsonb, given integer) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE WHEN n <> 8010 THEN n WHEN i <> 8010 THEN i WHEN coalesce(given, 8010) <> 8010 THEN given ELSE release_group_category(groups) END
|
||||
FROM (SELECT release_category(name) AS n, coalesce(release_category(inner_name), 8010) AS i) x
|
||||
$$`)
|
||||
// Whether a spot's subcategories of one type letter ({"a": [9], "z": [0]}) hold one of `vals`.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_has(subcats jsonb, letter text, vals integer[]) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((SELECT bool_or((e #>> '{}')::integer = ANY(vals)) FROM jsonb_array_elements(
|
||||
CASE WHEN jsonb_typeof(subcats->letter) = 'array' THEN subcats->letter ELSE '[]' END) e), false)
|
||||
$$`)
|
||||
// A spot's category (lib/spotnet/categories.ts) in Newznab's; null for pictures and the unknown.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_category(category integer, subcats jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE category
|
||||
WHEN 0 THEN CASE
|
||||
WHEN spot_has(subcats, 'z', '{3}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 6045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 6040
|
||||
WHEN spot_has(subcats, 'a', '{12,13}') THEN 6060 ELSE 6070 END
|
||||
WHEN spot_has(subcats, 'z', '{1}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 5045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 5040 ELSE 5030 END
|
||||
WHEN spot_has(subcats, 'z', '{2}') OR spot_has(subcats, 'a', '{5,11}') THEN 7020
|
||||
WHEN spot_has(subcats, 'z', '{4}') THEN NULL
|
||||
ELSE CASE WHEN spot_has(subcats, 'a', '{15}') THEN 2045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 2040 ELSE 2030 END
|
||||
END
|
||||
WHEN 1 THEN CASE WHEN spot_has(subcats, 'z', '{3}') THEN 3030 WHEN spot_has(subcats, 'a', '{2,4,7,8}') OR spot_has(subcats, 'c', '{8}') THEN 3040 ELSE 3010 END
|
||||
WHEN 2 THEN CASE WHEN spot_has(subcats, 'a', '{0,1,2}') THEN 4050 ELSE 1000 END
|
||||
WHEN 3 THEN 4000
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, NEW.groups, CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`)
|
||||
// ACCESS EXCLUSIVE for a moment, while the workers write: a few tries instead of a queue behind them.
|
||||
this.schema.raw(`
|
||||
DO $$
|
||||
BEGIN
|
||||
FOR attempt IN 1..30 LOOP
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'releases' AND column_name = 'category' AND is_generated = 'ALWAYS') THEN
|
||||
ALTER TABLE releases ALTER COLUMN category DROP EXPRESSION;
|
||||
END IF;
|
||||
DROP TRIGGER IF EXISTS releases_category ON releases;
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, groups ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category();
|
||||
RETURN;
|
||||
EXCEPTION WHEN lock_not_available THEN
|
||||
PERFORM pg_sleep(2);
|
||||
END;
|
||||
END LOOP;
|
||||
RAISE EXCEPTION 'releases: no lock for the category trigger in 30 tries';
|
||||
END
|
||||
$$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS releases_category ON releases`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS releases_set_category()`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_category(integer, jsonb)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_has(jsonb, text, integer[])`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_category_for(text, text, jsonb, integer)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_group_category(jsonb)`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
|
||||
* place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
|
||||
* inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
|
||||
* groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
|
||||
* with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
|
||||
* - posters get an id; group_sets holds each distinct group combination once (~140k); releases is created anew
|
||||
* without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
|
||||
* columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
|
||||
* (null: a tail release still provisional). The category and judge triggers follow the new columns;
|
||||
* releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts
|
||||
* (migration 6200), which holds the name and every searchable string learned later.
|
||||
* - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
|
||||
* release_final() goes: a release is final once finished.
|
||||
* - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
|
||||
* - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
|
||||
* release, not by search.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
|
||||
if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
|
||||
throw new Error(
|
||||
`The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
|
||||
)
|
||||
}
|
||||
})
|
||||
this.schema.raw(`
|
||||
TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
|
||||
release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
|
||||
judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
|
||||
// Spots matched to releases that are gone: the spot inspector matches them again.
|
||||
this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
|
||||
this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)
|
||||
|
||||
this.schema.raw(`DROP TABLE IF EXISTS fragments`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
|
||||
this.schema.raw(
|
||||
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
|
||||
)
|
||||
|
||||
// Posters by id (a release's poster_id); searched by trigram as before (poster_search).
|
||||
this.schema.raw(`DROP TABLE IF EXISTS posters`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE posters (
|
||||
id serial PRIMARY KEY,
|
||||
poster text NOT NULL UNIQUE
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
|
||||
// Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
|
||||
this.schema.raw(`
|
||||
CREATE TABLE group_sets (
|
||||
id serial PRIMARY KEY,
|
||||
groups jsonb NOT NULL UNIQUE
|
||||
)`)
|
||||
|
||||
this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE releases (
|
||||
id bigserial PRIMARY KEY,
|
||||
-- The upload's posting window (first and last article's Date).
|
||||
posted_at timestamptz,
|
||||
last_posted_at timestamptz,
|
||||
first_article bigint,
|
||||
last_article bigint,
|
||||
segments bigint NOT NULL DEFAULT 0,
|
||||
bytes bigint NOT NULL DEFAULT 0,
|
||||
-- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
|
||||
settled_at timestamptz,
|
||||
inspected_at timestamptz,
|
||||
first_seen_at timestamptz NOT NULL DEFAULT now(),
|
||||
-- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
|
||||
group_id integer,
|
||||
group_set_id integer NOT NULL,
|
||||
poster_id integer,
|
||||
file_total integer,
|
||||
files integer NOT NULL DEFAULT 0,
|
||||
chunks integer NOT NULL DEFAULT 0,
|
||||
category integer,
|
||||
-- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
|
||||
source smallint NOT NULL,
|
||||
readable boolean NOT NULL DEFAULT false,
|
||||
has_par2 boolean NOT NULL DEFAULT false,
|
||||
complete boolean,
|
||||
passworded boolean,
|
||||
sidecar_only boolean,
|
||||
-- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
|
||||
edge boolean NOT NULL DEFAULT false,
|
||||
-- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
|
||||
release_key bytea NOT NULL,
|
||||
identity_key bytea NOT NULL,
|
||||
name text NOT NULL,
|
||||
inner_name varchar(255)
|
||||
)`)
|
||||
this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
|
||||
this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
|
||||
// No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
|
||||
)
|
||||
|
||||
// The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
|
||||
CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
|
||||
// The judge's dirty marks (migration 4900), on the columns its rules read.
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
|
||||
WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
|
||||
OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
|
||||
OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
|
||||
OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
|
||||
OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
|
||||
OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
|
||||
OR old.complete IS DISTINCT FROM new.complete)
|
||||
EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
|
||||
// Chunks, rule counts and samples by the block that wrote them.
|
||||
this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(
|
||||
`ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(
|
||||
`ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
|
||||
)
|
||||
|
||||
// Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
|
||||
const hdd = env.get('CHUNK_TABLESPACE')
|
||||
if (hdd) {
|
||||
for (const table of ['release_inspections', 'probe_candidates']) {
|
||||
this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`
|
||||
DO $$ DECLARE i record; BEGIN
|
||||
FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
|
||||
EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
|
||||
END LOOP;
|
||||
END $$`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive
|
||||
* ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high),
|
||||
* read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the
|
||||
* prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and
|
||||
* counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE blocks (
|
||||
id bigserial PRIMARY KEY,
|
||||
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
|
||||
-- The first range's start and the last range's end.
|
||||
start_id bigint NOT NULL,
|
||||
end_id bigint NOT NULL,
|
||||
-- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were).
|
||||
next_id bigint NOT NULL,
|
||||
-- The posting time the block covers: estimated when planned, the worker's measured span once done.
|
||||
posted_low timestamptz,
|
||||
posted_high timestamptz,
|
||||
-- planned, leased, done, failed.
|
||||
status varchar(8) NOT NULL DEFAULT 'planned',
|
||||
-- As ranges': the tail before everything, then the history by era.
|
||||
priority integer NOT NULL DEFAULT 0,
|
||||
node_id integer,
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
rule_version integer,
|
||||
headers bigint NOT NULL DEFAULT 0,
|
||||
releases integer NOT NULL DEFAULT 0,
|
||||
leased_at timestamptz,
|
||||
heartbeat_at timestamptz,
|
||||
done_at timestamptz,
|
||||
error text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (group_id, start_id)
|
||||
)`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`)
|
||||
this.schema.raw(
|
||||
`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`)
|
||||
// The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture,
|
||||
// est_date, stored_at, plain_chunks and the measured posted_low/high stay.
|
||||
this.schema.raw(`
|
||||
UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0,
|
||||
spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL,
|
||||
rule_version = NULL
|
||||
WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS blocks`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* release_texts (REBUILD.md, "Voyager 3 … Schema"): every string search matches against, one table, one trigram
|
||||
* index. The release name goes in (kind 1) with whatever anyone learns later: the prober's file names inside an
|
||||
* archive, NFO titles, par2 set names, media titles (lib/release/text_kind.ts). Appending is an insert, never an
|
||||
* update of the wide releases row; a repeat insert is a no-op by the primary key; the judge's dirty mark follows
|
||||
* (judge_dirty_by_release_id, migration 4900), so a rule can react to a name arriving later. On the NVMe: search
|
||||
* reads it. releases keeps name and inner_name for display, without trigram indexes (migration 6000).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE release_texts (
|
||||
release_id bigint NOT NULL,
|
||||
-- lib/release/text_kind.ts: 1 name, 2 file, 3 nfo, 4 par2, 5 media.
|
||||
kind smallint NOT NULL,
|
||||
text varchar(255) NOT NULL,
|
||||
PRIMARY KEY (release_id, kind, text)
|
||||
)`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX release_texts_trgm ON release_texts USING gin (text gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_texts REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON release_texts REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS release_texts`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* pending_releases (REBUILD.md decision f, 2026-10-26): nothing enters the catalog without a usable name. A release
|
||||
* the window closed on whose files all have random names is written here instead of releases: its keys (as the
|
||||
* catalog would store them), its counts and article span, the upload tool from its message-id (with the group its
|
||||
* signature), and its probe targets by effort (the edge's candidates: a par2, an archive start, a lone file, a 7z
|
||||
* tail), no chunk. The pending ladder probes them cheapest first and moves a named one into the catalog, reading its
|
||||
* segments again from the corpus or the server; a signature that names nothing is discarded. On CHUNK_TABLESPACE
|
||||
* (the HDD): written once, read by the ladder, never by search.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE pending_releases (
|
||||
-- lib/release/keys.ts: the identity, the group and the first article; the identity alone.
|
||||
release_key bytea PRIMARY KEY,
|
||||
identity_key bytea NOT NULL,
|
||||
group_id integer NOT NULL,
|
||||
-- The stem or post the edge keyed it by (a random name: no release name, but the inner-stem merge's handle).
|
||||
name varchar(255) NOT NULL,
|
||||
poster text,
|
||||
posted_at timestamptz,
|
||||
last_posted_at timestamptz,
|
||||
first_article bigint NOT NULL,
|
||||
last_article bigint NOT NULL,
|
||||
files integer NOT NULL DEFAULT 0,
|
||||
segments bigint NOT NULL DEFAULT 0,
|
||||
bytes bigint NOT NULL DEFAULT 0,
|
||||
-- A file misses parts.
|
||||
partial boolean NOT NULL DEFAULT false,
|
||||
-- False: a tail release still open; a later chunk of its key adds its counts up.
|
||||
finished boolean NOT NULL DEFAULT true,
|
||||
-- lib/rules uploadTool: ngPost, nyuu, hex32, random, other, …
|
||||
tool varchar(16) NOT NULL,
|
||||
-- [{kind, messageId, file}] by effort (lib/edge ProbeCandidate); empty: nothing to read, only a signature.
|
||||
candidates jsonb NOT NULL DEFAULT '[]',
|
||||
-- 0 waiting, 1 named (in the catalog), 2 discarded (its signature's verdict, or every probe gave nothing).
|
||||
status smallint NOT NULL DEFAULT 0,
|
||||
tries smallint NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
probed_at timestamptz
|
||||
)`)
|
||||
// The ladder's order: a group's and tool's waiting releases, oldest first.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX pending_releases_ladder_idx ON pending_releases (group_id, tool, status, created_at)`
|
||||
)
|
||||
const hdd = env.get('CHUNK_TABLESPACE')
|
||||
if (hdd) {
|
||||
this.schema.raw(`ALTER TABLE pending_releases SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`ALTER INDEX pending_releases_pkey SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`ALTER INDEX pending_releases_ladder_idx SET TABLESPACE ${hdd}`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS pending_releases`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f, 2026-10-26): what the prober needs to work pending_releases cheapest
|
||||
* first and to learn which signatures never name anything.
|
||||
* - pending_releases: `file_total` (the counter's, for the catalog row once named), `inspections` (what each probe
|
||||
* showed, by kind), `named` (the usable name a probe found; status 4 until the namer moved it into the catalog).
|
||||
* Status: 0 waiting, 1 named and in the catalog, 2 discarded (its signature's verdict), 3 exhausted (every
|
||||
* target probed, no name), 4 named, awaiting the move, 9 taken by a running prober.
|
||||
* - pending_signatures: per (tool, group, poster) how many pending releases were probed to the end and how many
|
||||
* got a name; `dropped` once probed ≥ PENDING_MIN_PROBES and named ≤ PENDING_MAX_NAMED of them (voyager 2's
|
||||
* garbage learner, LOG.md 2026-10-08 (5)). The poster '' row is the tool's and group's together.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS file_total integer`)
|
||||
this.schema.raw(
|
||||
`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS inspections jsonb NOT NULL DEFAULT '[]'`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS named varchar(255)`)
|
||||
// The ladder's claim: the oldest waiting ones; the namer's: the named ones.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX IF NOT EXISTS pending_releases_waiting_idx ON pending_releases (created_at) WHERE status = 0`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX IF NOT EXISTS pending_releases_naming_idx ON pending_releases (probed_at) WHERE status = 4`
|
||||
)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE pending_signatures (
|
||||
tool varchar(16) NOT NULL,
|
||||
group_id integer NOT NULL,
|
||||
poster text NOT NULL DEFAULT '',
|
||||
probed integer NOT NULL DEFAULT 0,
|
||||
named integer NOT NULL DEFAULT 0,
|
||||
dropped boolean NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (tool, group_id, poster)
|
||||
)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS pending_signatures`)
|
||||
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_naming_idx`)
|
||||
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_waiting_idx`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS named`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS inspections`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS file_total`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5; 2026-10-26): a worker thread holds a group, not
|
||||
* a range, and indexes its ranges in article order through one carried window. One row a group: the holder and
|
||||
* its heartbeat, the range it works, and the snapshot of its window (the article it stands at: the next one to
|
||||
* index) that the coordinator keeps on bulk. Ranges stay what is planned, fetched and finished.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE group_leases (
|
||||
group_id integer PRIMARY KEY REFERENCES groups (id) ON DELETE CASCADE,
|
||||
-- The holder; null between holders. A holder quiet for LEASE_TTL_SECONDS is taken over.
|
||||
node_id integer,
|
||||
leased_at timestamptz,
|
||||
heartbeat_at timestamptz,
|
||||
-- The range the holder works (null between ranges).
|
||||
range_id bigint,
|
||||
-- The window's snapshot: the article it stands at (a resume point), and when it was taken; null: none.
|
||||
snapshot_next bigint,
|
||||
snapshot_at timestamptz,
|
||||
ranges_done integer NOT NULL DEFAULT 0,
|
||||
headers bigint NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX group_leases_held_idx ON group_leases (heartbeat_at) WHERE node_id IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS group_leases`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Lease lanes (REBUILD.md, "Voyager 3: window indexing", 1, amended 2026-10-27): a group has two cursors, each with
|
||||
* its own holder, window and snapshot. The `history` lane works the ranges the lease bounds hand out (the windows,
|
||||
* the history), the `tail` lane the tail planner's (priority at or below the tail's): a thread on the tail never
|
||||
* waits behind a group's history, and a held history group's tail isn't stuck behind its days of ranges.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`ALTER TABLE group_leases ADD COLUMN lane varchar(8) NOT NULL DEFAULT 'history'`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
|
||||
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id, lane)`)
|
||||
// Counts the leases handed out: odd and even ones start from the other half of the order's mix.
|
||||
this.schema.raw(`CREATE SEQUENCE group_lease_turns`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP SEQUENCE IF EXISTS group_lease_turns`)
|
||||
this.schema.raw(`DELETE FROM group_leases WHERE lane <> 'history'`)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
|
||||
this.schema.raw(`ALTER TABLE group_leases DROP COLUMN lane`)
|
||||
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id)`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* One "ancient" range a group, articles 1 … (its first range − 1): what had expired on the server before the
|
||||
* group was planned (the planner cuts from the low watermark). Status 'ancient': nothing to fetch, never leased,
|
||||
* so the tiling is complete from article 1 and the span below the low watermark is on record (user, 2026-10-28).
|
||||
* Planner.historyRows adds it for groups planned from now on.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(
|
||||
`INSERT INTO ranges (group_id, start_id, end_id, next_id, priority, status)
|
||||
SELECT group_id, 1, min(start_id) - 1, 1, 0, 'ancient' FROM ranges GROUP BY group_id HAVING min(start_id) > 1`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DELETE FROM ranges WHERE status = 'ancient'`)
|
||||
}
|
||||
}
|
||||
+36
-8
@@ -115,18 +115,26 @@ services:
|
||||
<<: *app-env
|
||||
NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38}
|
||||
|
||||
# Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
|
||||
# connections of its own. The account has 50: the coordinator's 38 (+ its
|
||||
# plain one), the prober's 8, the spotter's and the spot inspector's 1 each,
|
||||
# 2 to spare (a hedged request closes its connection and opens another)
|
||||
# (with 10 the spot inspector got 502 Too many connections).
|
||||
# Phase 2's prober: reads the probe queue's articles over connections of its
|
||||
# own. The account has 50 (NNTP_ACCOUNT_CONNECTIONS): 5 reserved (the
|
||||
# coordinator's plain one, the spotter's and the spot inspector's 1 each, 2
|
||||
# to spare: a hedged request closes its connection and opens another; with
|
||||
# 10 the spot inspector got 502 Too many connections), the other 45 shared
|
||||
# by the coordinator (NNTP_MIN_CONNECTIONS–NNTP_CONNECTIONS) and the prober
|
||||
# (PROBE_MIN_CONNECTIONS–PROBE_CONNECTIONS), headers first
|
||||
# (app/services/nntp_shares.ts).
|
||||
prober:
|
||||
<<: *app
|
||||
command: ['node', 'ace', 'probe:run']
|
||||
volumes: []
|
||||
# The pending namer reads a named release's articles again from the corpus (no spool).
|
||||
volumes:
|
||||
- ./corpus:/corpus:ro
|
||||
environment:
|
||||
<<: *app-env
|
||||
PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8}
|
||||
# No claims under this much free on the NVMe (its overlay root): 2 GB above the 4 GB floor (2026-10-24), for
|
||||
# what the tail, the merger and the judge write meanwhile.
|
||||
PROBE_MIN_FREE_GB: ${PROBE_MIN_FREE_GB:-6}
|
||||
|
||||
# The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP.
|
||||
merger:
|
||||
@@ -134,8 +142,8 @@ services:
|
||||
command: ['node', 'ace', 'releases:merge']
|
||||
volumes: []
|
||||
|
||||
# The junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable);
|
||||
# verdicts hide them from search. No NNTP.
|
||||
# The junk judge: releases judged by what their probes showed (dead, fake, repeated, locked, unusable), the changed
|
||||
# ones every 2 minutes, all daily; verdicts hide them from search. No NNTP.
|
||||
judge:
|
||||
<<: *app
|
||||
command: ['node', 'ace', 'releases:judge']
|
||||
@@ -167,6 +175,11 @@ services:
|
||||
environment:
|
||||
COORDINATOR_URL: http://node-api:3335
|
||||
NODE_TOKEN: ${NODE_TOKEN}
|
||||
# The window worker (REBUILD.md design 1 and 5): a group per thread, the window's snapshot to the coordinator
|
||||
# this often; the coordinator keeps them under SNAPSHOT_DIR (the spool's snapshots/ on bulk by default).
|
||||
EDGE_SNAPSHOT_MINUTES: ${EDGE_SNAPSHOT_MINUTES:-30}
|
||||
# Of the threads, this many on the groups' tail lane (the live tail; the rest work the history lane).
|
||||
EDGE_TAIL_THREADS: ${EDGE_TAIL_THREADS:-1}
|
||||
stop_grace_period: 60s
|
||||
depends_on:
|
||||
node-api:
|
||||
@@ -184,6 +197,21 @@ services:
|
||||
node-api:
|
||||
condition: service_started
|
||||
|
||||
# The replay harness into the scratch catalog usenet_replay on bulk (REBUILD.md, "The replay harness"):
|
||||
# docker compose run --rm --no-deps replay --group alt.binaries.ath
|
||||
replay:
|
||||
<<: *app
|
||||
profiles: [replay]
|
||||
restart: 'no'
|
||||
entrypoint: ['node', 'ace', 'replay:corpus']
|
||||
command: ['--help']
|
||||
environment:
|
||||
<<: *app-env
|
||||
DB_DATABASE: usenet_replay
|
||||
volumes:
|
||||
- ./corpus:/corpus:ro
|
||||
depends_on: []
|
||||
|
||||
# One-off ace commands: docker compose run --rm ace releases:duplicates
|
||||
ace:
|
||||
<<: *app
|
||||
|
||||
+8
-1
@@ -51,7 +51,14 @@ if [ -e "$CACHE/paused" ]; then
|
||||
docker compose up -d --no-deps node-api web
|
||||
else
|
||||
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
||||
docker compose up -d --no-deps edge
|
||||
if [ -e "$CACHE/no-edge" ]; then
|
||||
# Voyager 3 (2026-10-26, LOG.md (116)): the worker indexes range by range, without the carried window, and
|
||||
# would split releases at every range edge with nothing to join them; history comes from the corpus through
|
||||
# deploy/replay-window.sh until the window worker is built. Remove the marker to start it again.
|
||||
echo "== no-edge: the edge worker left stopped"
|
||||
else
|
||||
docker compose up -d --no-deps edge
|
||||
fi
|
||||
fi
|
||||
echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
|
||||
# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
|
||||
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
|
||||
# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
|
||||
# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
|
||||
# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
|
||||
# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
|
||||
# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
|
||||
# from the deployment root:
|
||||
# setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
|
||||
# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
|
||||
# DB=usenet_rebuild replays into the catalog itself (voyager 3 deployed, 2026-10-26: history comes from the corpus
|
||||
# this way until the window worker is built); the harness then needs REPLAY_INTO_CATALOG=1, passed here. OUT keeps
|
||||
# a run's logs and markers apart (.cache/replay-catalog for the catalog's).
|
||||
set -u
|
||||
ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
|
||||
PRIORITY=${PRIORITY:-0}
|
||||
PARALLEL=${PARALLEL:-4}
|
||||
MEMORY=${MEMORY:-12g}
|
||||
IMAGE=${IMAGE:-usenet-voyager3:latest}
|
||||
DB=${DB:-usenet_replay}
|
||||
OUT=${OUT:-$ROOT/.cache/replay}
|
||||
mkdir -p "$OUT"
|
||||
cd "$ROOT" || exit 1
|
||||
|
||||
say() { echo "$(date -Is) $*"; }
|
||||
catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }
|
||||
|
||||
# The window's groups: name, first article, last article, whether every range is stored; smallest first.
|
||||
window() {
|
||||
catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
|
||||
FROM ranges r JOIN groups g ON g.id = r.group_id
|
||||
WHERE r.capture AND r.priority = $PRIORITY
|
||||
GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
|
||||
}
|
||||
|
||||
replay() {
|
||||
local group=$1 from=$2 to=$3
|
||||
say "$group: replaying $from–$to"
|
||||
if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
|
||||
-e NODE_ENV=production -e HOST=0.0.0.0 -e PORT=3334 -e APP_URL=http://web:3334 -e SPOOL_DIR=/spool -e CORPUS_DIR=/corpus \
|
||||
-e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE="$DB" -e CHUNK_TABLESPACE=pg_default -e REPLAY_INTO_CATALOG=1 \
|
||||
-v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
|
||||
node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
|
||||
touch "$OUT/$group.done"
|
||||
say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
|
||||
else
|
||||
touch "$OUT/$group.failed"
|
||||
say "$group: FAILED (see $OUT/$group.log)"
|
||||
fi
|
||||
rm -f "$OUT/$group.running"
|
||||
}
|
||||
|
||||
while true; do
|
||||
todo=0
|
||||
while read -r group from to complete; do
|
||||
[ -n "$group" ] || continue
|
||||
[ -e "$OUT/$group.done" ] && continue
|
||||
[ -e "$OUT/$group.failed" ] && continue
|
||||
todo=$((todo + 1))
|
||||
[ "$complete" = 1 ] || continue
|
||||
[ -e "$OUT/$group.running" ] && continue
|
||||
[ -e "$ROOT/.cache/paused" ] && continue
|
||||
while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
|
||||
touch "$OUT/$group.running"
|
||||
replay "$group" "$from" "$to" &
|
||||
sleep 5
|
||||
done <<< "$(window)"
|
||||
if [ "$todo" = 0 ]; then
|
||||
wait
|
||||
say "every group of the window at priority $PRIORITY is replayed"
|
||||
exit 0
|
||||
fi
|
||||
sleep 300
|
||||
done
|
||||
@@ -7,5 +7,10 @@ flock 9
|
||||
rm -f .cache/paused
|
||||
date "+%F %T resuming"
|
||||
docker compose up -d --no-deps coordinator prober merger judge spotter spot-inspector
|
||||
docker compose up -d --no-deps edge
|
||||
if [ -e .cache/no-edge ]; then
|
||||
# Voyager 3 without its window worker yet (deploy.sh, LOG.md (116)): the edge stays stopped.
|
||||
echo "no-edge: the edge worker left stopped"
|
||||
else
|
||||
docker compose up -d --no-deps edge
|
||||
fi
|
||||
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||
Generated
+10
@@ -44,6 +44,15 @@ version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
@@ -145,6 +154,7 @@ name = "edge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bincode",
|
||||
"chrono",
|
||||
"flate2",
|
||||
"libdeflater",
|
||||
|
||||
+2
-1
@@ -2,7 +2,7 @@
|
||||
name = "edge"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)"
|
||||
description = "The edge worker (edge-node) and its benchmark (edge-bench): voyager 3's window indexing in Rust"
|
||||
|
||||
[[bin]]
|
||||
name = "edge-bench"
|
||||
@@ -14,6 +14,7 @@ path = "src/bin/edge-node.rs"
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.22"
|
||||
bincode = "1.3"
|
||||
chrono = "0.4"
|
||||
flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
|
||||
libdeflater = "1.26.1"
|
||||
|
||||
+74
-18
@@ -6,18 +6,21 @@ from the node API, reads their prefetched XOVER chunks, finds releases and
|
||||
posts batches. `edge-bench` (`src/main.rs`) replays a captured range through
|
||||
the same indexing (`src/lib.rs`) for timing and for comparing.
|
||||
|
||||
The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the
|
||||
reference: `check.ts` runs both on a capture and compares every output field
|
||||
by field. An identity or rule change goes into both and has to come out
|
||||
identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test`
|
||||
and `check-fixtures.sh`, which compares both on the committed captures in
|
||||
`fixtures/` (25k overview lines each of a.b.models, a.b.ath and
|
||||
a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they
|
||||
match.
|
||||
Since voyager 3 (2026-10-25) the Rust indexer is the one implementation of
|
||||
the window (REBUILD.md, "Voyager 3: window indexing"; `src/indexer.rs`):
|
||||
files and releases are finished when the frontier (the median Date of the
|
||||
last 1,024 plausibly dated articles) is 24 h past their last part, or at
|
||||
the block's end (provisionally, as edges), and written once. The
|
||||
TypeScript `RangeIndexer` (`lib/edge`) keeps the range path for the corpus
|
||||
scanner and the tests; the field-by-field comparison (`check.ts`) went
|
||||
with the parity. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs
|
||||
`cargo test` and `bench-fixtures.sh`, which replays the committed captures
|
||||
in `fixtures/` (25k overview lines each of a.b.ath, a.b.boneless, a.b.fta
|
||||
and a.b.models, in 5 chunks).
|
||||
|
||||
```sh
|
||||
cargo test --release --manifest-path edge/Cargo.toml
|
||||
edge/check-fixtures.sh # from the repository root
|
||||
edge/bench-fixtures.sh # from the repository root
|
||||
```
|
||||
|
||||
In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores
|
||||
@@ -110,11 +113,30 @@ files, segments, payload size, candidates, fragments and samples. The
|
||||
summary can hide compensating differences, so compare the outputs
|
||||
themselves.
|
||||
|
||||
## Comparing the output
|
||||
## Replaying a group through one window
|
||||
|
||||
A corpus group directory (`<corpus>/<group>/<start>-<end>/`, each range with
|
||||
its `range.json`) is replayed in article order with the window carried from
|
||||
range to range, the end finished after the last range (REBUILD.md, "The
|
||||
replay harness"):
|
||||
|
||||
```sh
|
||||
./target/release/edge-bench --group /corpus/alt.binaries.ath --rules rules.json --batch-chunks 5 --wire - > batches.jsonl
|
||||
# --from <article> skips ranges ending before it, --to <article> ranges starting after it (one window of the
|
||||
# group's history), --limit <n> stops after n ranges, --no-finish leaves the window open
|
||||
```
|
||||
|
||||
`--rules` takes the lease's rules JSON (`src/lease.rs`: the catalog's rule
|
||||
set and group priorities, as `replay:corpus` writes it); without it the
|
||||
defaults with only the group indexed apply. `--wire -` writes each batch as
|
||||
the worker posts it to stdout, one JSON per line, and the report to stderr;
|
||||
`node ace replay:corpus` reads them into the batch store. Each range end
|
||||
prints the frontier, the releases out and the releases and files still open.
|
||||
|
||||
## The dump
|
||||
|
||||
```sh
|
||||
./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl
|
||||
cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl
|
||||
```
|
||||
|
||||
`--dump` writes every output of every batch as JSON lines:
|
||||
@@ -212,17 +234,45 @@ them:
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
|
||||
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--tail-threads T] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
```
|
||||
|
||||
- **One range per thread** (default one per core): one thread reads the
|
||||
coordinator's stream and inflates a frame ahead, another posts the
|
||||
batches in order (one may wait), the indexer goes on meanwhile.
|
||||
- **Two lanes** (2026-10-27): `--tail-threads T` (EDGE_TAIL_THREADS; one of
|
||||
two or more threads by default) puts that many threads on the groups'
|
||||
*tail* lane (`?lane=tail` on the group routes): the tail planner's
|
||||
ranges, through a window and snapshot of their own
|
||||
(`<group>-tail.gz`). The other threads work the history lane: the
|
||||
windows and the history the coordinator's bounds hand out. A tail thread
|
||||
never wades through a group's history first, and a held history group's
|
||||
tail isn't stuck behind its days of ranges; the coordinator orders each
|
||||
lane's leases by alternating the group with the most planned articles
|
||||
and the longest unheld one.
|
||||
|
||||
- **One group per thread** (default one per core; REBUILD.md, "Voyager 3:
|
||||
window indexing", 1 and 5): a thread leases a group (`POST
|
||||
groups/lease`), takes its ranges in article order (`POST
|
||||
groups/:id/next`), streams each range's chunks (`GET ranges/:id/stream`:
|
||||
the corpus or the spool, the worker can't tell) through one carried
|
||||
window, posts a batch at least at every range end and finishes the
|
||||
range; one thread reads the stream and inflates a frame ahead, another
|
||||
posts the batches in order, the indexer goes on meanwhile. The window's
|
||||
snapshot (bincode, gzip) goes to the coordinator every
|
||||
`--snapshot-minutes` (EDGE_SNAPSHOT_MINUTES, 30) right after a batch,
|
||||
and when the group is given back: caught up (`next` answers 204: the
|
||||
tail), a stop, a lost lease or a failure; a stop or a failed stream
|
||||
posts the open batch first, so the snapshot stands at its resume point
|
||||
(a batch the coordinator refused can't, and that window is lost); the
|
||||
next holder restores it (`GET groups/:id/snapshot`) and continues at
|
||||
its article. `--groups N`
|
||||
stops after N groups; `--ranges N` is the old per-range lease
|
||||
(`POST lease`), one range per thread without a carried window.
|
||||
- **As `commands/edge_node.ts`:** retries for network trouble and 5xx;
|
||||
409 means the lease is lost (the range stops, isn't given back); a
|
||||
batch once 500k articles or 100k files wait (a picture group's batches
|
||||
went over the server's 256 MB limit); a chunk that doesn't inflate is
|
||||
reported (the coordinator fetches it plain next time); SIGTERM/SIGINT
|
||||
reported (the coordinator fetches it plain next time) after the batch
|
||||
before it went out, and a held group asks for the range again from
|
||||
that chunk, its window kept; SIGTERM/SIGINT
|
||||
give each range back after its current chunk; `--max-mbit` is the
|
||||
node's download cap, shared by its threads. Plus a heartbeat when no
|
||||
batch went out for a minute.
|
||||
@@ -234,5 +284,11 @@ COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-nod
|
||||
same (releases with every column, chunks, candidates, rule counts,
|
||||
fragments with their JSON).
|
||||
- **On homelab:** compose service `edge` (deploy/compose.yaml), the
|
||||
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8);
|
||||
it talks to `node-api`.
|
||||
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8),
|
||||
EDGE_TAIL_THREADS (1); it talks to `node-api`.
|
||||
- **End to end:** `tests/e2e/edge_node.spec.ts` runs this binary
|
||||
(`edge/target/release/edge-node`, or EDGE_NODE_BIN) against the test
|
||||
server, database, spool and corpus: a chunk the corpus fill stores while
|
||||
the stream waits, a failed stream and the restart that restores the
|
||||
snapshot, a restart whose snapshot file is gone, a takeover. `node ace
|
||||
test e2e` after `cargo build --release`; CI builds the binary first.
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# The Rust indexer over the committed captures (edge/fixtures: 25k overview lines each of a.b.ath, a.b.boneless,
|
||||
# a.b.fta and a.b.models, in 5 chunks): a smoke test of the window on real headers, exiting non-zero when the bench
|
||||
# fails on one. CI runs it after cargo test; run it from the repository root after a change to the indexer.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
|
||||
for capture in edge/fixtures/*/; do
|
||||
echo "== $(basename "$capture")"
|
||||
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 | tail -1
|
||||
done
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/sh
|
||||
# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k
|
||||
# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared
|
||||
# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from
|
||||
# the repository root after any change to either side.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
|
||||
out=$(mktemp -d)
|
||||
trap 'rm -rf "$out"' EXIT
|
||||
status=0
|
||||
for capture in edge/fixtures/*/; do
|
||||
name=$(basename "$capture")
|
||||
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1
|
||||
echo "== $name"
|
||||
node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1
|
||||
done
|
||||
exit $status
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
/**
|
||||
* Compares edge-bench's output (--dump) with the TypeScript RangeIndexer
|
||||
* on the same captured range, item by item: releases (every field, and the
|
||||
* payload decoded with lib/chunks), samples, probe candidates, rule counts,
|
||||
* fragments and batch boundaries.
|
||||
*
|
||||
* node --import=@poppinss/ts-exec edge/check.ts <dir> <dump.jsonl> [--wire batches.jsonl] [--batch-chunks 10] [--idle 5]
|
||||
*
|
||||
* With --wire (edge-bench --wire): also each batch as the Rust worker
|
||||
* posts it, against toWire of the TypeScript batch, field by field.
|
||||
*/
|
||||
import { readFile, readdir } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { RangeIndexer, toWire, type EdgeBatch } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
|
||||
const [dir, dumpPath] = process.argv.slice(2)
|
||||
const flag = (name: string, fallback: number) => {
|
||||
const i = process.argv.indexOf(name)
|
||||
return i === -1 ? fallback : Number(process.argv[i + 1])
|
||||
}
|
||||
const batchChunks = flag('--batch-chunks', 10)
|
||||
const idle = flag('--idle', 5)
|
||||
|
||||
const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
|
||||
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
|
||||
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
|
||||
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
|
||||
const batches: EdgeBatch[] = []
|
||||
for (const [i, file] of files.entries()) {
|
||||
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
|
||||
indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
|
||||
const last = i === files.length - 1
|
||||
if (last) indexer.finish()
|
||||
if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
|
||||
}
|
||||
|
||||
/** Canonical lines per kind, in a stable order, from both sides. */
|
||||
type Item = Record<string, unknown>
|
||||
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
|
||||
const payloadBytes = { ts: 0, rs: 0 }
|
||||
let side: 'ts' | 'rs' = 'ts'
|
||||
const release = (r: Item, b: number) => {
|
||||
const { payload, ...rest } = r as Item & { payload: Buffer }
|
||||
// The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
|
||||
// differently); what the payload decodes to is.
|
||||
payloadBytes[side] += payload.length
|
||||
return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
|
||||
}
|
||||
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
batches.forEach((batch, b) => {
|
||||
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers, postedLow: batch.postedLow, postedHigh: batch.postedHigh })
|
||||
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
||||
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
||||
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
|
||||
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
|
||||
})
|
||||
|
||||
const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
side = 'rs'
|
||||
let b = -1
|
||||
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
|
||||
if (!line) continue
|
||||
const { t, ...item } = JSON.parse(line)
|
||||
if (t === 'b') b++
|
||||
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
|
||||
else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
|
||||
else rs[t].push({ b, ...item })
|
||||
}
|
||||
|
||||
const keyOf: Record<string, (i: Item) => string> = {
|
||||
b: (i) => String(i.b),
|
||||
r: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
s: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
|
||||
n: (i) => `${i.b} ${i.rule} ${i.key}`,
|
||||
f: (i) => `${i.b} ${i.fileKey}`,
|
||||
}
|
||||
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
|
||||
let failed = false
|
||||
for (const kind of Object.keys(keyOf)) {
|
||||
const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const missing = [...a.keys()].filter((k) => !z.has(k))
|
||||
const extra = [...z.keys()].filter((k) => !a.has(k))
|
||||
const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
|
||||
const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
|
||||
failed ||= !ok
|
||||
console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
|
||||
for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`)
|
||||
for (const k of differ.slice(0, 2)) {
|
||||
const x = a.get(k)!
|
||||
const y = z.get(k)!
|
||||
const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
|
||||
console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
|
||||
}
|
||||
}
|
||||
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
|
||||
|
||||
const wireIndex = process.argv.indexOf('--wire')
|
||||
if (wireIndex !== -1) {
|
||||
// Order within a batch doesn't count; payloads by what they decode to.
|
||||
const by = (f: (x: any) => string) => (a: any, b: any) => (f(a) < f(b) ? -1 : f(a) > f(b) ? 1 : 0)
|
||||
const normal = (w: any) => ({
|
||||
...w,
|
||||
releases: w.releases.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
samples: w.samples.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
candidates: [...w.candidates].sort(by((c) => JSON.stringify(c))),
|
||||
counts: [...w.counts].sort(by((c) => `${c.rule} ${c.key}`)),
|
||||
fragments: [...w.fragments].sort(by((f) => `${f.fileKey} ${f.segments[0]?.[1]}`)),
|
||||
})
|
||||
const rust = (await readFile(process.argv[wireIndex + 1], 'utf8')).split('\n').filter(Boolean).map((l) => normal(JSON.parse(l)))
|
||||
const typescript = batches.map((b) => normal(JSON.parse(JSON.stringify(toWire(b)))))
|
||||
let wireSame = rust.length === typescript.length
|
||||
const firstDiff = (x: any, y: any, path: string): string | null => {
|
||||
if (isDeepStrictEqual(x, y)) return null
|
||||
if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null) return `${path}: ${JSON.stringify(x)?.slice(0, 120)} vs ${JSON.stringify(y)?.slice(0, 120)}`
|
||||
for (const k of new Set([...Object.keys(x), ...Object.keys(y)])) {
|
||||
const d = firstDiff(x[k], y[k], `${path}.${k}`)
|
||||
if (d) return d
|
||||
}
|
||||
return path
|
||||
}
|
||||
for (let i = 0; i < Math.max(rust.length, typescript.length); i++) {
|
||||
const d = firstDiff(typescript[i], rust[i], `batch ${i}`)
|
||||
if (d) {
|
||||
wireSame = false
|
||||
console.log(`DIFF wire: ${d}`)
|
||||
break
|
||||
}
|
||||
}
|
||||
if (wireSame) console.log(`same wire batches: ${typescript.length} TypeScript, ${rust.length} Rust`)
|
||||
failed ||= !wireSame
|
||||
}
|
||||
process.exit(failed ? 1 : 0)
|
||||
+396
-72
@@ -1,16 +1,24 @@
|
||||
//! The edge worker in Rust (REBUILD.md, part 2; commands/edge_node.ts): leases
|
||||
//! ranges from the coordinator, reads their XOVER chunks from its stream,
|
||||
//! indexes them (the same RangeIndexer as edge-bench, kept identical to
|
||||
//! the TypeScript by check.ts) and posts the batches. One range per thread.
|
||||
//! The window worker in Rust (REBUILD.md, "Voyager 3: window indexing", 1 and 5): each thread leases a *group*
|
||||
//! from the coordinator, takes its ranges in article order, reads each range's XOVER chunks from the coordinator's
|
||||
//! stream (the corpus or the spool, the worker can't tell), indexes them through one carried window and posts the
|
||||
//! batches, a batch at least at every range end; the window's snapshot goes to the coordinator every
|
||||
//! --snapshot-minutes and when the group is given back, and the next holder restores it. A group with no range
|
||||
//! left (caught up: the tail) is given back and leased again later.
|
||||
//!
|
||||
//! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
|
||||
//! COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--tail-threads T] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
|
||||
//!
|
||||
//! SIGTERM/SIGINT: each thread gives its range back after the current chunk
|
||||
//! (the next worker resumes at the last stored batch).
|
||||
//! `--tail-threads T` (EDGE_TAIL_THREADS; 1 of 2 or more threads by default): that many of the threads lease the
|
||||
//! groups' *tail* lane (`?lane=tail`: the tail planner's ranges, with a window and snapshot of their own), the rest
|
||||
//! the history lane (the windows and the history the coordinator's bounds hand out).
|
||||
//!
|
||||
//! SIGTERM/SIGINT: each thread finishes the chunk, posts its batch, snapshots and gives its group back.
|
||||
//! `--ranges`: the old range leases (POST lease), one range per thread without a carried window (kept for a
|
||||
//! coordinator without group leases).
|
||||
|
||||
use edge::indexer::{Batch, RangeIndexer};
|
||||
use edge::names::Rules;
|
||||
use edge::lease::{GroupLeaseJson, LeaseJson, NextRangeJson, RangeSpanJson};
|
||||
use edge::wire;
|
||||
use flate2::read::GzDecoder;
|
||||
use flate2::write::GzEncoder;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
@@ -34,63 +42,6 @@ fn log(level: &str, message: &str) {
|
||||
println!("[ {level} ] {message}");
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LeaseJson {
|
||||
range: RangeJson,
|
||||
rules: RulesJson,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RangeJson {
|
||||
id: i64,
|
||||
group: String,
|
||||
start: i64,
|
||||
end: i64,
|
||||
next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RulesJson {
|
||||
version: i64,
|
||||
priorities: HashMap<String, u32>,
|
||||
#[serde(default)]
|
||||
dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
per_article_max_parts: Option<u32>,
|
||||
#[serde(default)]
|
||||
sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
dropped_posters: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
fn rules(self) -> Rules {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
if let Some(n) = self.per_article_max_parts {
|
||||
rules.per_article_max_parts = n;
|
||||
}
|
||||
if let Some(p) = self.sample_percent {
|
||||
rules.sample_percent = p;
|
||||
}
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StoredJson {
|
||||
@@ -220,6 +171,81 @@ impl Api {
|
||||
}
|
||||
}
|
||||
|
||||
// Group leases (REBUILD.md design 1 and 5), per lane (`history` or `tail`).
|
||||
|
||||
fn lease_group(&self, lane: &str) -> Result<Option<GroupLeaseJson>, ApiError> {
|
||||
let mut response = self.request("POST", &format!("groups/lease?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
204 => Ok(None),
|
||||
200 => {
|
||||
let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("groups/lease: {e}")))?;
|
||||
serde_json::from_str(&text).map(Some).map_err(|e| ApiError::Other(format!("groups/lease: {e}")))
|
||||
}
|
||||
status => Err(ApiError::Other(format!("groups/lease: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn next_range(&self, group: i64, lane: &str) -> Result<Option<RangeSpanJson>, ApiError> {
|
||||
let mut response = self.request("POST", &format!("groups/{group}/next?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
204 => Ok(None),
|
||||
200 => {
|
||||
let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("next: {e}")))?;
|
||||
serde_json::from_str::<NextRangeJson>(&text).map(|n| Some(n.range)).map_err(|e| ApiError::Other(format!("next: {e}")))
|
||||
}
|
||||
status => Err(ApiError::Other(format!("next: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn group_call(&self, group: i64, lane: &str, action: &str) -> Result<(), ApiError> {
|
||||
let response = self.request("POST", &format!("groups/{group}/{action}?lane={lane}"), None, false, "application/json")?;
|
||||
match response.status().as_u16() {
|
||||
200..=299 => Ok(()),
|
||||
status => Err(ApiError::Other(format!("{action}: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The window's snapshot (bincode), gzipped, standing at `next`.
|
||||
fn put_snapshot(&self, group: i64, lane: &str, next: i64, snapshot: &[u8]) -> Result<usize, ApiError> {
|
||||
let mut gz = GzEncoder::new(Vec::with_capacity(snapshot.len() / 3), flate2::Compression::new(1));
|
||||
gz.write_all(snapshot).unwrap();
|
||||
let body = gz.finish().unwrap();
|
||||
let url = format!("{}/api/v1/node/groups/{group}/snapshot?next={next}&lane={lane}", self.base);
|
||||
let auth = format!("Bearer {}", self.token);
|
||||
for attempt in 1..=RETRIES {
|
||||
let result = self.agent.put(&url).header("Authorization", &auth).header("Content-Type", "application/octet-stream").send(&body[..]);
|
||||
match result {
|
||||
Ok(response) => match response.status().as_u16() {
|
||||
200..=299 => return Ok(body.len()),
|
||||
409 => return Err(ApiError::LeaseLost("snapshot: the group is no longer this node's".into())),
|
||||
status if status < 500 => return Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
|
||||
status if attempt == RETRIES => return Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
|
||||
_ => {}
|
||||
},
|
||||
Err(error) if attempt == RETRIES => return Err(ApiError::Other(format!("snapshot: {error}"))),
|
||||
Err(_) => {}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(60_000.min(1000 * 2u64.pow(attempt))));
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
/// The group's snapshot: its bytes (inflated) and the article it stands at; None without one.
|
||||
fn get_snapshot(&self, group: i64, lane: &str) -> Result<Option<(i64, Vec<u8>)>, ApiError> {
|
||||
let response = self.request("GET", &format!("groups/{group}/snapshot?lane={lane}"), None, false, "application/octet-stream")?;
|
||||
match response.status().as_u16() {
|
||||
404 => Ok(None),
|
||||
200 => {
|
||||
let next: i64 = response.headers().get("x-snapshot-next").and_then(|v| v.to_str().ok()).and_then(|v| v.parse().ok()).ok_or_else(|| ApiError::Other("snapshot: no X-Snapshot-Next".into()))?;
|
||||
let mut gz = GzDecoder::new(response.into_body().into_reader());
|
||||
let mut bytes = Vec::new();
|
||||
gz.read_to_end(&mut bytes).map_err(|e| ApiError::Other(format!("snapshot: {e}")))?;
|
||||
Ok(Some((next, bytes)))
|
||||
}
|
||||
status => Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn corrupt(&self, range: i64, from: i64) -> Result<(), ApiError> {
|
||||
let body = format!("{{\"from\":{from}}}");
|
||||
let response = self.request("POST", &format!("ranges/{range}/corrupt"), Some(body.as_bytes()), false, "application/json")?;
|
||||
@@ -332,6 +358,8 @@ fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Result<Vec<u8>, Str
|
||||
struct Config {
|
||||
batch_articles: i64,
|
||||
bytes_per_second: Option<f64>,
|
||||
/// Between two snapshots of a held group's window.
|
||||
snapshot_every: Duration,
|
||||
/// Ranges whose batch was too large (413), and how much smaller their batches are now (halved each time).
|
||||
smaller: Mutex<HashMap<i64, i64>>,
|
||||
}
|
||||
@@ -509,6 +537,272 @@ fn run_range(api: &Arc<Api>, lease: LeaseJson, config: &Config, stopping: &Atomi
|
||||
}
|
||||
}
|
||||
|
||||
/// What one range of a held group came to.
|
||||
enum RangeOutcome {
|
||||
Done,
|
||||
/// A chunk didn't inflate: reported (the coordinator fetches it again, plain the second time) after the batch
|
||||
/// before it went out, and the range is asked for again from there; the window stays.
|
||||
Again,
|
||||
/// The group is given back (a stop, a lost lease, a failure): no more ranges.
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// One range of a held group through the carried window: the chunks streamed, batches posted at the range's end at
|
||||
/// least, the range finished. The indexer keeps the window for the next range. `group_name` is the label for the
|
||||
/// log (the tail lane's says so).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn run_group_range(api: &Arc<Api>, group_id: i64, lane: &str, group_name: &str, range: RangeSpanJson, indexer: &mut RangeIndexer, config: &Config, stopping: &AtomicBool, totals: &Arc<Mutex<Totals>>) -> RangeOutcome {
|
||||
let started = Instant::now();
|
||||
indexer.begin_range(range.next, range.end);
|
||||
let post_error: Arc<Mutex<Option<ApiError>>> = Arc::new(Mutex::new(None));
|
||||
let last_contact = Arc::new(AtomicI64::new(0));
|
||||
let since = Instant::now();
|
||||
let (post_tx, post_rx) = sync_channel::<Batch>(1);
|
||||
let poster = {
|
||||
let (api, totals, post_error, last_contact) = (api.clone(), totals.clone(), post_error.clone(), last_contact.clone());
|
||||
std::thread::spawn(move || {
|
||||
for batch in post_rx {
|
||||
if post_error.lock().unwrap().is_some() {
|
||||
continue;
|
||||
}
|
||||
let at = Instant::now();
|
||||
match api.batch(range.id, &batch) {
|
||||
Ok((releases, candidates, bytes)) => {
|
||||
let mut t = totals.lock().unwrap();
|
||||
t.post_ms += at.elapsed().as_millis();
|
||||
t.bytes += bytes;
|
||||
t.headers += batch.headers;
|
||||
t.releases += releases;
|
||||
t.candidates += candidates;
|
||||
last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
|
||||
}
|
||||
Err(e) => *post_error.lock().unwrap() = Some(e),
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
let frames = match api.stream(range.id) {
|
||||
Ok(stream) => read_frames(stream, config.bytes_per_second),
|
||||
Err(e) => {
|
||||
log("error", &format!("{group_name} {}–{}: stream failed: {e}; the group goes back.", range.start, range.end));
|
||||
drop(post_tx);
|
||||
let _ = poster.join();
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
};
|
||||
let (batch_articles, max_pending) = config.limits(range.id);
|
||||
let mut batch_from = range.next;
|
||||
let mut reached_end = false;
|
||||
let mut corrupt: Option<(i64, i64, String)> = None;
|
||||
let mut failure: Option<String> = None;
|
||||
let headers_before = totals.lock().unwrap().headers;
|
||||
for frame in frames.iter() {
|
||||
if stopping.load(Ordering::Relaxed) || post_error.lock().unwrap().is_some() {
|
||||
break;
|
||||
}
|
||||
if since.elapsed().as_millis() as i64 - last_contact.load(Ordering::Relaxed) > HEARTBEAT.as_millis() as i64 {
|
||||
last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
|
||||
let _ = api.group_call(group_id, lane, "heartbeat");
|
||||
}
|
||||
match frame {
|
||||
Frame::KeepAlive => continue,
|
||||
Frame::End => break,
|
||||
Frame::Failed(e) => {
|
||||
failure = Some(e);
|
||||
break;
|
||||
}
|
||||
Frame::Chunk { from, to, text } => {
|
||||
let data = match text {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
corrupt = Some((from, to, e));
|
||||
break;
|
||||
}
|
||||
};
|
||||
indexer.add_text(&String::from_utf8_lossy(&data), to + 1);
|
||||
reached_end = to >= range.end;
|
||||
// The window carries on past the range's end: no finish here; a batch at the end at least.
|
||||
if reached_end || to + 1 - batch_from >= batch_articles || indexer.pending() >= max_pending {
|
||||
if post_tx.send(indexer.take()).is_err() {
|
||||
break;
|
||||
}
|
||||
batch_from = to + 1;
|
||||
}
|
||||
if reached_end {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Stopped mid-range (a stop, a stream that failed, a chunk that doesn't inflate): the batch so far goes out,
|
||||
// so the snapshot stands at a stored resume point and the window survives the group going back; a chunk
|
||||
// reported corrupt is read again from its own start. Only a batch the coordinator refused stays open.
|
||||
if !reached_end && post_error.lock().unwrap().is_none() && indexer.next() > batch_from {
|
||||
let _ = post_tx.send(indexer.take());
|
||||
}
|
||||
drop(post_tx);
|
||||
let _ = poster.join();
|
||||
let post_error = post_error.lock().unwrap().take();
|
||||
if let Some((from, to, e)) = corrupt {
|
||||
// The server's compressed reply cut short (LOG.md 2026-10-11): the coordinator fetches the chunk again,
|
||||
// plain the second time, and sets the range aside after that; the window carries on either way.
|
||||
log("warn", &format!("{group_name} {}–{}: chunk {from}–{to} doesn't inflate ({e}); reported, the range is asked for again", range.start, range.end));
|
||||
if let Err(e) = api.corrupt(range.id, from) {
|
||||
log("warn", &format!("{group_name} {}–{}: reporting the chunk: {e}; the group goes back", range.start, range.end));
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
if post_error.is_none() {
|
||||
return RangeOutcome::Again;
|
||||
}
|
||||
}
|
||||
match post_error {
|
||||
Some(ApiError::LeaseLost(m)) => {
|
||||
log("warn", &format!("{group_name} {}–{}: {m}", range.start, range.end));
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
Some(ApiError::TooLarge(m)) => {
|
||||
let divisor = {
|
||||
let mut smaller = config.smaller.lock().unwrap();
|
||||
let d = smaller.entry(range.id).or_insert(1);
|
||||
*d = (*d * 2).min(64);
|
||||
*d
|
||||
};
|
||||
log("warn", &format!("{group_name} {}–{}: {m}; the group goes back, its batches 1/{divisor} the size from now on.", range.start, range.end));
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
Some(ApiError::Other(m)) => {
|
||||
log("error", &format!("{group_name} {}–{}: {m}; the group goes back.", range.start, range.end));
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if let Some(e) = failure {
|
||||
log("error", &format!("{group_name} {}–{}: {e}; the group goes back.", range.start, range.end));
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
if !reached_end {
|
||||
return RangeOutcome::Stop;
|
||||
}
|
||||
match api.call(range.id, "finish") {
|
||||
Ok(()) => {
|
||||
let t = totals.lock().unwrap();
|
||||
let seconds = started.elapsed().as_secs_f64();
|
||||
let headers = t.headers - headers_before;
|
||||
log(
|
||||
"info",
|
||||
&format!(
|
||||
"{group_name} {}–{}: {} headers in {:.0} s ({:.0}/s); {} releases and {} files open in the window.",
|
||||
range.start,
|
||||
range.end,
|
||||
headers,
|
||||
seconds,
|
||||
headers as f64 / seconds.max(0.001),
|
||||
indexer.open_releases(),
|
||||
indexer.open_files()
|
||||
),
|
||||
);
|
||||
RangeOutcome::Done
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name} {}–{}: finish: {e}", range.start, range.end));
|
||||
RangeOutcome::Stop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The window's snapshot to the coordinator, standing at the indexer's next article.
|
||||
fn snapshot(api: &Api, group_id: i64, lane: &str, group_name: &str, indexer: &RangeIndexer) {
|
||||
match indexer.snapshot() {
|
||||
Ok(bytes) => match api.put_snapshot(group_id, lane, indexer.next(), &bytes) {
|
||||
Ok(gz) => log("info", &format!("{group_name}: snapshot at {} ({:.1} MB, {:.1} MB gzipped).", indexer.next(), bytes.len() as f64 / 1e6, gz as f64 / 1e6)),
|
||||
Err(e) => log("warn", &format!("{group_name}: snapshot failed: {e}")),
|
||||
},
|
||||
Err(e) => log("warn", &format!("{group_name}: no snapshot: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// One held group (one lane of it): its window restored from the snapshot (or new), its ranges one after another
|
||||
/// until it is caught up or the worker stops; then a snapshot, and the group given back.
|
||||
fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, lane: &str, config: &Config, stopping: &AtomicBool) {
|
||||
let group_id = lease.group.id;
|
||||
let name = lease.group.name.clone();
|
||||
// The log's label; the indexer gets the group's name itself (keys, xrefs).
|
||||
let group_name = if lane == "tail" { format!("{name} (tail)") } else { name.clone() };
|
||||
let rules = lease.rules.rules();
|
||||
let mut indexer = match lease.snapshot {
|
||||
Some(s) => match api.get_snapshot(group_id, lane) {
|
||||
Ok(Some((next, bytes))) => match RangeIndexer::restore(&bytes, rules.clone(), 5) {
|
||||
Ok(ix) => {
|
||||
log("info", &format!("{group_name}: window restored from the snapshot at {next} ({:.1} MB).", bytes.len() as f64 / 1e6));
|
||||
ix
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name}: the snapshot at {} can't be restored ({e}); the window starts empty.", s.next));
|
||||
RangeIndexer::for_group(&name, rules.clone(), 5)
|
||||
}
|
||||
},
|
||||
Ok(None) => {
|
||||
log("warn", &format!("{group_name}: the lease names a snapshot at {} but there is none; the window starts empty.", s.next));
|
||||
RangeIndexer::for_group(&name, rules.clone(), 5)
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name}: snapshot: {e}; the group goes back."));
|
||||
let _ = api.group_call(group_id, lane, "release");
|
||||
return;
|
||||
}
|
||||
},
|
||||
None => RangeIndexer::for_group(&name, rules.clone(), 5),
|
||||
};
|
||||
let totals = Arc::new(Mutex::new(Totals::default()));
|
||||
let started = Instant::now();
|
||||
let mut snapshotted = Instant::now();
|
||||
let mut ranges = 0;
|
||||
loop {
|
||||
if stopping.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let range = match api.next_range(group_id, lane) {
|
||||
Ok(Some(range)) => range,
|
||||
Ok(None) => {
|
||||
log("info", &format!("{group_name}: caught up after {ranges} ranges; the group goes back."));
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
log("warn", &format!("{group_name}: next range: {e}; the group goes back."));
|
||||
break;
|
||||
}
|
||||
};
|
||||
if range.next != indexer.next() && ranges > 0 {
|
||||
log("warn", &format!("{group_name}: the window stands at {} but the next range resumes at {}: the window carries on from there.", indexer.next(), range.next));
|
||||
}
|
||||
match run_group_range(api, group_id, lane, &group_name, range, &mut indexer, config, stopping, &totals) {
|
||||
RangeOutcome::Done => ranges += 1,
|
||||
// The same range again from the reported chunk: the stream waits for its new copy.
|
||||
RangeOutcome::Again => {}
|
||||
RangeOutcome::Stop => break,
|
||||
}
|
||||
if snapshotted.elapsed() >= config.snapshot_every {
|
||||
snapshot(api, group_id, lane, &group_name, &indexer);
|
||||
snapshotted = Instant::now();
|
||||
}
|
||||
}
|
||||
snapshot(api, group_id, lane, &group_name, &indexer);
|
||||
let _ = api.group_call(group_id, lane, "release");
|
||||
let t = totals.lock().unwrap();
|
||||
log(
|
||||
"info",
|
||||
&format!(
|
||||
"{group_name}: {ranges} ranges, {} headers in {:.0} s, {} release chunks, {} probe candidates; posted {:.1} MB in {:.1} s.",
|
||||
t.headers,
|
||||
started.elapsed().as_secs_f64(),
|
||||
t.releases,
|
||||
t.candidates,
|
||||
t.bytes as f64 / 1e6,
|
||||
t.post_ms as f64 / 1000.0
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
@@ -520,9 +814,16 @@ fn main() {
|
||||
.filter(|&n| n > 0)
|
||||
.unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1));
|
||||
let ranges: Option<usize> = flag("--ranges").map(|v| v.parse().unwrap());
|
||||
// Of the threads, this many on the groups' tail lane: one by default when there are two or more.
|
||||
let tail_threads: usize = flag("--tail-threads")
|
||||
.or_else(|| std::env::var("EDGE_TAIL_THREADS").ok())
|
||||
.map(|v| v.parse().unwrap())
|
||||
.unwrap_or(if threads >= 2 { 1 } else { 0 });
|
||||
assert!(tail_threads <= threads, "--tail-threads {tail_threads} is more than the {threads} threads");
|
||||
let max_mbit: Option<f64> = flag("--max-mbit").or_else(|| std::env::var("EDGE_MAX_MBIT").ok()).map(|v| v.parse().unwrap()).filter(|&m: &f64| m > 0.0);
|
||||
let config = Arc::new(Config {
|
||||
batch_articles: flag("--batch-articles").map(|v| v.parse().unwrap()).unwrap_or(500_000),
|
||||
snapshot_every: Duration::from_secs(60 * flag("--snapshot-minutes").or_else(|| std::env::var("EDGE_SNAPSHOT_MINUTES").ok()).map(|v| v.parse::<u64>().unwrap()).unwrap_or(30)),
|
||||
// The node's cap, shared by its threads.
|
||||
bytes_per_second: max_mbit.map(|m| m * 1e6 / 8.0 / threads as f64),
|
||||
smaller: Mutex::new(HashMap::new()),
|
||||
@@ -531,22 +832,41 @@ fn main() {
|
||||
signal_hook::flag::register(signal_hook::consts::SIGTERM, stopping.clone()).unwrap();
|
||||
signal_hook::flag::register(signal_hook::consts::SIGINT, stopping.clone()).unwrap();
|
||||
let api = Arc::new(Api::new(&url, &token));
|
||||
log("info", &format!("Starting {threads} worker threads{}.", max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
|
||||
log("info", &format!("Starting {threads} worker threads{}{}.", if range_leases_wanted(&ranges) { String::new() } else { format!(", {tail_threads} on the tail") }, max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
|
||||
let groups: Option<usize> = flag("--groups").map(|v| v.parse().unwrap());
|
||||
let range_leases = ranges.is_some();
|
||||
let workers: Vec<_> = (0..threads)
|
||||
.map(|_| {
|
||||
.map(|i| {
|
||||
let (api, config, stopping) = (api.clone(), config.clone(), stopping.clone());
|
||||
let lane = if i < tail_threads { "tail" } else { "history" };
|
||||
std::thread::spawn(move || {
|
||||
let mut done = 0;
|
||||
while !stopping.load(Ordering::Relaxed) && ranges.is_none_or(|n| done < n) {
|
||||
match api.lease() {
|
||||
if range_leases {
|
||||
while !stopping.load(Ordering::Relaxed) && ranges.is_none_or(|n| done < n) {
|
||||
match api.lease() {
|
||||
Ok(Some(lease)) => {
|
||||
run_range(&api, lease, &config, &stopping);
|
||||
done += 1;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
log("warn", &format!("Lease failed: {e}"));
|
||||
idle(&stopping);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
while !stopping.load(Ordering::Relaxed) && groups.is_none_or(|n| done < n) {
|
||||
match api.lease_group(lane) {
|
||||
Ok(Some(lease)) => {
|
||||
run_range(&api, lease, &config, &stopping);
|
||||
run_group(&api, lease, lane, &config, &stopping);
|
||||
done += 1;
|
||||
}
|
||||
Ok(None) if ranges.is_some() => break,
|
||||
Ok(None) if groups.is_some() => break,
|
||||
Ok(None) => idle(&stopping),
|
||||
Err(e) => {
|
||||
log("warn", &format!("Lease failed: {e}"));
|
||||
log("warn", &format!("Group lease failed: {e}"));
|
||||
idle(&stopping);
|
||||
}
|
||||
}
|
||||
@@ -559,6 +879,10 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
fn range_leases_wanted(ranges: &Option<usize>) -> bool {
|
||||
ranges.is_some()
|
||||
}
|
||||
|
||||
fn idle(stopping: &AtomicBool) {
|
||||
let until = Instant::now() + IDLE;
|
||||
while Instant::now() < until && !stopping.load(Ordering::Relaxed) {
|
||||
|
||||
+1064
-146
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,99 @@
|
||||
//! A lease as the coordinator sends it (lib/edge/lease.ts): the range and the rules, JSON with the TypeScript
|
||||
//! names. The edge-node worker takes one per range; edge-bench's replay takes the rules alone (`--rules`).
|
||||
|
||||
use crate::names::Rules;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct LeaseJson {
|
||||
pub range: RangeJson,
|
||||
pub rules: RulesJson,
|
||||
}
|
||||
|
||||
/// A group lease (lib/edge/lease.ts GroupLease): the group a worker thread holds, the rules, the window's snapshot if any.
|
||||
#[derive(Deserialize)]
|
||||
pub struct GroupLeaseJson {
|
||||
pub group: GroupJson,
|
||||
pub rules: RulesJson,
|
||||
pub snapshot: Option<SnapshotJson>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GroupJson {
|
||||
pub id: i64,
|
||||
pub name: String,
|
||||
pub priority: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SnapshotJson {
|
||||
pub next: i64,
|
||||
}
|
||||
|
||||
/// The group's next range (lib/edge/lease.ts NextRange).
|
||||
#[derive(Deserialize)]
|
||||
pub struct NextRangeJson {
|
||||
pub range: RangeSpanJson,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone, Copy)]
|
||||
pub struct RangeSpanJson {
|
||||
pub id: i64,
|
||||
pub start: i64,
|
||||
pub end: i64,
|
||||
pub next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct RangeJson {
|
||||
pub id: i64,
|
||||
pub group: String,
|
||||
pub start: i64,
|
||||
pub end: i64,
|
||||
pub next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RulesJson {
|
||||
pub version: i64,
|
||||
pub priorities: HashMap<String, u32>,
|
||||
#[serde(default)]
|
||||
pub dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub per_article_max_parts: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub dropped_posters: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
/// The rules with today's defaults under them (a field the stored set doesn't have keeps the default).
|
||||
pub fn rules(self) -> Rules {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
if let Some(n) = self.per_article_max_parts {
|
||||
rules.per_article_max_parts = n;
|
||||
}
|
||||
if let Some(p) = self.sample_percent {
|
||||
rules.sample_percent = p;
|
||||
}
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
@@ -5,5 +5,6 @@
|
||||
pub mod codec;
|
||||
pub mod indexer;
|
||||
pub mod js;
|
||||
pub mod lease;
|
||||
pub mod names;
|
||||
pub mod wire;
|
||||
+192
-90
@@ -1,10 +1,11 @@
|
||||
//! The edge worker's indexing in Rust, on a range captured by `edge:bench
|
||||
//! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as
|
||||
//! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off
|
||||
//! idle ones, and every N chunks judge and encode the releases. `--dump`
|
||||
//! writes every output as JSON lines for edge/check.ts.
|
||||
//! The Rust indexer over stored chunks (REBUILD.md, part 2 and "Voyager 3"): a range captured by `edge:bench
|
||||
//! --capture` or stored in the corpus, or a group's corpus ranges in order through one window (`--group`, the
|
||||
//! replay harness behind `replay:corpus`). Same work as the edge-node worker: inflate each chunk, scan it,
|
||||
//! aggregate files, finish releases through the window, and every N chunks judge and encode them. `--dump`
|
||||
//! writes every output as JSON lines; `--wire` the batches as the worker posts them (`-`: stdout).
|
||||
//!
|
||||
//! edge-bench <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl]
|
||||
//! edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json]
|
||||
//! [--batch-chunks 10] [--idle 5] [--no-finish] [--dump out.jsonl] [--wire batches.jsonl|-] [--releases out.jsonl]
|
||||
|
||||
use base64::Engine;
|
||||
use edge::indexer::{self, Batch, RangeIndexer};
|
||||
@@ -123,7 +124,7 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(",\"postedLast\":");
|
||||
opt_num(r.posted_last, line);
|
||||
line.push_str(&format!(
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"finished\":{},\"edge\":{},\"partial\":{},\"payload\":\"{}\"",
|
||||
r.files,
|
||||
r.segments,
|
||||
r.bytes,
|
||||
@@ -131,6 +132,9 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
r.last_article,
|
||||
r.readable,
|
||||
r.has_par2,
|
||||
r.finished,
|
||||
r.edge,
|
||||
r.partial,
|
||||
b64.encode(&r.payload)
|
||||
));
|
||||
};
|
||||
@@ -162,122 +166,220 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
|
||||
emit(&mut line);
|
||||
}
|
||||
for f in &batch.fragments {
|
||||
line.push_str("{\"t\":\"f\",\"fileKey\":");
|
||||
json_str(&f.file_key, &mut line);
|
||||
line.push_str(",\"name\":");
|
||||
json_str(&f.name, &mut line);
|
||||
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
|
||||
opt_num(f.posted_at, &mut line);
|
||||
line.push_str(",\"segments\":[");
|
||||
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
|
||||
if i > 0 {
|
||||
line.push(',');
|
||||
}
|
||||
line.push_str(&format!("[{n},"));
|
||||
json_str(id, &mut line);
|
||||
line.push_str(&format!(",{bytes},{article}]"));
|
||||
}
|
||||
|
||||
/// A stored range: its directory and article span (range.json).
|
||||
struct Range {
|
||||
dir: String,
|
||||
start: i64,
|
||||
end: i64,
|
||||
}
|
||||
|
||||
/// The range in a directory with a range.json, and its group.
|
||||
fn range_in(dir: &str) -> Option<(String, Range)> {
|
||||
let json = std::fs::read_to_string(format!("{dir}/range.json")).ok()?;
|
||||
Some((json_field(&json, "group"), Range { dir: dir.to_string(), start: json_field(&json, "start").parse().ok()?, end: json_field(&json, "end").parse().ok()? }))
|
||||
}
|
||||
|
||||
/// A group directory's complete ranges in article order (the corpus layout: `<group>/<start>-<end>/`).
|
||||
fn ranges_in(group_dir: &str) -> (String, Vec<Range>) {
|
||||
let mut group = String::new();
|
||||
let mut ranges: Vec<Range> = Vec::new();
|
||||
for entry in std::fs::read_dir(group_dir).unwrap_or_else(|e| panic!("{group_dir}: {e}")) {
|
||||
let path = entry.unwrap().path();
|
||||
if !path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
line.push_str("]}");
|
||||
emit(&mut line);
|
||||
if let Some((g, range)) = range_in(path.to_str().unwrap()) {
|
||||
group = g;
|
||||
ranges.push(range);
|
||||
}
|
||||
}
|
||||
ranges.sort_by_key(|r| r.start);
|
||||
(group, ranges)
|
||||
}
|
||||
|
||||
fn date(ms: Option<i64>) -> String {
|
||||
match ms.and_then(chrono::DateTime::from_timestamp_millis) {
|
||||
Some(t) => t.format("%Y-%m-%d %H:%M").to_string(),
|
||||
None => "-".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Files waiting in finished releases after which a batch goes out whatever its chunks (edge-node's limit).
|
||||
const MAX_PENDING_FILES: usize = 100_000;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let dir = args.get(1).expect("usage: edge-bench <dir> [--batch-chunks N] [--idle N] [--dump file]").clone();
|
||||
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
let has = |name: &str| args.iter().any(|a| a == name);
|
||||
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
|
||||
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
|
||||
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: a file, or stdout (`-`) for replay:corpus.
|
||||
let wire_path = flag("--wire");
|
||||
let wire_stdout = wire_path.as_deref() == Some("-");
|
||||
let mut wire_out: Option<Box<dyn Write>> = match wire_path.as_deref() {
|
||||
Some("-") => Some(Box::new(std::io::BufWriter::with_capacity(1 << 20, std::io::stdout().lock()))),
|
||||
Some(p) => Some(Box::new(std::io::BufWriter::new(std::fs::File::create(p).unwrap()))),
|
||||
None => None,
|
||||
};
|
||||
// One line per release chunk without its payload, with its first subjects (deploy/corpus_sweep.py).
|
||||
let mut releases_out = flag("--releases").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
|
||||
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
|
||||
let group = json_field(&range, "group");
|
||||
let start: i64 = json_field(&range, "start").parse().unwrap();
|
||||
let end: i64 = json_field(&range, "end").parse().unwrap();
|
||||
let mut files: Vec<String> = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||
files.sort();
|
||||
// One range, or a group's ranges in order through one window (the window carries over from range to range;
|
||||
// the end is only finished after the last one, unless --no-finish: a replay to be continued).
|
||||
let (group, ranges) = match flag("--group") {
|
||||
Some(dir) => {
|
||||
let (group, mut ranges) = ranges_in(&dir);
|
||||
// --from: ranges ending before it are skipped (a replay continued); --to: ranges starting after it (one
|
||||
// window of the group's history, the corpus holding others); --limit: this many at most.
|
||||
if let Some(from) = flag("--from").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.end >= from);
|
||||
}
|
||||
if let Some(to) = flag("--to").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.start <= to);
|
||||
}
|
||||
if let Some(limit) = flag("--limit").map(|v| v.parse::<usize>().unwrap()) {
|
||||
ranges.truncate(limit);
|
||||
}
|
||||
(group, ranges)
|
||||
}
|
||||
None => {
|
||||
let positional = (1..args.len()).find(|&i| !args[i].starts_with("--") && !args[i - 1].starts_with("--")).map(|i| args[i].clone());
|
||||
let dir = positional.expect(usage);
|
||||
let (group, range) = range_in(&dir).unwrap_or_else(|| panic!("{dir}: no range.json"));
|
||||
(group, vec![range])
|
||||
}
|
||||
};
|
||||
if ranges.is_empty() {
|
||||
eprintln!("no complete range found");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let finish = !has("--no-finish");
|
||||
let rules = match flag("--rules") {
|
||||
Some(path) => {
|
||||
let json = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
|
||||
serde_json::from_str::<edge::lease::RulesJson>(&json).unwrap_or_else(|e| panic!("{path}: {e}")).rules()
|
||||
}
|
||||
None => {
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
names::Rules::defaults(priorities)
|
||||
}
|
||||
};
|
||||
// The report goes to stderr when the wire takes stdout.
|
||||
let say = |line: String| {
|
||||
if wire_stdout {
|
||||
eprintln!("{line}");
|
||||
} else {
|
||||
println!("{line}");
|
||||
}
|
||||
};
|
||||
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, names::Rules::defaults(priorities), idle);
|
||||
let (start, end) = (ranges[0].start, ranges[ranges.len() - 1].end);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, rules, idle);
|
||||
|
||||
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
|
||||
let (mut compressed, mut text_bytes, mut chunks) = (0usize, 0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, samples
|
||||
let mut finished = 0usize;
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
let mut decompressor = libdeflater::Decompressor::new();
|
||||
let wall = Instant::now();
|
||||
for (i, file) in files.iter().enumerate() {
|
||||
let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
|
||||
compressed += raw.len();
|
||||
let t = Instant::now();
|
||||
let data = inflate(&raw, &mut decompressor);
|
||||
inflate_s += t.elapsed().as_secs_f64();
|
||||
text_bytes += data.len();
|
||||
let t = Instant::now();
|
||||
let text = String::from_utf8_lossy(&data);
|
||||
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||
indexer.add_text(&text, next);
|
||||
let last = i + 1 == files.len();
|
||||
if last {
|
||||
indexer.finish();
|
||||
let mut since_batch = 0usize;
|
||||
for (ri, range) in ranges.iter().enumerate() {
|
||||
let range_wall = Instant::now();
|
||||
let headers_before = out.0;
|
||||
// After a gap between stored ranges the batch starts at this range (every range ends with a batch taken).
|
||||
if ri > 0 {
|
||||
indexer.skip_to(range.start);
|
||||
}
|
||||
if !last && (i + 1) % batch_chunks != 0 {
|
||||
let mut files: Vec<String> = std::fs::read_dir(&range.dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||
files.sort();
|
||||
for (i, file) in files.iter().enumerate() {
|
||||
let raw = std::fs::read(format!("{}/{file}", range.dir)).unwrap();
|
||||
compressed += raw.len();
|
||||
let t = Instant::now();
|
||||
let data = inflate(&raw, &mut decompressor);
|
||||
inflate_s += t.elapsed().as_secs_f64();
|
||||
text_bytes += data.len();
|
||||
chunks += 1;
|
||||
let t = Instant::now();
|
||||
let text = String::from_utf8_lossy(&data);
|
||||
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||
indexer.add_text(&text, next);
|
||||
since_batch += 1;
|
||||
let range_end = i + 1 == files.len();
|
||||
let last = range_end && ri + 1 == ranges.len();
|
||||
if last && finish {
|
||||
indexer.finish();
|
||||
}
|
||||
if !range_end && since_batch < batch_chunks && indexer.pending() < MAX_PENDING_FILES {
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
continue;
|
||||
}
|
||||
let batch = indexer.take();
|
||||
since_batch = 0;
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
continue;
|
||||
out.0 += batch.headers;
|
||||
out.1 += batch.releases.len();
|
||||
for r in &batch.releases {
|
||||
keys.insert(r.key.clone());
|
||||
out.2 += r.files;
|
||||
out.3 += r.segments;
|
||||
out.4 += r.payload.len();
|
||||
finished += r.finished as usize;
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
w.flush().unwrap();
|
||||
}
|
||||
if let Some(w) = releases_out.as_mut() {
|
||||
dump_releases(&batch, &group, w);
|
||||
}
|
||||
}
|
||||
let batch = indexer.take();
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
out.0 += batch.headers;
|
||||
out.1 += batch.releases.len();
|
||||
for r in &batch.releases {
|
||||
keys.insert(r.key.clone());
|
||||
out.2 += r.files;
|
||||
out.3 += r.segments;
|
||||
out.4 += r.payload.len();
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.fragments.len();
|
||||
out.7 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
}
|
||||
if let Some(w) = releases_out.as_mut() {
|
||||
dump_releases(&batch, &group, w);
|
||||
if ranges.len() > 1 {
|
||||
eprintln!(
|
||||
"{} {}–{}: {} headers in {:.1} s; frontier {}; {} releases out ({} finished), {} releases and {} files open; RSS {} MB",
|
||||
group,
|
||||
range.start,
|
||||
range.end,
|
||||
out.0 - headers_before,
|
||||
range_wall.elapsed().as_secs_f64(),
|
||||
date(indexer.frontier()),
|
||||
out.1,
|
||||
finished,
|
||||
indexer.open_releases(),
|
||||
indexer.open_files(),
|
||||
peak_rss_mb()
|
||||
);
|
||||
}
|
||||
}
|
||||
let total = inflate_s + index_s;
|
||||
#[allow(static_mut_refs)]
|
||||
let p = unsafe { indexer::PROF };
|
||||
eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
|
||||
println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
|
||||
println!(
|
||||
"{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",
|
||||
out.0,
|
||||
total,
|
||||
out.0 as f64 / total,
|
||||
wall.elapsed().as_secs_f64()
|
||||
);
|
||||
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
|
||||
println!(
|
||||
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
|
||||
say(format!("{group} {start}–{end}: {} ranges, {chunks} chunks, {} MB compressed, {} MB text", ranges.len(), compressed / 1_000_000, text_bytes / 1_000_000));
|
||||
say(format!("{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)", out.0, total, out.0 as f64 / total, wall.elapsed().as_secs_f64()));
|
||||
say(format!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb()));
|
||||
say(format!(
|
||||
" kept: {} release chunks ({} releases, {} finished), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} samples",
|
||||
out.1,
|
||||
keys.len(),
|
||||
finished,
|
||||
out.2,
|
||||
out.3,
|
||||
out.4 as f64 / 1e6,
|
||||
out.4 as f64 / out.3.max(1) as f64,
|
||||
out.5,
|
||||
out.6,
|
||||
out.7
|
||||
);
|
||||
out.6
|
||||
));
|
||||
}
|
||||
+67
-7
@@ -40,11 +40,11 @@ static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||||
(re(r"(?i)\.tar$"), ""),
|
||||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$"), ""),
|
||||
]
|
||||
});
|
||||
|
||||
@@ -147,10 +147,13 @@ fn strip_volume_digits(s: &[u8]) -> Option<usize> {
|
||||
((2..=3).contains(&d) && n > d && s[n - d - 1] == b'.').then(|| n - d - 1)
|
||||
}
|
||||
|
||||
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||||
const SPLIT_EXTS: [&[u8]; 23] = [
|
||||
b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob", b"wmv", b"mov", b"mpg", b"mpeg", b"divx", b"webm", b"flv", b"m2v",
|
||||
b"mts",
|
||||
];
|
||||
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||||
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||||
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||||
const MEDIA_EXTS: [&[u8]; 16] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf", b"wmv", b"mov", b"mpg", b"mpeg"];
|
||||
|
||||
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||||
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||||
@@ -446,6 +449,43 @@ fn finish_remainder(s: &str) -> String {
|
||||
js::trim(&out).to_string()
|
||||
}
|
||||
|
||||
/// displayName: a subject name as shown, brackets the counters left unmatched removed ("… Facialized 002/477] - …"
|
||||
/// gave "… Facialized ]"). Only the name: the key keeps the remainder.
|
||||
pub fn display_name(remainder: &str) -> String {
|
||||
let b = remainder.as_bytes();
|
||||
if !b.iter().any(|c| matches!(c, b'[' | b']' | b'(' | b')')) {
|
||||
return remainder.to_string();
|
||||
}
|
||||
let mut drop = vec![false; b.len()];
|
||||
for (open, close) in [(b'[', b']'), (b'(', b')')] {
|
||||
let mut opens: Vec<usize> = Vec::new();
|
||||
for (i, &c) in b.iter().enumerate() {
|
||||
if c == open {
|
||||
opens.push(i);
|
||||
} else if c == close && opens.pop().is_none() {
|
||||
drop[i] = true;
|
||||
}
|
||||
}
|
||||
for i in opens {
|
||||
drop[i] = true;
|
||||
}
|
||||
}
|
||||
if !drop.contains(&true) {
|
||||
return remainder.to_string();
|
||||
}
|
||||
let kept: Vec<u8> = b.iter().zip(&drop).filter(|(_, &d)| !d).map(|(&c, _)| c).collect();
|
||||
let kept = String::from_utf8(kept).expect("only ASCII brackets removed");
|
||||
let mut out = String::with_capacity(kept.len());
|
||||
for c in kept.chars() {
|
||||
if c == ' ' && out.ends_with(' ') {
|
||||
continue;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
let name = js::trim(&out);
|
||||
if name.is_empty() { remainder.to_string() } else { name.to_string() }
|
||||
}
|
||||
|
||||
/// counterTotal: the subject's file counter total, first form found first.
|
||||
pub fn counter_total(subject: &str) -> Option<String> {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
@@ -838,11 +878,11 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||
if !image && test(&RANDOM_VOLUME, name) && stem.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-') && looks_random(&stripped) {
|
||||
parts.push(&stem);
|
||||
}
|
||||
return Release { key: hash(&parts), name: remainder, source: "subject", file_total };
|
||||
return Release { key: hash(&parts), name: display_name(&remainder), source: "subject", file_total };
|
||||
}
|
||||
if let Some(total) = post_total.filter(|&n| n != 0 && !tag) {
|
||||
if is_post_token(&remainder) {
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: display_name(&remainder), source: "post", file_total };
|
||||
}
|
||||
}
|
||||
// A scene music post's index files ("00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg") under a bare counter go
|
||||
@@ -1030,7 +1070,7 @@ static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(bin|zip)$"));
|
||||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||||
@@ -1234,6 +1274,11 @@ pub fn explored(key: &str, percent: f64) -> bool {
|
||||
}
|
||||
|
||||
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||||
|
||||
/// A sidecar by its extension (the judge's SIDECAR): no release of its own, and no album's member.
|
||||
pub fn is_sidecar(name: &str) -> bool {
|
||||
test(&SIDECAR, name)
|
||||
}
|
||||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||
|
||||
pub struct JudgedFile<'a> {
|
||||
@@ -1326,6 +1371,21 @@ mod tests {
|
||||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||||
assert_eq!(file_stem("a.7z.002"), "a");
|
||||
assert_eq!(file_stem("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
assert_eq!(file_stem("clip.mov.12"), "clip");
|
||||
assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_names() {
|
||||
let porn4you = "webmaster@hdbangvidz.com (Porn4You)";
|
||||
let a = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 002/477] - 0178.jpg", "0178.jpg");
|
||||
let b = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 003/477] - 0179.jpg", "0179.jpg");
|
||||
assert_eq!(a.name, "Babe with Plump Bubble Butt Fucked and Facialized");
|
||||
assert_eq!(a.key, b.key);
|
||||
assert_eq!(display_name("[Group] Show (2020)"), "[Group] Show (2020)");
|
||||
assert_eq!(display_name("( Show [x] name"), "Show [x] name");
|
||||
assert_eq!(display_name("]"), "]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+10
-42
@@ -1,7 +1,7 @@
|
||||
//! A batch as the coordinator takes it (lib/edge/wire.ts toWire): JSON with
|
||||
//! the TypeScript names, chunk payloads in base64.
|
||||
|
||||
use crate::indexer::{Batch, Candidate, Fragment, ReleaseChunk};
|
||||
use crate::indexer::{Batch, Candidate, ReleaseChunk};
|
||||
use base64::Engine;
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -24,6 +24,11 @@ struct WireRelease<'a> {
|
||||
last_article: i64,
|
||||
readable: bool,
|
||||
has_par2: bool,
|
||||
finished: bool,
|
||||
edge: bool,
|
||||
partial: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool: Option<&'a str>,
|
||||
payload: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
rule: Option<&'a str>,
|
||||
@@ -49,30 +54,6 @@ struct WireCount<'a> {
|
||||
files: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragmentRelease<'a> {
|
||||
key: &'a str,
|
||||
name: &'a str,
|
||||
source: &'a str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
file_total: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragment<'a> {
|
||||
file_key: &'a str,
|
||||
name: &'a str,
|
||||
poster: &'a str,
|
||||
subject: &'a str,
|
||||
total_parts: u32,
|
||||
groups: &'a [String],
|
||||
release: Option<WireFragmentRelease<'a>>,
|
||||
posted_at: Option<i64>,
|
||||
segments: Vec<(u32, &'a str, i64, i64)>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireBatch<'a> {
|
||||
@@ -85,7 +66,6 @@ struct WireBatch<'a> {
|
||||
samples: Vec<WireRelease<'a>>,
|
||||
candidates: Vec<WireCandidate<'a>>,
|
||||
counts: Vec<WireCount<'a>>,
|
||||
fragments: Vec<WireFragment<'a>>,
|
||||
}
|
||||
|
||||
fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRelease<'a> {
|
||||
@@ -106,6 +86,10 @@ fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRel
|
||||
last_article: r.last_article,
|
||||
readable: r.readable,
|
||||
has_par2: r.has_par2,
|
||||
finished: r.finished,
|
||||
edge: r.edge,
|
||||
partial: r.partial,
|
||||
tool: r.tool,
|
||||
payload: base64::engine::general_purpose::STANDARD.encode(&r.payload),
|
||||
rule: rule.map(|(rule, _)| rule),
|
||||
rule_key: rule.map(|(_, key)| key),
|
||||
@@ -116,21 +100,6 @@ fn candidate(c: &Candidate) -> WireCandidate<'_> {
|
||||
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
|
||||
}
|
||||
|
||||
fn fragment(f: &Fragment) -> WireFragment<'_> {
|
||||
WireFragment {
|
||||
file_key: &f.file_key,
|
||||
name: &f.name,
|
||||
poster: &f.poster,
|
||||
subject: &f.subject,
|
||||
total_parts: f.total_parts,
|
||||
groups: &f.groups,
|
||||
// A named file always has a release; fileTotal only when truthy, as fragmentOf.
|
||||
release: Some(WireFragmentRelease { key: &f.release.key, name: &f.release.name, source: f.release.source, file_total: f.release.file_total_truthy() }),
|
||||
posted_at: f.posted_at,
|
||||
segments: f.segments.iter().map(|(n, id, bytes, article)| (*n, id.as_str(), *bytes, *article)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// toWire(batch) as JSON bytes.
|
||||
pub fn to_json(batch: &Batch) -> Vec<u8> {
|
||||
let wire = WireBatch {
|
||||
@@ -143,7 +112,6 @@ pub fn to_json(batch: &Batch) -> Vec<u8> {
|
||||
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
||||
candidates: batch.candidates.iter().map(candidate).collect(),
|
||||
counts: batch.counts.iter().map(|(rule, key, articles, files)| WireCount { rule, key, articles: *articles, files: *files }).collect(),
|
||||
fragments: batch.fragments.iter().map(fragment).collect(),
|
||||
};
|
||||
serde_json::to_vec(&wire).unwrap()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { SegmentScanner, scanOverviewText } from './scan.js'
|
||||
|
||||
/**
|
||||
* What a corpus chunk left out (REBUILD.md, corpus; LOG.md 2026-10-23 (101)): the overview lines the edge worker only
|
||||
* counts and never indexes, decided per line as RangeIndexer.line does: a crosspost an indexed group with a lower
|
||||
* priority number owns (by owner), and a subject that names no file part (obfuscated). Article numbers ascending,
|
||||
* stored as differences. Release-level rules (garbage, per-article, executable, poster) judge files put together
|
||||
* across chunks, so they aren't applied here.
|
||||
*/
|
||||
export interface ChunkDrops {
|
||||
version: 1
|
||||
/** The rule set (rule_sets.version) and group priorities the decision was made with. */
|
||||
ruleVersion: number
|
||||
priorities: Record<string, number>
|
||||
/** Overview lines in the chunk, and kept. */
|
||||
lines: number
|
||||
kept: number
|
||||
/** By owning group, article numbers as differences (the first from 0). */
|
||||
crosspost: Record<string, number[]>
|
||||
obfuscated: number[]
|
||||
}
|
||||
|
||||
const deltas = (numbers: number[]) => numbers.map((n, i) => (i === 0 ? n : n - numbers[i - 1]))
|
||||
|
||||
/** The article numbers back from ChunkDrops differences. */
|
||||
export function undelta(differences: number[]): number[] {
|
||||
let at = 0
|
||||
return differences.map((d) => (at += d))
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk's overview text without the lines the edge worker drops on their own, and what was dropped. Lines kept
|
||||
* byte for byte (with their line ends); lines the worker skips uncounted (no number or message-id) are kept.
|
||||
*/
|
||||
export function filterOverviews(text: string, group: string, priorities: Record<string, number>, ruleVersion: number): { kept: string; drops: ChunkDrops } {
|
||||
const scanner = new SegmentScanner()
|
||||
const crosspost = new Map<string, number[]>()
|
||||
const obfuscated: number[] = []
|
||||
const dropped = new Set<number>()
|
||||
let lines = 0
|
||||
scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => {
|
||||
lines++
|
||||
if (!Number.isFinite(n) || !messageId) return
|
||||
const owner = xref ? scanner.owner(xref, group, priorities) : group
|
||||
if (owner !== group) {
|
||||
const list = crosspost.get(owner)
|
||||
if (list) list.push(n)
|
||||
else crosspost.set(owner, [n])
|
||||
dropped.add(n)
|
||||
} else if (!scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) {
|
||||
obfuscated.push(n)
|
||||
dropped.add(n)
|
||||
}
|
||||
})
|
||||
const out: string[] = []
|
||||
let start = 0
|
||||
while (start < text.length) {
|
||||
let end = text.indexOf('\n', start)
|
||||
end = end === -1 ? text.length : end + 1
|
||||
const tab = text.indexOf('\t', start)
|
||||
const n = tab !== -1 && tab < end ? Number.parseInt(text.slice(start, tab), 10) : Number.NaN
|
||||
if (!dropped.has(n)) out.push(text.slice(start, end))
|
||||
start = end
|
||||
}
|
||||
const sorted = (list: number[]) => [...list].sort((a, b) => a - b)
|
||||
return {
|
||||
kept: out.join(''),
|
||||
drops: {
|
||||
version: 1,
|
||||
ruleVersion,
|
||||
priorities: Object.fromEntries(Object.entries(priorities).filter(([name]) => crosspost.has(name) || name === group)),
|
||||
lines,
|
||||
kept: lines - dropped.size,
|
||||
crosspost: Object.fromEntries([...crosspost].map(([owner, list]) => [owner, deltas(sorted(list))])),
|
||||
obfuscated: deltas(sorted(obfuscated)),
|
||||
},
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
|
||||
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
|
||||
export { toWire, fromWire, type WireBatch } from './wire.js'
|
||||
export type { Lease } from './lease.js'
|
||||
export { mergeFragments } from './merge.js'
|
||||
export type { Lease, GroupLease, Lane, NextRange } from './lease.js'
|
||||
export type { EdgeSegment, FileMeta } from './scan.js'
|
||||
export { throttle } from './throttle.js'
|
||||
export { filterOverviews, undelta, type ChunkDrops } from './corpus_filter.js'
|
||||
@@ -5,3 +5,27 @@ export interface Lease {
|
||||
range: { id: number; group: string; start: number; end: number; next: number }
|
||||
rules: Rules
|
||||
}
|
||||
|
||||
/**
|
||||
* A group's two cursors (REBUILD.md, "Voyager 3: window indexing", 1): `history` works the ranges the lease bounds
|
||||
* hand out (the windows, the history), `tail` the tail planner's ranges. Each lane has its own holder, window and
|
||||
* snapshot; the group routes take `?lane=` (history by default).
|
||||
*/
|
||||
export type Lane = 'history' | 'tail'
|
||||
|
||||
/**
|
||||
* A group lease (REBUILD.md, "Voyager 3: window indexing", 1): the group a worker thread holds, the lane, the rules,
|
||||
* and the snapshot of the lane's window when there is one (its next article; the body comes from GET
|
||||
* groups/:id/snapshot).
|
||||
*/
|
||||
export interface GroupLease {
|
||||
group: { id: number; name: string; priority: number }
|
||||
lane: Lane
|
||||
rules: Rules
|
||||
snapshot: { next: number } | null
|
||||
}
|
||||
|
||||
/** The group's next range in article order, leased to the holder (POST groups/:id/next; 204 when caught up). */
|
||||
export interface NextRange {
|
||||
range: { id: number; start: number; end: number; next: number }
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
import type { AggregatedFile } from '../files/index.js'
|
||||
import type { Fragment } from './range_indexer.js'
|
||||
import type { EdgeSegment, FileMeta } from './scan.js'
|
||||
|
||||
/**
|
||||
* The fragments of one file from neighbouring ranges, put together
|
||||
* (REBUILD.md, fragments): the complete file when every part from 1 to its
|
||||
* total is there (the first article seen of a part wins), else null.
|
||||
*/
|
||||
export function mergeFragments(fragments: Fragment[]): AggregatedFile<EdgeSegment> | null {
|
||||
if (fragments.length === 0) return null
|
||||
const first = fragments[0]
|
||||
const meta: FileMeta = {
|
||||
fileKey: first.fileKey,
|
||||
name: first.name,
|
||||
poster: first.poster,
|
||||
subject: first.subject,
|
||||
totalParts: first.totalParts,
|
||||
groups: first.groups,
|
||||
release: first.release ? { ...first.release, source: first.release.source as any } : null,
|
||||
}
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const fragment of fragments) {
|
||||
const postedAt = fragment.postedAt ?? Number.NaN
|
||||
for (const [number, messageId, bytes, articleNumber] of fragment.segments) {
|
||||
const known = parts.get(number)
|
||||
if (!known || articleNumber < known.articleNumber) {
|
||||
parts.set(number, { fileKey: meta.fileKey, number, totalParts: meta.totalParts, messageId, bytes, articleNumber, postedAt, file: meta })
|
||||
}
|
||||
}
|
||||
}
|
||||
if (parts.size < meta.totalParts) return null
|
||||
for (let n = 1; n <= meta.totalParts; n++) if (!parts.has(n)) return null
|
||||
return { fileKey: meta.fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }
|
||||
}
|
||||
@@ -26,6 +26,20 @@ export interface ReleaseChunk {
|
||||
/** A file with a readable name; a par2 file. */
|
||||
readable: boolean
|
||||
hasPar2: boolean
|
||||
/**
|
||||
* Voyager 3: the window closed on the release (default), or it is a tail release sent provisionally, finished by a
|
||||
* later chunk of the same release key (the totals then are the release's; the payload only the files not sent yet).
|
||||
*/
|
||||
finished?: boolean
|
||||
/** Still open at its block's end, or posted within the window of its start: the block merger joins it with the neighbour block's part. */
|
||||
edge?: boolean
|
||||
/** A file of the release misses parts (the window closed on it): not complete, whatever the counter says. */
|
||||
partial?: boolean
|
||||
/**
|
||||
* Voyager 3, an unnamed release (readable false): the upload tool from its first article's message-id, its
|
||||
* signature with the group; the payload is empty (no chunk until a probe names it: pending_releases).
|
||||
*/
|
||||
tool?: string
|
||||
payload: Buffer
|
||||
}
|
||||
|
||||
@@ -88,7 +102,8 @@ export interface EdgeBatch {
|
||||
samples: DroppedChunk[]
|
||||
candidates: ProbeCandidate[]
|
||||
counts: RuleCount[]
|
||||
fragments: Fragment[]
|
||||
/** The range path's cut-off files (the TypeScript indexer); the Rust edge's window has none, the store ignores them. */
|
||||
fragments?: Fragment[]
|
||||
}
|
||||
|
||||
export interface RangeIndexerOptions {
|
||||
@@ -108,6 +123,11 @@ export interface RangeIndexerOptions {
|
||||
* chunks seen so far), as uploads in busy groups spread over minutes.
|
||||
*/
|
||||
edgeArticles?: number
|
||||
/**
|
||||
* A file given up short of parts is a file of its release all the same (partial), as the Rust edge's window writes
|
||||
* it, not a fragment: the pending namer reads such a release again (LOG.md 2026-10-29 (121)).
|
||||
*/
|
||||
partialFiles?: boolean
|
||||
}
|
||||
|
||||
const MAX_CANDIDATES = 4
|
||||
@@ -203,7 +223,7 @@ export class RangeIndexer {
|
||||
|
||||
/** Files completed and fragments cut since the last take: how big the next batch is. */
|
||||
get pending(): number {
|
||||
let files = this.batch.fragments.length
|
||||
let files = this.batch.fragments!.length
|
||||
for (const list of this.completed.values()) files += list.length
|
||||
return files
|
||||
}
|
||||
@@ -259,6 +279,18 @@ export class RangeIndexer {
|
||||
else byFile.set(s.fileKey, [s])
|
||||
}
|
||||
for (const list of byFile.values()) {
|
||||
if (this.options.partialFiles) {
|
||||
// As the Rust window's files(): one article of a multi-part random file is no release.
|
||||
if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) {
|
||||
this.count('per-article', uploadTool(list[0].messageId), 1, 0)
|
||||
continue
|
||||
}
|
||||
// One segment per part (two copies of a file share its key here), the first article seen.
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const s of list.sort((a, b) => a.articleNumber - b.articleNumber)) if (!parts.has(s.number)) parts.set(s.number, s)
|
||||
this.files([{ fileKey: list[0].fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }])
|
||||
continue
|
||||
}
|
||||
// One article of a "file" of many parts under a random name: a random subject per article, which
|
||||
// never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09).
|
||||
if (list.length === 1 && isRandomFile(list[0].file.name)) {
|
||||
@@ -287,7 +319,7 @@ export class RangeIndexer {
|
||||
fragment.segments.push([s.number, s.messageId, s.bytes, s.articleNumber])
|
||||
if (Number.isFinite(s.postedAt) && (fragment.postedAt === null || s.postedAt < fragment.postedAt)) fragment.postedAt = s.postedAt
|
||||
}
|
||||
this.batch.fragments.push(fragment)
|
||||
this.batch.fragments!.push(fragment)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,6 +447,7 @@ export function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[], uploa
|
||||
lastArticle,
|
||||
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
|
||||
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
||||
partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
|
||||
payload: encodeChunk(chunkFiles),
|
||||
}
|
||||
}
|
||||
@@ -434,6 +467,8 @@ export function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[],
|
||||
if (!tail || volume > tail.volume) tail = { volume, file: f }
|
||||
continue
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if (volume === 0 && f.segments.length > 1 && !tail) tail = { volume, file: f }
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name })
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ interface Counter {
|
||||
function findCounter(subject: string, after: number): Counter | null {
|
||||
let open = subject.length
|
||||
while (true) {
|
||||
// lastIndexOf('(', -1) searches from 0 again: a "(" at the start that isn't a counter looped forever.
|
||||
if (open === 0) return null
|
||||
open = subject.lastIndexOf('(', open - 1)
|
||||
if (open <= after) return null
|
||||
let i = open + 1
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001: its type; the listing is at
|
||||
* its end, the 7z-tail), ZIP (its first entries) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
if (/\.(bin|zip)$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */
|
||||
/** Newznab categories this indexer assigns (release_category_for() and spot_category() in SQL, migration …5200). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 1000, name: 'Console', subcats: [] },
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3030, name: 'Audiobook' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [{ id: 4050, name: 'Games' }] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }, { id: 5070, name: 'Anime' }] },
|
||||
{ id: 6000, name: 'XXX', subcats: [{ id: 6040, name: 'x264' }, { id: 6045, name: 'UHD' }, { id: 6060, name: 'ImgSet' }, { id: 6070, name: 'Other' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }, { id: 7030, name: 'Comics' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
|
||||
+8
-1
@@ -1,4 +1,11 @@
|
||||
export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
|
||||
export {
|
||||
XoverClient,
|
||||
MissingArticle,
|
||||
TruncatedReply,
|
||||
parseOverviews,
|
||||
type Overview,
|
||||
type XoverClientOptions,
|
||||
} from './xover_client.js'
|
||||
export { parseXrefGroups } from './xref.js'
|
||||
export { parseGroupResponse, type GroupInfo } from './group.js'
|
||||
export { hedge, type Request, type Hedged } from './hedge.js'
|
||||
+71
-11
@@ -36,6 +36,14 @@ const DOT_LINE = Buffer.from('.\r\n')
|
||||
*/
|
||||
/** The server doesn't have the article (430): taken down or expired. */
|
||||
export class MissingArticle extends Error {}
|
||||
/**
|
||||
* A compressed reply the server cut short: it ends its compressed output at the first ".\r\n" in it, as if that
|
||||
* were the terminator, and sends nothing more (Eweka; "78 01 2e 0d 0a" for 16 of 14k small XOVERs, LOG.md
|
||||
* 2026-10-25 (105)). The same request without compression answers.
|
||||
*/
|
||||
export class TruncatedReply extends Error {}
|
||||
/** How long a compressed reply that doesn't inflate may stay silent before it counts as cut short. */
|
||||
const TRUNCATED_MS = 2_000
|
||||
|
||||
export class XoverClient {
|
||||
private chunks: Buffer[] = []
|
||||
@@ -54,6 +62,10 @@ export class XoverClient {
|
||||
get buffered() {
|
||||
return this.length
|
||||
}
|
||||
/** The first `n` bytes received and not consumed yet (what a reply that stalled left; nntp:probe). */
|
||||
bufferedHead(n = 32): Buffer {
|
||||
return this.buffer().subarray(0, n)
|
||||
}
|
||||
private lastDataAt = 0
|
||||
private longestGap = 0
|
||||
|
||||
@@ -85,7 +97,11 @@ export class XoverClient {
|
||||
|
||||
static async connect(options: XoverClientOptions): Promise<XoverClient> {
|
||||
const sock = options.secure
|
||||
? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host })
|
||||
? tls.connect({
|
||||
host: options.host,
|
||||
port: options.port,
|
||||
servername: options.servername || options.host,
|
||||
})
|
||||
: net.connect({ host: options.host, port: options.port })
|
||||
const client = new XoverClient(sock, options.timeoutMs ?? 120_000)
|
||||
// The greeting and login are short: a server holding a connection back (too many open) fails it in 30 s, not timeoutMs.
|
||||
@@ -94,10 +110,14 @@ export class XoverClient {
|
||||
await client.expect(await client.line(), ['200', '201'])
|
||||
if (options.user) {
|
||||
const reply = await client.command(`AUTHINFO USER ${options.user}`)
|
||||
if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281'])
|
||||
if (reply.startsWith('381'))
|
||||
await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), [
|
||||
'281',
|
||||
])
|
||||
else await client.expect(reply, ['281'])
|
||||
}
|
||||
if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
if (options.compress)
|
||||
client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
} catch (error) {
|
||||
client.close()
|
||||
throw error
|
||||
@@ -127,7 +147,9 @@ export class XoverClient {
|
||||
const reply = await this.command(`LIST ACTIVE ${pattern}`)
|
||||
await this.expect(reply, ['215'])
|
||||
// With XFEATURE COMPRESS GZIP on, Eweka compresses this list too ("215 NewsGroups Follow [COMPRESS=GZIP]").
|
||||
const body = reply.includes('COMPRESS=GZIP') ? (await this.compressedRaw(true)).text : await this.plainBody()
|
||||
const body = reply.includes('COMPRESS=GZIP')
|
||||
? (await this.compressedRaw(true)).text
|
||||
: await this.plainBody()
|
||||
return body
|
||||
.split('\r\n')
|
||||
.filter(Boolean)
|
||||
@@ -154,7 +176,11 @@ export class XoverClient {
|
||||
return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
|
||||
} finally {
|
||||
this.measuring = false
|
||||
this.lastTiming = { replyMs: repliedAt - sentAt, bodyMs: performance.now() - repliedAt, longestGapMs: this.longestGap }
|
||||
this.lastTiming = {
|
||||
replyMs: repliedAt - sentAt,
|
||||
bodyMs: performance.now() - repliedAt,
|
||||
longestGapMs: this.longestGap,
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -180,7 +206,9 @@ export class XoverClient {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return []
|
||||
await this.expect(reply, ['224'])
|
||||
const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody()
|
||||
const data = reply.includes('COMPRESS=GZIP')
|
||||
? await this.compressedBody()
|
||||
: await this.plainBody()
|
||||
return parseOverviews(data)
|
||||
}
|
||||
|
||||
@@ -199,7 +227,8 @@ export class XoverClient {
|
||||
}
|
||||
|
||||
private async expect(reply: string, codes: string[]) {
|
||||
if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
if (!codes.some((code) => reply.startsWith(code)))
|
||||
throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
}
|
||||
|
||||
private buffer(): Buffer {
|
||||
@@ -223,10 +252,28 @@ export class XoverClient {
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
}
|
||||
|
||||
/** As more(), but false when nothing came within `ms`. */
|
||||
private async moreWithin(ms: number): Promise<boolean> {
|
||||
if (this.failure) throw this.failure
|
||||
const came = await new Promise<boolean>((resolve) => {
|
||||
const timer = setTimeout(() => resolve(false), ms)
|
||||
this.wake = () => {
|
||||
clearTimeout(timer)
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
this.wake = undefined
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
return came
|
||||
}
|
||||
|
||||
private async line(): Promise<string> {
|
||||
while (true) {
|
||||
const at = this.buffer().indexOf('\r\n')
|
||||
if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1')
|
||||
if (at !== -1)
|
||||
return this.take(at + 2)
|
||||
.subarray(0, at)
|
||||
.toString('latin1')
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
@@ -267,7 +314,9 @@ export class XoverClient {
|
||||
*/
|
||||
async bodies(messageIds: string[]): Promise<(Buffer | null)[]> {
|
||||
return this.timed(async () => {
|
||||
this.sock.write(messageIds.map((m) => `BODY ${m.startsWith('<') ? m : `<${m}>`}\r\n`).join(''))
|
||||
this.sock.write(
|
||||
messageIds.map((m) => `BODY ${m.startsWith('<') ? m : `<${m}>`}\r\n`).join('')
|
||||
)
|
||||
const out: (Buffer | null)[] = []
|
||||
for (let i = 0; i < messageIds.length; i++) {
|
||||
const reply = await this.line()
|
||||
@@ -309,7 +358,11 @@ export class XoverClient {
|
||||
return ''
|
||||
}
|
||||
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
|
||||
if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.')
|
||||
if (at !== -1)
|
||||
return this.take(at + TERMINATOR.length)
|
||||
.subarray(0, at + 2)
|
||||
.toString('utf8')
|
||||
.replace(/^\.\./gm, '.')
|
||||
searched = all.length
|
||||
await this.more()
|
||||
}
|
||||
@@ -359,7 +412,14 @@ export class XoverClient {
|
||||
this.take(all.length)
|
||||
return { raw, text: inflated.toString('utf8') }
|
||||
} catch {
|
||||
// ".\r\n" inside the compressed data: wait for more.
|
||||
// ".\r\n" inside the compressed data: wait for more, but not for the reply timeout: the server may have
|
||||
// cut its output there (TruncatedReply), and then nothing comes. Raw, the chunk is returned as it is
|
||||
// and reported by the worker; the text is wanted here.
|
||||
if (!(await this.moreWithin(TRUNCATED_MS))) {
|
||||
this.take(all.length)
|
||||
throw new TruncatedReply(`NNTP: compressed reply cut short after ${all.length} bytes`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
await this.more()
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
export { inspect, innerName, MAX_NAMES, PROBE_VERSION, type Inspection } from './inspect.js'
|
||||
export { inspect, innerName, usableName, MAX_NAMES, PROBE_VERSION, type Inspection } from './inspect.js'
|
||||
+77
-9
@@ -2,7 +2,10 @@ import { createHash } from 'node:crypto'
|
||||
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||
import { parseZipStart } from '#lib/zip/index'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
import { fileStem, looksRandom } from '#lib/release/index'
|
||||
|
||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||
export interface Inspection {
|
||||
@@ -23,7 +26,10 @@ export interface Inspection {
|
||||
*/
|
||||
hash16k?: string
|
||||
/** A par2 body's recovery set: every file it describes (not kept in release_inspections; par2_files). */
|
||||
par2?: { setId: string; files: { fileId: string; hash16k: string; length: number; name: string }[] }
|
||||
par2?: {
|
||||
setId: string
|
||||
files: { fileId: string; hash16k: string; length: number; name: string }[]
|
||||
}
|
||||
}
|
||||
|
||||
const HASH_BYTES = 16 * 1024
|
||||
@@ -55,12 +61,22 @@ function largestFirst(items: { name: string; size: number }[]): string[] {
|
||||
.map((i) => i.name.slice(0, MAX_NAME_LENGTH))
|
||||
}
|
||||
|
||||
/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
|
||||
export function inspect(body: Buffer): Inspection {
|
||||
/**
|
||||
* One article's body: its yEnc header, what the first bytes are, and for par2, RAR, ZIP and 7z what's inside.
|
||||
* Throws when it isn't yEnc. `kind` is the probe candidate's: a 7z-tail's article ends the archive.
|
||||
*/
|
||||
export function inspect(body: Buffer, kind?: string): Inspection {
|
||||
const segment = decodeBody(body)
|
||||
const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
|
||||
const result: Inspection = {
|
||||
yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH),
|
||||
size: segment.meta.begin.size,
|
||||
type: sniffContentType(segment.data),
|
||||
}
|
||||
// The file's first 16 KiB, or all of a smaller file.
|
||||
if (segment.offset === 0 && (segment.data.length >= HASH_BYTES || segment.data.length === result.size)) {
|
||||
if (
|
||||
segment.offset === 0 &&
|
||||
(segment.data.length >= HASH_BYTES || segment.data.length === result.size)
|
||||
) {
|
||||
result.hash16k = createHash('md5').update(segment.data.subarray(0, HASH_BYTES)).digest('hex')
|
||||
}
|
||||
if (isPar2(segment.data)) {
|
||||
@@ -71,7 +87,12 @@ export function inspect(body: Buffer): Inspection {
|
||||
result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
|
||||
result.par2 = {
|
||||
setId: set.recoverySetId,
|
||||
files: set.files.map((f) => ({ fileId: f.fileId, hash16k: f.md5_16k, length: f.length, name: f.name.slice(0, MAX_NAME_LENGTH) })),
|
||||
files: set.files.map((f) => ({
|
||||
fileId: f.fileId,
|
||||
hash16k: f.md5_16k,
|
||||
length: f.length,
|
||||
name: f.name.slice(0, MAX_NAME_LENGTH),
|
||||
})),
|
||||
}
|
||||
}
|
||||
return result
|
||||
@@ -85,19 +106,66 @@ export function inspect(body: Buffer): Inspection {
|
||||
result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
|
||||
}
|
||||
}
|
||||
if (result.type === 'zip') {
|
||||
const start = parseZipStart(segment.data)
|
||||
if (start) {
|
||||
result.entries = start.files.length
|
||||
result.names = largestFirst(start.files)
|
||||
result.passworded = start.encrypted
|
||||
}
|
||||
}
|
||||
// 7z keeps its listing at the end: the last segment of the last volume (a 7z-tail), or a small archive whole.
|
||||
if (
|
||||
(kind === '7z-tail' || result.type === '7z') &&
|
||||
segment.offset + segment.data.length === result.size
|
||||
) {
|
||||
const listing = parseSevenZipTail(segment.data)
|
||||
if (listing) {
|
||||
result.entries = listing.files.length
|
||||
if (listing.files.length > 0) result.names = largestFirst(listing.files)
|
||||
result.passworded = listing.encrypted
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* The usable name one probe of an unnamed release found (the pending ladder, REBUILD.md decision f): the largest
|
||||
* named file of a par2 set or an archive's listing, or a lone file's yEnc name, when it isn't packaging and isn't
|
||||
* random itself (an obfuscated upload's par2 names its random volumes; its yEnc name is mostly the random name
|
||||
* again). Null: this probe named nothing, the next target is tried.
|
||||
*/
|
||||
export function usableName(inspection: Inspection, kind: string): string | null {
|
||||
const usable = (n: string | undefined) => {
|
||||
if (!n) return null
|
||||
const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null
|
||||
}
|
||||
const named = (inspection.names ?? []).map(usable).find((n) => n)
|
||||
if (named) return named
|
||||
return kind === 'lone' || !inspection.names?.length ? usable(inspection.yencName) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* A release's best inner name from its inspections (by probe kind): the
|
||||
* largest named entry of an archive, else the largest file of a par2 set
|
||||
* that isn't packaging, else a lone file's yEnc name. Null when none says
|
||||
* more than the release's own name.
|
||||
*/
|
||||
export function innerName(inspections: { kind: string; result: Inspection }[], releaseName = ''): string | null {
|
||||
export function innerName(
|
||||
inspections: { kind: string; result: Inspection }[],
|
||||
releaseName = ''
|
||||
): string | null {
|
||||
const of = (kind: string) => inspections.filter((i) => i.kind === kind).map((i) => i.result)
|
||||
const content = (kind: string) => of(kind).flatMap((i) => i.names ?? []).find((n) => !PACKAGING.test(n))
|
||||
const pick = content('archive') ?? content('par2') ?? of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const content = (kind: string) =>
|
||||
of(kind)
|
||||
.flatMap((i) => i.names ?? [])
|
||||
.find((n) => !PACKAGING.test(n))
|
||||
const pick =
|
||||
content('archive') ??
|
||||
content('7z-tail') ??
|
||||
content('par2') ??
|
||||
of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const name = pick?.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
// The release's own name with an extension says nothing new (a random-named file's yEnc name is mostly its random name).
|
||||
return name && !name.toLowerCase().startsWith(releaseName.toLowerCase() || '\0') ? name : null
|
||||
|
||||
+29
-5
@@ -40,12 +40,13 @@ const SUFFIXES = [
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r\d{2,3}))\.\d{2,3}$/i,
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts), and other video
|
||||
// ("Morning Lovers.wmv.001" to ".058" were 58 releases; LOG.md 2026-10-23 (97)).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r\d{2,3}))\.\d{2,3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
@@ -104,6 +105,29 @@ export function subjectRemainder(subject: string): string {
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* A subject name as shown: brackets the counters left unmatched removed ("… Facialized 002/477] - 0177.jpg", the
|
||||
* counter's "[" missing, gave "… Facialized ]"; LOG.md 2026-10-29 (121)). Only the name: the key keeps the remainder.
|
||||
*/
|
||||
export function displayName(remainder: string): string {
|
||||
if (!/[[\]()]/.test(remainder)) return remainder
|
||||
const drop = new Set<number>()
|
||||
for (const [open, close] of [['[', ']'], ['(', ')']]) {
|
||||
const opens: number[] = []
|
||||
for (let i = 0; i < remainder.length; i++) {
|
||||
if (remainder[i] === open) opens.push(i)
|
||||
else if (remainder[i] === close) {
|
||||
if (opens.length > 0) opens.pop()
|
||||
else drop.add(i)
|
||||
}
|
||||
}
|
||||
for (const i of opens) drop.add(i)
|
||||
}
|
||||
if (drop.size === 0) return remainder
|
||||
const name = remainder.split('').filter((_, i) => !drop.has(i)).join('').replace(/ {2,}/g, ' ').trim()
|
||||
return name || remainder
|
||||
}
|
||||
|
||||
/**
|
||||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||||
@@ -447,7 +471,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
if (!image && RANDOM_VOLUME.test(input.name) && /^[A-Za-z0-9_-]+$/.test(stem) && looksRandom(stem.replace(/[-_]/g, ''))) parts.push(stem)
|
||||
return {
|
||||
key: hash(...parts),
|
||||
name: remainder,
|
||||
name: displayName(remainder),
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
}
|
||||
@@ -455,7 +479,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||||
// [i/M] counter it groups the files of one post whose names are random.
|
||||
if (!tag && postTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', poster, remainder, postTotal), name: remainder, fileTotal, source: 'post' }
|
||||
return { key: hash('post', poster, remainder, postTotal), name: displayName(remainder), fileTotal, source: 'post' }
|
||||
}
|
||||
// A scene music post's index files under a bare counter go with its tracks: the artist and the total ("[01/13] -
|
||||
// "00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg"" with "[05/13] - "01-ammar-turk_uit_de_kroeg.mp3"": the
|
||||
|
||||
Loaded 100 of 164 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user