daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: daan/usenet-indexer:44c0c73b0584d6acf3efb9904965771f53c951bb
daan/usenet-indexer:master daan/usenet-indexer:rebuild daan/usenet-indexer:voyager-3 daan/usenet-indexer:monetization daan/usenet-indexer:experiment/adonisjs
..
compare: daan/usenet-indexer:monetization
daan/usenet-indexer:rebuild daan/usenet-indexer:voyager-3 daan/usenet-indexer:monetization daan/usenet-indexer:master daan/usenet-indexer:experiment/adonisjs
1 Commits
44c0c73b05 .. monetization
Author SHA1 Message Date
Daan MeijerandClaude Opus 5.5 1322e42fb7 Accounts with limits: a Newznab key and a tier per user, daily API and NZB counts, invites
Test and deploy / test (push) Successful in 47s
Details
Test and deploy / edge (push) Successful in 1m44s
Details
Test and deploy / deploy (push) Skipped
Details 
ACCESS=accounts (config/access.ts; open stays the default and behaves as
before): Newznab and the catalog's releases routes name a user (apikey or
the web interface's token), whose tier sets searches and NZBs per UTC day
(Newznab 500/501, the catalog 429) and how far back they see (102/403).
Counted in one upsert per use, so the last one can't be taken twice;
caps, stats and refused requests aren't counted. Signup takes an invite
(SIGNUP), which may give a tier; ace access:invite, access:tier (extends a
running tier), access:suspend. The web interface signs in and shows the
day's usage and the Newznab key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
 2026-10-17 00:16:45 +02:00
197 changed files with 2930 additions and 18836 deletions
Expand all files Collapse all files

No files matched your search

.env.example
+5
View File
@@ -19,6 +19,11 @@ DB_USER=usenet_indexer
DB_PASSWORD=
DB_DATABASE=usenet_rebuild
# Access (config/access.ts): open (NEWZNAB_API_KEY or none, no limits) or accounts
# (a Newznab key per user, tiers with daily limits; ace access:invite, access:tier, access:suspend)
# ACCESS=open
# SIGNUP=open|invite|closed (with accounts: invite unless set)
#--------------------------------------------------------------------
# CORS (configure allowed origins for API access)
#--------------------------------------------------------------------
.gitea/workflows/deploy.yaml
+19 -12
View File
@@ -59,28 +59,35 @@ jobs:
node-version-file: .nvmrc
cache: npm
- run: npm ci
- name: Rust toolchain
run: |
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
# The edge worker (edge/, Rust): built first, since the e2e suite runs edge/target/release/edge-node against
# the node API; then its own tests, and the Rust and TypeScript RangeIndexers compared on the committed
# captures (edge/fixtures).
- name: Build the edge worker
run: cargo build --release --manifest-path edge/Cargo.toml --bin edge-node
- name: Type check
run: npx tsc --noEmit
- name: Migrate
run: node ace migration:run --force
- name: Test
run: node ace test
# The edge worker (edge/, Rust): its tests, and the Rust and TypeScript
# RangeIndexers compared on the committed captures (edge/fixtures).
edge:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version-file: .nvmrc
cache: npm
- run: npm ci
- name: Rust toolchain
run: |
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: cargo test
run: cargo test --release --manifest-path edge/Cargo.toml
- name: The Rust indexer over the fixtures
run: edge/bench-fixtures.sh
- name: Rust and TypeScript identical on the fixtures
run: edge/check-fixtures.sh
deploy:
needs: [test]
needs: [test, edge]
if: github.ref == 'refs/heads/rebuild'
runs-on: ubuntu-latest
steps:
Dockerfile
+9 -11
View File
@@ -8,15 +8,6 @@
# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
ARG NODE_VERSION=24.16
# The Rust worker (before the app: its image carries edge-bench). Registry and target dir are build caches, so a rebuild only compiles what changed.
FROM rust:1-bookworm AS edge-build
WORKDIR /src
COPY edge ./
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/src/target \
cargo build --release --locked --bin edge-node --bin edge-bench \
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
# The app, compiled, with production dependencies only.
FROM node:${NODE_VERSION}-bookworm AS app-build
WORKDIR /src
@@ -31,12 +22,19 @@ FROM node:${NODE_VERSION}-bookworm-slim AS app
ENV NODE_ENV=production
WORKDIR /app
COPY --from=app-build /src/build ./
# The Rust indexer's bench: `node ace replay:corpus` runs a group's corpus through it (the replay harness).
COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
USER node
# Every service picks its own command, e.g. `node ace coordinator:run`.
CMD ["node", "ace", "list"]
# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed.
FROM rust:1-bookworm AS edge-build
WORKDIR /src
COPY edge ./
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/src/target \
cargo build --release --locked --bin edge-node --bin edge-bench \
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
FROM debian:bookworm-slim AS edge
COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
REBUILD.md
+18 -649
View File
@@ -5,8 +5,7 @@ rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
decisions D1–D42 are superseded; its domain model still holds, and its
features not yet here are the phase 2 backlog below.
Branch `rebuild`; its successor is designed under *Voyager 3* below (branch
`voyager-3`). Phase 1 finds releases from headers and makes their NZBs.
Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
Everything learned from articles (names from yEnc headers, par2, archive
listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
@@ -60,21 +59,9 @@ most workers on batcave, a few on poweredge-kantoor with
`edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
the cap is shared by the node's processes).
Every group is tiled from article 1 (2026-10-28): one "ancient" range,
status `ancient`, covers 1 … low watermark − 1, what had expired on the
server before the group was planned. Never leased, fetched or filled —
nothing selects it — and the status panel reports it apart ("Ancient: …
articles below the low watermark"); migration 6700 added it for the
groups planned before, `Planner.historyRows` for groups planned since.
Also built (2026-10-09): the live tail (new articles planned every 2
minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
fetched before history; since 2026-10-27 a round grows the group's open
tail range, a planned one at the tail's priority nothing spooled or
took, up to RANGE_ARTICLES before it adds rows, as a tail nothing leased
had become a sliver a group a round, 127k rows of a median 14k articles;
`ranges:merge-tail` packs what such rounds left); the catalog API,
Newznab (`/api`) and the web
fetched before history); the catalog API, Newznab (`/api`) and the web
interface, from voyager 1 (the current build), with categories from the
name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
`database/data/voyager1_signatures.tsv`).
@@ -151,9 +138,7 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
indexed groups in the article's Xref, the one with the lowest priority
number; see the group list) → group articles into files → files
into releases (`lib/release` rules) → drop garbage (random name +
dropped signature or tool), bare executables, dropped posters, and the
single-archive fakes (stub archives and nonsense pieces, below) → set
aside per-article random subjects and
dropped signature or tool) → set aside per-article random subjects and
obfuscated subjects (counted only; phase 2). Returns per batch: release
chunks, edge fragments, probe candidates, counters. Holds no state beyond
its lease.
@@ -204,22 +189,13 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
number; merged when the neighbouring range is done, then written as a
chunk. Never staged per segment.
8. **Settling**. A release is settled when nothing was added to it for 6
hours (30 minutes once its last post is that old: history), and its
completeness decided: complete when every file its [i/N] counter
announced was found. Without a counter, by its par2 sets once their index
is probed (`par2_completeness`): every file a set describes is there by
name or by its first 16 KiB, or, renamed, as many files and bytes; decided
again after a par2 merge and before the pruner drops a set's file list.
Segments aren't compared. Phase 2 picks its work from settled releases.
8. **Settling**. A release is settled when the ranges covering its posting
span are done and no chunk arrived for a day: completeness is computed
then (expected vs found files and segments). Phase 2 picks its work from
settled releases.
9. **NZB and API**. NZB built from a release's chunks, with a known
password in its meta; Newznab search (names, inner names) and get; a web
interface. Categories (Newznab's) by a trigger on releases (migration
5200): from the name, else the inner name the prober found, else a spot's
category (spot_category, set by the spot inspector), else the groups
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
in batches.
9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
search (names) and get. Web UI later.
## Group list
@@ -245,39 +221,6 @@ range with the deployed build's edge-bench (`replay`, ~seconds a range) and
reports suspects and the change since the previous build (`report`). A rule
change is judged on the whole corpus before a reset.
Since 2026-10-23 also history fetched ahead of the indexing: while the
catalog is over its NVMe cap, ranges marked `capture` (priority >= 0, not
leased) are fetched into the corpus by idle loops, at most perGroup /
perRange of a group at a time, to be indexed from disk once there's room
(LOG.md 2026-10-23 (100)). Stored filtered (`CORPUS_FILTER`, default on):
without the lines the edge worker only counts, decided per line as it does
(a crosspost another indexed group owns, a subject that names no file part;
~48% of headers), the chunk compressed again as zlib; beside it a `.drops`
(gzip JSON: rule version, the owners' priorities, per rule the article
numbers as differences, crossposts by owner) and `filtered` in range.json.
The worker makes the same releases of a filtered chunk; rule_counts lack
those two rules' counts. `corpus:filter` filters chunks stored raw. A
corpus filtered this way no longer serves a sweep of those two rules.
Windows by posting time (2026-10-25; voyager 3's test data, LOG.md (104)):
`ranges:dates` samples when each active group's articles were posted (one
XOVER of 32 articles every 500M articles of the group, 16 … 1000 samples,
the median Date of each slice, kept monotone; `group_dates`; over plain
connections: Eweka cuts a compressed reply at the first ".\r\n" in its
output, which some small replies begin with, `TruncatedReply`, LOG.md
2026-10-25 (105)) and estimates
every range's posting date from the curve (`ranges.est_date`; the edges of
a window are off by up to one sample interval, three weeks in boneless).
`ranges:capture --from 2025-01-01 --to 2026-01-01` marks every active
group's ranges posted in the window `capture`: planned ones not started at
`--priority` (0), done ones as they are, both fetched into the corpus by the
fill, which takes capture ranges in priority order, done ones too (they're
fetched again, corpus only), and skips those complete (`ranges.stored_at`,
set when it sees the range.json). A year of every group is the data to
replay voyager 3's 30-day blocks on: marked 2026-10-25, 2016 (priority 0,
30B articles, ~0.56 TB) and 2021 (priority 1, 87B, ~1.6 TB); a recent year
is ~200B articles, ~3.6 TB.
## Phase 2
Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
@@ -295,21 +238,6 @@ files the set describes, into the one with the most files (chunks,
candidates, inspections, counts; the par2's shared name when none was
readable); `release_merges` keeps what went where. Obfuscated uploads whose
files were each a release of their own (`lone` candidates) become one.
Readable files have no hash: a name pass (2026-10-23) joins the releases
of the set's poster in its group posted within 12 hours whose files the
set all names (no case or punctuation; sidecars, par2 and images aside),
as `via = 'name'`: an album's tracks and its index, an index beside its
differently spelled video. `releases:merge-names` for sets from before.
Also merged: lone files whose inner names share a random stem (`inner`),
and small sidecar-only releases into their upload (`sidecar`).
It reads what's new an hour (`created_at`) at a time, its place kept after
each, and merges only the sets held or matched by two releases or more; a
file whose first 16 KiB and size more than 20 files share is a common one and
matches nothing (LOG.md 2026-10-20 (85)).
A set's file list is dropped once it can't merge anything more: complete (all
its files in its holder), judged (every holder junk), or final (every holder
inside a run of done ranges, those ranges' candidates probed, its hashes seen
by the merger); `par2_sets` stays (LOG.md 2026-10-20 (87)).
**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
@@ -324,7 +252,7 @@ with 68,032 releases with a password):
| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed |
| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
@@ -332,11 +260,9 @@ Live indexing (the tail) runs since 2026-10-09.
**Junk** (voyager 1's second filter, D42). After the header pass's rules
(garbage signatures, executables, per-article and obfuscated subjects), the
judge (`releases:judge`, service `judge`) judges releases by what the
prober, the merges and the spots found: every 2 minutes the releases whose
facts changed (noted by triggers in `judge_dirty`), daily every release (for
what time changes: a day after settling, ranges read beside it). In this
order (the first rule wins): *dead* (every probed first article gone), *fake* (a film, episode
judge (`releases:judge`, service `judge`) judges releases hourly by what the
prober, the merges and the spots found, in this order (the first rule
wins): *dead* (every probed first article gone), *fake* (a film, episode
or music name whose content is a program: archive entries all
executables, or the file itself one by name or by its first bytes, sniffed
`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
@@ -386,487 +312,6 @@ processes, one NNTP connection each:
`spot-text`, and the spot), not on the catalog row; the catalog only says
`passworded`.
## Voyager 3: window indexing (design, 2026-10-25)
Branch `voyager-3` (a push to it only runs the tests; `rebuild` keeps
running on homelab until the reset). Goal: the whole history (~1,220 B
articles, 3% indexed on 2026-10-25) indexed and searchable on a 4 TB NVMe
and the 15 TB HDD. The catalog costs 3.9 B of NVMe per article today
(144 GB for 36.5 B), which puts the full history at ~4.7 TB; the target is
≤ 2.5 B per article (~3 TB) with room for the WAL and bloat. Sources:
LOG.md 2026-10-24 (102).
What is wrong today: a release is written as soon as one range sees it,
updated by every later range that finds more parts, merged afterwards
(fragments, par2, names) and settled by wall-clock quiet time, which means
little in a backfill. That churn is most of the WAL (0.6 TB a day) and the
20 k write IOPS bursts, and leaves bloat. Half the rows are single-article
releases, a third of them one poster's image dump (27.7 M rows, LOG.md
(102)).
1. **One cursor a group (amended 2026-10-26; was: blocks).** A group's
history is still fetched as ranges of 2 M articles in chunks of 100 k
(the spool, the corpus, the prefetcher stay) and leased as ranges, but
a group's ranges are indexed in article order by one worker at a time,
and the window below is *carried* from range to range: the next range
finishes what the previous one left open. The first design leased
blocks of 30 days of posting time, ran a group's blocks in parallel and
joined their edges afterwards (a block merger). The user asked why
(LOG.md 2026-10-25 (110)): downloading stays parallel through the
prefetcher and a Rust thread parses faster than a connection delivers,
so one worker a group loses nothing but the end of a reset, when the
last big group runs alone. The merger, the block cutting and the
`blocks` lease go. Costs: one seam a group where the history meets the
tail (the releases in flight then end as two incomplete halves; history
runs forward in one pass a group, priority only between groups), and
the carried state must survive a restart (5 below). Measured on six
consecutive ranges of a.b.ath (2016): one range is 15 hours of posting,
so without the carry no release would finish inside a range (LOG.md
2026-10-26 (111)).
**The group lease (2026-10-26, the user's question in LOG.md (117)):**
a worker thread leases a *group*, not a range, and asks the coordinator
for the group's next range each time it finished one; the coordinator
keeps one cursor a group (`group_leases`: the holder, its heartbeat,
the next range in article order, the snapshot it stands at). The
answer is the range (id, span, resume point), the rules version and the
stream URL — nothing about where the chunks are: the stream serves a
stored range's chunks from the corpus file and a live one's from the
spool, so a worker needs no mount and no corpus layout, and a remote
worker is the same binary with another URL. The overhead is a copy
through the node API: the corpus is ~30 B an article compressed (2016;
16–18 for 2021 and 2025), eight local workers at the bench's rate are
~170 MB/s, which the HDD under the corpus bounds either way; a remote
worker is bounded by its link (100 Mbit ≈ 400k articles/s). History
therefore no longer waits for the corpus fill: a group's stored ranges
are read from disk, an unstored one is fetched live and captured into
the corpus on the way, and the prefetcher runs ahead of each held
group's cursor (plus the capture work left). A group with no planned
range left is the tail: the worker keeps the group and asks again
every minute, the planner's tail ranges fall into its order. A lease
whose holder went quiet for LEASE_TTL_SECONDS is taken over: the new
holder downloads the group's snapshot (5) and redoes the ranges since
it; the batch store's keys make the redo idempotent. The replay
harness (8) stays for offline runs; the driver on homelab goes once the
worker runs.
2. **The window.** The worker reads a block in article order and keeps its
open files and releases in memory (counters, names, sizes; the segment
lists spooled to a file per block on the worker's disk). The *frontier*
is the median Date of the last 1,024 articles with a plausible date
(within 7 days of the frontier; 1,024 such dates in a row that agree
among themselves move it, the group having paused); Date headers alone
aren't trusted (clock skew, bogus years). 1,024, not more: the median
trails the newest article by half the ring, a day in a group of 500
articles a day. A release is **finished** when the frontier is 24 h past
its last part (`WINDOW_MS`): written once, with its completeness decided
then (every announced file there and no file short of parts; par2
decides later for those without a counter), never updated by the header
pass again. Incomplete releases are kept (user, 2026-10-24). A fill
posted after the window is a release of its own (the judge's `repeated`
rule already knows reposts). At a lease's end (the tail's poll, a worker
stopping; amended 2026-10-26, was: the block's end) the releases the
window hasn't closed on are written **provisionally** (`settled_at`
null, `finished: false`, flagged `edge`): a release being uploaded
shows in the catalog, and the carried window finishes the same row
later, since the release key is the identity, the group and the first
article; the store's finish path adds the files not sent before. No
merger: the carry does the joining. The settler goes: `settled_at` is
the finish. The **tail** needs no path of its own: its lease ends leave
such provisional releases, the next lease's continuation finishes them.
One exception (2026-10-27): a provisional release that is complete
(every file its counter announced found) and quiet for six hours is
settled by the coordinator (`Settler.settleComplete`, every five
minutes), as the finish would: no counted file can still come, and the
continuation may be hours away or never (the first run's range leases
left 109k provisional rows at their blocks' end edges, 606 of them
complete, shown as still collecting; the user, release 10697).
3. **Albums.** Files of one poster in one owner group whose subjects differ
only by a number sequence, or share a directory-like prefix, posted
within the window, become one release (`source = 'album'`): its name the
shared template (or the prefix), its files kept by name so each stays
findable; at least two files, else the single stays a release. This is
for subjects that name no release (today's `post`/`stem` singles); a
poster's ten different cracks in a day stay ten releases. The stake,
from voyager 2's final analysis (LOG.md (112)): 77% of its 83.7 M
releases had one file, 50% one segment (0.3% of the segments);
a.b.pictures.erotica.porn alone was 27.7 M single-file releases of one
poster. A counter total shared by many stems of one poster within the
window is a batch counter, not a file count (Yenc-PP-A&A's [n/68067]):
`file_total` null for them, not 68,067 incomplete singles.
**Built 2026-10-26** (edge/src/indexer.rs, LOG.md (113)): a release the
window closed on with one file named by its stem or post, the name with
a number and no archive, volume, sidecar or random name, waits in the
album of its poster and template (the name's digit runs as `#`, the
extension apart: `#_#.jpg.data`) posted within WINDOW_MS of the album's
first file, at most ALBUM_MAX_FILES (1,000) a album; the album closes a
window after that span and is written as one release (`source: album`,
no file total) named by the members' shared numbers (`1481439079025_#
.jpg.data`), a lone member as the release it was. The batch store writes
the members' names as `release_texts` kind 6 (`member`, the first 20).
The directory-like prefix variant is not built (no case seen in the
corpus samples). The batch counter: a total ≥ 100 under which one poster
started 20+ releases within 100 batches gives none of them a file
total. On six a.b.ath ranges of 2016: 46 albums of 790 files among
2,099 chunks, most of them a subject parser miss (`[n/60] Shogun I
(1980) Shogun I (1980).avi.012`: the counter text taken into the name,
a volume each) that the album rule glues back together.
4. **A slimmer catalog.** `releases` loses its text copies: `poster_id` →
`posters(id)` (the table gets an id; 33 B → 4), `group_set_id` →
`group_sets` (the distinct group combinations: 139,927 in voyager 2's
83.7 M releases, LOG.md (112); 85 B jsonb → 4),
`release_key` and `identity_key` as 20-byte `bytea` (41 B each → 20),
`source` a smallint. ~150 of ~430 B a row; the poster and key indexes
shrink with them (~12 GB of 50 today); `releases_category_newest_idx`
(5.3 GB, used once) goes, category browsing filters the newest index.
Today's 85 GB would be ~55. Cold tables move to the HDD tablespace:
`release_inspections` and `probe_candidates` (28 GB, written once, not
read by search); `probe_cache` is there already.
5. **Memory and disk at the edge.** The busiest group posts ~50 M articles
a day; at a file per ~50 articles that is ~1 M open files of ~100 B in
memory per block, and their segment lists (~25 B each) in the block's
spool file, read back when a release is finished. Per worker thread a
few hundred MB at worst. The carried window (amended 2026-10-26) has to
survive a restart (deploys restart the workers daily): its counters and
names are small (kilobytes for most groups, ~100 MB for the busiest),
the open files' segment lists are the bulk (~1.25 GB for a day of the
busiest group). A snapshot at every 2 M range would be ~600 B/article
of writes, so the state is snapshotted at a lease's end and on a timer,
to a file on bulk, and a crash redoes the ranges since the last snapshot
from the corpus or the prefetch cache: the deterministic keys make the
redo idempotent. **The snapshot goes through the API** (2026-10-26):
`PUT /node/groups/:id/snapshot` with the serialized window (the
frontier, the open files and releases, the open albums, the upload
numbers, the batch-counter table; gzip) and the article it stands at
(the last batch's `next`), kept by the coordinator on bulk as
`<snapshot dir>/<group id>`; `GET` gives it back to the next holder. A
worker snapshots when it gives a group back (a stop) and every
SNAPSHOT_MINUTES while it holds one, always right after a batch was
stored, so the snapshot's article is a resume point. The replay
harness (8) keeps a group's window in one process without one.
**Built 2026-10-26** (LOG.md (117)): migration 6500 `group_leases`;
`app/services/group_leases.ts` (lease by the most urgent planned
range within LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY, the longest free
first; takeover after LEASE_TTL_SECONDS, the ranges from the snapshot's
article on planned again; next; heartbeat; release; the snapshot
streamed to `SNAPSHOT_DIR/<group id>.gz`); the node API's
`groups/lease`, `groups/:id/next|heartbeat|release|snapshot`; the
range stream serving a stored range's chunks from the corpus; the
prefetcher ahead of held groups' cursors, skipping chunks the corpus
has; `RangeIndexer::for_group/begin_range/snapshot/restore` (bincode,
the files a table, SNAPSHOT_VERSION 1); `edge-node` holding groups by
default (`--ranges` keeps the old path). Tests: the lease order,
takeover and redo, the endpoints with a corpus-served stream and the
snapshot through the API, the snapshot restoring a window between two
ranges (the two-range release one, the album joined). Deployed and
started 2026-10-26 22:37 (LOG.md (118)). A chunk that doesn't inflate
(a live reply cut short; the spool doesn't check) is reported after
the batch before it went out and the range asked for again from that
chunk: the window stays (the first run lost it, and the open batch).
The lease bounds alone don't select a window: the history planner's
oldest buckets are priority 0–2 too, so the first run also took 2026
history at 0 (25k ranges in 40 groups, fetched live) while 1,739 of
the 2016 corpus ranges sat at voyager 2's leftover priorities (done
when the window was marked, planned again by the reset): hence
`LEASE_CAPTURE_ONLY` (the lease, next and the prefetcher's run-ahead
take capture ranges only) and the one-off repair of the priorities.
The stream polls the corpus as well as the spool while it waits for a
chunk: the corpus fill claims chunks of the same range ahead of the
leased run (the prefetcher's claims are per range and chunk), stores
them in the corpus only, and the run leaves them there; a stream that
looked at the corpus once, seconds before that store, waited its whole
hour on the spool and the worker lost the group and its open batch
(three groups on the first morning). A failed stream now posts the
open batch before the group goes back, like a stop does, so the
window's snapshot survives it; only a batch the coordinator refused
(lease lost, too large) still loses the window.
**Two lanes a group (2026-10-27; migration 6600, LOG.md (119)).** A
group's ranges in article order through one window meant a thread
that leased a group for its tail range first waded through the
group's history (older articles come first), and a held history
group's tail waited behind days of its ranges; and the bounds that ran
the 2016 window alone (0..0, capture only) shut the tail out entirely.
Now `group_leases` has a row per group and *lane*: the `history` lane
works the ranges the bounds hand out (the windows, the history), the
`tail` lane the tail planner's (priority at or below the tail's,
whatever the bounds), each with its own holder, window, snapshot
(`<group>-tail.gz`) and takeover redo (the lane's ranges only, and
only those the window did after the snapshot: leased, done since it,
or read past it). The edge puts `--tail-threads` of its threads on the
tail lane (EDGE_TAIL_THREADS; 1 of 2 or more by default, which is
what homelab's 16 run with), the rest on history; the prefetcher runs ahead of every held
cursor. The seam a group is the same one as before, once: where the
history lane reaches the first range the tail lane did, it is caught
up. **The lease order** within a lane and priority is a mix (the
user, 2026-10-27): one lease starts from the group with the most
planned articles, the next from the longest unheld one (never held
first), alternating on a sequence (`group_lease_turns`), so a big
group snapshotted at a stop doesn't wait behind hundreds of small
never-held groups and a quiet group's tail is visited all the same.
**`coordinator:status`** (`app/services/status.ts`, `--json`) prints
the catalog's room against `CATALOG_MAX_NVME_GB`, the ranges by band
(tail, raised, each capture window, the history by thousand), the
last hour's indexed and stored articles, the spool, the tail backlog,
the held groups with their snapshots' age and size, the snapshot
files, and the phase-2 backlogs (provisional releases, probe
candidates, the pending ladder, the judge's dirty marks). **End to
end** (`tests/e2e`, suite `e2e`, CI builds the binary first): the Rust
worker against the real node API, database, spool and corpus — the
corpus-fill race, a failed stream and the restart that restores the
snapshot, a restart whose snapshot file is gone, a takeover.
6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
(30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
those 73% are random 32-hex tokens, 27% `[PRiVATE]` hashes, ~0.3% posts
with a readable name and no counter. The filter (crosspost, counterless)
loses nothing voyager 3 would index; albums work from filtered chunks
(the image dump's subjects carry "(1/1)"). Indexing without a counter at
all (obfuscated uploads, ~31% of all lines) would need bodies, and the
raw corpus (2,328 ranges) for a sweep.
7. **Where it lives.** The window, blocks and albums are in the Rust edge
(`edge/`), the batch store, the planner and leases; the TypeScript
RangeIndexer keeps the scanner (the corpus filter, tests) and drops its
parity with the Rust worker for the window (the `check.ts` comparison
covered the range path, which goes). New migrations rebuild the schema
from a reset, no in-place conversion of the 81 M rows.
8. **Test before the reset.** Replay the corpus (the raw ranges and the
captures, several B articles) with the new edge into a scratch database
on bulk: NVMe bytes per article (the one number that decides whether
4 TB is enough), the share of albums and incomplete releases, and a
sample of releases against today's. Speed is measured after the reset,
on the NVMe.
9. **The reset.** Tail first, then history by priority, read from the
corpus where stored. Survives: `probe_cache` (HDD, by message-id),
`spots`, `groups` and their priorities, `nntp_shares`, the corpus.
Recomputed: releases, chunks, inspections, candidates, par2 data,
verdicts, file hashes. Resets are paused (2026-10-12); this one waits
for the user's go-ahead, and for the new NVMe.
### Schema (built 2026-10-25; migrations 6000 and 6100)
Migration 6000 *is* the reset: it truncates every table the indexing
recomputes and refuses to run on a catalog with releases unless
`VOYAGER3_RESET=1`, so merging `voyager-3` into `rebuild` is the go-ahead.
Survives: `groups`, `ranges` (planned again from their start, with their
priorities, `capture`, `est_date`, `stored_at`, `plain_chunks` and measured
`posted_low/high`), the corpus, `probe_cache`, `spots` (unlinked from
releases; the inspector matches them again), `nntp_shares`, `nodes`,
`users`, the rule sets.
- `releases`: created anew, 8-byte columns first: `id`, `posted_at`,
`last_posted_at`, `first_article`, `last_article`, `segments`, `bytes`,
`settled_at` (the finish; null while a tail release is provisional),
`inspected_at`, `first_seen_at`; `group_id`, `group_set_id`, `poster_id`,
`file_total`, `files`, `chunks`, `category`; `source smallint`
(`lib/release/source.ts`: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6
album); `readable`, `has_par2`, `complete`, `passworded`, `sidecar_only`,
`edge` (open at its block's end); `release_key` and `identity_key` as
20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`,
`last_seen_at`. Indexes: pkey, release_key unique, (identity_key,
group_id), newest, (poster_id, newest), passworded, `(group_id,
identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS
NULL` for the provisional ones. No trigram indexes: search matches
`release_texts`. The category trigger reads the groups from
`group_sets`; the judge's dirty trigger watches `poster_id` instead of
`poster`.
- `pending_releases` (migrations 6300 and 6400, the HDD): the unnamed
releases outside the catalog (decision f): `release_key` (primary),
`identity_key`, `group_id`, `name` (the random stem), `poster` text, the
posting span, `first_article`, `last_article`, counts, `file_total`,
`partial`, `finished`, `tool`, `candidates` jsonb (the probe targets),
`inspections` jsonb (what each probe showed), `named`, `status` (0
waiting, 1 named and in the catalog, 2 discarded, 3 exhausted, 4 named
awaiting the move, 9 taken), `tries`, `created_at`, `probed_at`; partial
indexes on the waiting (by `created_at`) and the named (by `probed_at`).
- `pending_signatures` (6400): per (tool, group, poster; poster '' for the
tool and group together) `probed`, `named`, `dropped`, `updated_at`.
- `release_texts(release_id, kind smallint, text varchar(255))` (6200),
primary key on all three, one trigram index (fastupdate): every string
search matches against, appended by whoever learns one and never an
update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name
(the release name, stored once more so one index serves everything),
2 file (the prober's names inside an archive), 3 nfo (an NFO's title),
4 par2 (a set's name), 5 media (a container's title); at most 20 per
release and kind (`textsToStore`). A repeat insert is a no-op; the
judge's dirty mark follows an insert or delete, so a rule can react to a
name arriving later. Search: the trigram index yields release ids, joined
to `releases` by primary key (never an OR across the two tables, which
the planner cannot serve from indexes). `inner_name` stays on `releases`
as the one display string. Facets to filter on (resolution, codec,
language) are not texts: columns, or a `meta jsonb` with a
`jsonb_path_ops` index once one is proven useful.
- `posters(id serial, poster unique)` with its trigram index;
`group_sets(id serial, groups jsonb unique)`, a sorted array of names.
- `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume
point), `posted_low/high`, `status`, `priority`, `node_id`, `attempts`,
`rule_version`, `headers`, `releases`, `leased_at`, `heartbeat_at`,
`done_at`, `error`; indexes todo (priority, id) and leased.
`ranges.block_id` points at the block; the ranges' own lease columns stay
until the leases move (then a migration drops them).
- `release_chunks`, `rule_counts`, `rule_samples`: `range_id` → `block_id`.
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
`settled_at IS NOT NULL`), the settler, the fragment merger,
`releases_category_newest_idx`.
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
rule read all of `release_inspections` every round, 17 min on the HDD;
the judge must read it by release or keep what it needs on the NVMe.
### Batch store (built 2026-10-25)
`app/services/batch_store.ts`: every release chunk of a batch is one
release, inserted once. Its keys (`lib/release/keys.ts`): `identity_key`
the identity's SHA-1 as 20 bytes; `release_key` the SHA-1 of the identity,
the owner group and the release's first article, so it is deterministic
(no numbering under a group lock, no catalog lookups before writing) and a
repost, starting at another article, is a release of its own. Its poster
and group set by id (`app/services/catalog_refs.ts`: insert what's new,
read the ids, cache them per process; the tests forget the cache per
test), its source as a code, its name into `release_texts` (kind 1). The
chunk's own counts go in as the release's; `complete` is decided then from
the counter. A finished release's probe candidates are claimable at once
(`ready`, `unreadable`); a provisional tail release (`finished: false` on
the chunk, `settled_at` null) is finished by a later chunk of the same
release key, which adds the files not sent before (sums, one more chunk),
sets `settled_at` and makes its candidates claimable; a finished release
sent again is left as it is (`ON CONFLICT … WHERE settled_at IS NULL`).
Gone: the join of a chunk into an earlier release of its key (the window
keeps an upload together; the block merger joins what a block's end cut),
the join across groups (FTD's spread uploads: a gap, to be handled by the
block merger or the par2 merger across groups), the numbering of release
keys, the group lock, `fragments` (ignored when sent; the Rust edge still
sends them). The chunk type carries `finished` and `edge` (both optional;
the Rust edge doesn't send them yet, so its releases are finished, not
edges). Interim until blocks are leased: chunks, rule counts and samples
are keyed by `ranges.block_id`, and a range without a block gets a block
of its own in the store (`blockOf`).
Readers moved with it: search joins `posters` and `group_sets`, matches
words in `release_texts` (`r.id IN (SELECT release_id … ILIKE)`), searches
posters by id, filters `source` by code and returns the source name; the
par2 merger, the sidecar folder and the judge's scraped rule use
`poster_id` (`release_merges.from_key` holds the key as hex); the spot
inspector matches by bytea identity keys and inserts its spot releases by
id references with their title in `release_texts`; the spotter and the NZB
service compare the source code. Tests insert fixtures through
`tests/functional/helpers/releases.ts` (`insertRelease`: text keys, poster
and groups by name, source by name). The unit and functional suites pass.
### The window in the Rust edge (built 2026-10-25)
`edge/src/indexer.rs`: the scanner and the file aggregator as before; then
the window instead of the range path. `Frontier` keeps a ring of the last
`FRONTIER_ARTICLES` (1,024) plausible dates (within `PLAUSIBLE_MS`, 7
days, of the frontier; 1,024 implausible dates in a row that agree move
it), recomputed once per chunk; established after 64 dated articles
(`settle` at the block's end for smaller blocks); its first value is the
block's start in posting time. Per chunk: completed files, then the files
whose last part is before frontier − `WINDOW_MS` (24 h; undated ones idle
for `idle` chunks), go into the open release of their key whose posting
span is within `UPLOAD_GAP_MS` (3 h) of theirs, else start one (the
upload number per key, distinct within a batch); releases whose last part
is before frontier − `WINDOW_MS` are judged and written `finished`, as an
`edge` when posted within `WINDOW_MS` of the block's start. At the end
(`finish`) everything left is written: finished when the window closed on
it, else `finished: false, edge: true`. A file short of parts stays in
its release and sets `partial` (the store: `complete = false`). A lone
article of a multi-part random file is still counted `per-article`, not
written; before the block's end a scrambled name counts as random too
(`looks_scrambled`: a random subject and poster per article, 160k of
alt.binaries.storage's releases; LOG.md (122)).
**Single-archive rules (built 2026-10-31; LOG.md (123)-(127)).** The judge
(`names::judge`, `RuleSet.judge`) gets each file's articles seen and their
bytes, and whether the release is *settled* (the window closed on it: a
par2 can still join an open one; the start edge doesn't count, since a par2
follows its rar and 7% of the catalog's releases are start edges). A settled
release of one unsplit zip, rar or 7z and nothing else is dropped as a
**stub archive** (`stub-archive`, key: the legs tripped) when posted to
2+ groups, under 100 MB (the fakes stop at ~90 MB; hash-named uploads,
real albums and passworded videos above it don't), and (A) its articles
are ≤ 256 KiB with the release ≥ 1 MB (Power-Post's 250 kB default; a
one-article release always has small articles, which exempts e-books and
sidecars), or (X) posted to 4+ groups, or (W) by a webmail poster. A
ROT13 volume name (".cnegN.rar" is ".partN.rar": a.b.erotica's passworded
videos, LOG.md (128)) is a volume, not a single archive. A release
touching a music, e-book, comics, tabs or fonts group is exempt: those
communities post real single small archives the same way (karaoke zips to
a.b.sounds.karaoke + a.b.test with Power-Post's articles; 16 of 32,308
known fakes lost, LOG.md (127)). These are the
exe-stub and recycled-installer floods under film and software names:
55% of the catalog on 2026-10-30, 99.3% of the known fakes, 0 known
content in the 217k sample, verified on 1,500 decoded articles. A settled
single archive of one article and 100 kB–1 MB under a release-like,
unscrambled name without spaces and without an e-book format in it is
dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool):
the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of
the catalog; under 100 kB are cue/log sidecars and adverts, and the names
with spaces are books and albums. Both rules are switched by the
rule set (`dropStubArchives`, `dropNonsensePieces`), sampled like every
drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
judges as each batch completes files, so it passes settled without the
window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now),
`check.ts` and the fixture comparison (the parity with the TypeScript
indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
(incomplete releases are kept). The wire carries `finished`, `edge` and
`partial`; the store reads them (`partial` into `complete`). Memory: the
window holds a day of a group's segments in memory (the busiest group's
~50 M); the spool file per block of the design is not built yet.
### The replay harness (built 2026-10-26)
`node ace replay:corpus --group <name> [--dir CORPUS_DIR] [--from article]
[--to article] [--limit ranges] [--batch-chunks 5] [--no-finish]` (commands/
replay_corpus.ts; design 8): a group's complete corpus ranges in article
order through one window in the Rust edge (`edge-bench --group <dir>
--rules rules.json --wire -`: the window carried from range to range, the
end finished after the last range; `edge/src/lease.rs` has the lease JSON
the worker and the bench share) and the batch store into a scratch
catalog, as the worker would store them: each range leased to a node named
`replay`, its batches stored at its resume point (pulled from the bench's
stdout as the store keeps up), the range finished when its last batch is
in. It refuses a database not named `*_replay` or `*_scratch`.
The rules are the scratch catalog's current set and group priorities:
copy `groups` and `rule_sets` from the catalog for a faithful replay. The
report: releases finished, provisional, edges; complete, incomplete,
undecided; readable, with par2; files, segments as a share of the
headers; bytes per header of the NVMe tables (releases, release_texts,
posters, group_sets, rule counts and samples) and of those on
CHUNK_TABLESPACE (release_chunks, probe_candidates, release_inspections).
The app image carries `edge-bench` for it (Dockerfile); on homelab the
scratch catalog is a second database (`usenet_replay`, default tablespace
`nvme` like the catalog, the chunk tables on `pg_default`; migrated with
the voyager-3 image built from the deploy cache's repo, `groups` and
`rule_sets` copied over), the corpus mounted read-only.
`deploy/replay-window.sh` runs a window there: each group whose window
ranges are all stored is replayed once with the window's bounds as
`--from`/`--to` (the corpus holds other windows of the same group), 4
groups at a time, logs and `.done` markers in `.cache/replay/`, looping
until the window is done (LOG.md 2026-10-26 (111) has the first local
numbers, (112) the homelab run). **Deployed 2026-10-26** (LOG.md (116)):
voyager 3 runs on homelab without its edge worker (`.cache/no-edge`:
deploy.sh and the nightly resume leave it stopped, since range by range
it would split releases at every edge with no merger left), and history
comes from the corpus through this driver into the catalog itself
(`DB=usenet_rebuild OUT=.cache/replay-catalog`, the harness allowed by
`REPLAY_INTO_CATALOG=1`), the prober's ladder, the merger and the judge
working on it as it fills; the scratch run's numbers (63 groups) were
read before (LOG.md (115)). The tail waits for the window worker.
Next: the replay on homelab over the 2016 window (the NVMe bytes per
header for real), the window worker (design 1 and 5: group leases, the
stream serving stored chunks, the snapshot through the API; being built,
LOG.md (117)), the prober writing `release_texts` kind 2, the spot/NZB
match on pending keys (decision f), the tail's provisional writes at a
range end (the window's open releases shown while they upload: the files
sent provisionally marked, the finish sending the rest). Gone from the list: block
leases, the planner and the block merger (design 1, amended).
## Decisions
a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
@@ -876,84 +321,8 @@ c. Group list: editable, owner by priority number (above). **Accepted.**
d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
e. Headers through the coordinator's relay, with prefetch (part 1).
**Accepted.**
f. **Nothing enters the catalog without a usable name** (amended
2026-10-26, the user's direction; before: random-named releases keep
no chunk but their rows). A release the window closed on whose files
all have random names (`isUnreadableName`, `names::is_unreadable_name`:
a name read token by token, so separators, spaces and random
"extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
(migration 6300, the HDD) with its keys, counts, article span, upload
tool and probe targets, no row in `releases`, `release_texts` or
`posters`, no chunk. The pending ladder works them **from least to most
effort**: (0) no fetch: a signature verdict (tool and group, poster)
learned from earlier probes, a spot or NZB that names the message-ids;
(1) a par2 main file's first article (it names the files); (2) the first
article of the first file (the yEnc name, an archive's listing in its
first 16 KiB); (3) a 7z's tail; then nothing more. A probe that yields
a usable name moves the release into the catalog, its segments read
again from the corpus (the range stored) or the server; one that
yields none counts against its signature, and a signature with
≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
2026-10-08 (5)) is **discarded** from then on, a sample of it still
probed to notice a change. **By class** (2026-11-02, LOG.md (130)): a
pending release's class is its tool, group, size bucket, name shape and
partial flag (`lib/release/classes.ts`, `names::pending_class`, the SQL
`pending_class()`; `pending_releases.klass`), and `pending_classes` keeps
each class's climbs and names; the claim takes the classes with the best
record first (`classScore`, a prior for the unexplored, at most a quarter
of a claim from one class), the cheapest target article first within one
(`target_bytes`), and the oldest of the rest; a class with the learner's
thresholds is dropped like a signature, and the rule set's
`droppedClasses` keep its releases out of the pending store at the edge,
the explored sample excepted. A probed file whose first 16 KiB and size
match a named catalog release's (`file_hashes`) takes that name without
another fetch (`app/services/hash_twins.ts`), on the ladder and for
catalog releases without one. The reason, in the 2016 replay: one poster's
hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
groups) were 76% of the rows and three quarters of the NVMe bytes for
11% of the segments and nothing anyone can find; the full history has
725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
edge sends an unnamed release with an empty payload and its tool, the
batch store writes `pending_releases` (LOG.md (114)); the ladder in the
prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
the oldest waiting releases' next target (par2, archive, lone, 7z-tail
in that order), at least PENDING_SHARE (25%) of each claim next to the
catalog's own candidates, answered from `probe_cache` when the article
was read before; `usableName` (lib/probe/inspect.ts) takes the largest
named file of a par2 set or a listing, or a lone file's yEnc name, that
is neither packaging nor random; a name sets status 4, no name the next
target or status 3 (exhausted); each finished climb counts on
`pending_signatures` for (tool, group, poster) and (tool, group), and a
signature with PENDING_MIN_PROBES (100) climbs and at most
PENDING_MAX_NAMED (0.5%) named is dropped: every five minutes the
waiting releases of dropped signatures are discarded (status 2), the
rules' `samplePercent` of them left to climb. The namer
(`app/services/pending_namer.ts`, in the prober's process): a named
release's article span read again from the corpus (the stored range's
chunks) or by XOVER on a connection of its own, through the TypeScript
indexer with `partialFiles` (a file given up short of parts stays a
partial file of its release, as the Rust window writes it; without it
the range path's fragment rule left nothing to find: by 2026-10-29
about 52k of 210k named releases failed, all partial), written as a release under the pending key with `source`
weak-stem, the name as `inner_name`, its chunk and its candidates (the
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
the password flag). Not built: the spot/NZB match on pending keys (step
0 of the ladder beyond signatures), and a signature by poster alone
across groups.
g. The account's NNTP connections shared by priority (`nntp_shares`): the
coordinator (headers) before the prober, each between its minimum and
maximum, 5 kept for the spotter, the spot inspector, the plain
connection and reconnects. A process grows only into connections the
others closed, so the account never goes over its limit. Replaces the
fixed split (38 + 8), under which the prober stayed at 8 while history
was paused and the coordinator's connections idle (LOG.md 2026-10-18
(75)). **Accepted** 2026-10-18.
h. Idle NNTP connections closed: one with nothing to do for
`NNTP_IDLE_CLOSE_SECONDS` (60) is closed and opened again with work
(the coordinator's pool on its next sync, the plain one too; a dry
prober lane; the spotter between rounds; the spot inspector with no
spots). Eweka dropped the quiet ones and the next command failed
("Spot inspection failed: NNTP connection closed"), and idle ones took
the account's connections from downloads. **Accepted** 2026-10-22.
f. Random-named releases (no file with a readable name) keep no chunk in
the header pass: their file keys and probe candidates only, until
phase 2 names them (bodies) and fetches their segments again. The full
history is ~3 TB with them, ~1.2–1.4 TB without (725 M of the 1.25 B
kept files; LOG.md 2026-10-09). **Accepted** 2026-10-09, not built yet.
adonisrc.ts
-6
View File
@@ -91,12 +91,6 @@ export default defineConfig({
name: 'functional',
timeout: 30000,
},
// The Rust edge worker against the real node API (tests/e2e): needs edge/target/release/edge-node.
{
files: ['tests/e2e/**/*.spec.{ts,js}'],
name: 'e2e',
timeout: 120000,
},
],
forceExit: false,
},
app/controllers/new_account_controller.ts
+26 -3
View File
@@ -1,13 +1,36 @@
import User from '#models/user'
import db from '@adonisjs/lucid/services/db'
import { signupValidator } from '#validators/user'
import type { HttpContext } from '@adonisjs/core/http'
import UserTransformer from '#transformers/user_transformer'
import AccessService from '#services/access_service'
class InviteGone extends Error {}
export default class NewAccountController {
async store({ request, serialize }: HttpContext) {
const { fullName, email, password } = await request.validateUsing(signupValidator)
async store({ request, response, serialize }: HttpContext) {
const { fullName, email, password, invite } = await request.validateUsing(signupValidator)
if (AccessService.signup === 'closed') return response.forbidden({ error: 'Signup is closed' })
if (AccessService.signup === 'invite' && !invite) return response.forbidden({ error: 'An invite code is needed to sign up' })
const user = await User.create({ fullName, email, password })
// The user and their invite together: an invite that's gone rolls the signup back.
const user = await db
.transaction(async (trx) => {
const created = await User.create({ fullName, email, password }, { client: trx })
if (AccessService.signup !== 'invite') return created
const gives = await AccessService.redeem(invite!, created.id, trx)
if (!gives) throw new InviteGone()
if (gives.tier) {
AccessService.grantTier(created, gives.tier, gives.tier_days)
await created.useTransaction(trx).save()
}
return created
})
.catch((error) => {
if (error instanceof InviteGone) return null
throw error
})
if (!user) return response.forbidden({ error: 'Unknown or used invite code' })
const token = await User.accessTokens.create(user)
return serialize({
app/controllers/newznab_controller.ts
+38 -13
View File
@@ -2,6 +2,7 @@ import type { HttpContext } from '@adonisjs/core/http'
import env from '#start/env'
import NzbService from '#services/nzb_service'
import SearchService from '#services/search_service'
import AccessService, { AccessDenied } from '#services/access_service'
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
const LIMITS = { max: 100, default: 50 }
@@ -35,22 +36,37 @@ function int(value: unknown): number | undefined {
* Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
*/
export default class NewznabController {
async handle({ request, response }: HttpContext) {
async handle(ctx: HttpContext) {
const xml = (body: string, type = 'application/xml') => {
ctx.response.header('Content-Type', `${type}; charset=utf-8`)
return ctx.response.send(body)
}
try {
return await this.answer(ctx, xml)
} catch (error) {
if (error instanceof AccessDenied) return xml(errorXml(error.newznabCode, error.message))
throw error
}
}
private async answer(ctx: HttpContext, xml: (body: string, type?: string) => void) {
const { request, response } = ctx
const qs = request.qs()
const t = String(qs.t ?? '')
const server = this.server(request)
const xml = (body: string, type = 'application/xml') => {
response.header('Content-Type', `${type}; charset=utf-8`)
return response.send(body)
}
const apiKey = configuredApiKey()
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
if (t === 'caps') return xml(capsXml(server, LIMITS))
// With accounts the key is a user's (AccessService); else the one configured, if any.
const grant = await AccessService.authorize(ctx, qs.apikey)
const apiKey = grant ? grant.user.apiKey : configuredApiKey()
if (!grant && apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
const server = this.server(request, apiKey)
if (t === 'caps') return xml(capsXml(server, LIMITS, { available: AccessService.accounts && AccessService.signup !== 'closed', open: AccessService.accounts && AccessService.signup === 'open' }))
if (t === 'get') {
const id = int(qs.id)
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
// Unknown ids and releases past the tier's retention aren't counted.
if (!(await AccessService.checkRelease(grant, id))) return xml(errorXml(300, 'No such item'))
await AccessService.charge(grant, 'grab')
const nzb = await NzbService.build(id)
if (!nzb) return xml(errorXml(300, 'No such item'))
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
@@ -59,6 +75,7 @@ export default class NewznabController {
}
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
await AccessService.charge(grant, 'api')
const requested = String(qs.cat ?? '')
.split(',')
@@ -79,7 +96,7 @@ export default class NewznabController {
categories,
readable: true,
notIncomplete: true,
maxAgeDays: int(qs.maxage),
maxAgeDays: AccessService.maxAgeDays(grant, int(qs.maxage)),
season: int(qs.season),
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
sort: 'newest',
@@ -105,12 +122,20 @@ export default class NewznabController {
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
}
})
return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml')
// What's left of the day, for clients that show it (newznab:apilimits).
const limits = grant && {
apiCurrent: grant.usage.apiHits,
apiMax: grant.limits.apiHits,
grabCurrent: grant.usage.grabs,
grabMax: grant.limits.grabs,
}
return xml(resultsXml(server, items, offset, result?.total ?? 0, limits || undefined), 'application/rss+xml')
}
private server(request: HttpContext['request']): NewznabServer {
/** Download links carry the caller's key. */
private server(request: HttpContext['request'], apiKey: string | undefined): NewznabServer {
const proto = request.header('x-forwarded-proto') ?? 'http'
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() }
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey }
}
}
app/controllers/node_controller.ts
+13 -107
View File
@@ -1,15 +1,17 @@
import type { HttpContext } from '@adonisjs/core/http'
import { Readable } from 'node:stream'
import { setTimeout as sleep } from 'node:timers/promises'
import db from '@adonisjs/lucid/services/db'
import Leases, { LeaseLost } from '#services/leases'
import GroupLeases, { LANES } from '#services/group_leases'
import BatchStore, { StaleBatch } from '#services/batch_store'
import Spool, { type ChunkSpan } from '#services/spool'
import Corpus from '#services/corpus'
import coordinatorConfig from '#config/coordinator'
import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
import Spool from '#services/spool'
import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
/**
* How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
* coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
* (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
*/
const CHUNK_WAIT_MS = 60 * 60 * 1000
/** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
const KEEPALIVE_MS = 20_000
@@ -28,109 +30,18 @@ export default class NodeController {
return lease
}
// Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
// ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
// `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
/** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
private lane(ctx: HttpContext): Lane | null {
const lane = ctx.request.qs().lane ?? 'history'
if (LANES.includes(lane)) return lane
ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
return null
}
async leaseGroup(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
const lease = await GroupLeases.lease(this.node(ctx), lane)
if (!lease) return ctx.response.noContent()
return lease
}
async nextRange(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
try {
const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
if (!next) return ctx.response.noContent()
return next
} catch (error) {
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
throw error
}
}
async groupHeartbeat(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
}
async releaseGroup(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
}
/**
* PUT /groups/:id/snapshot?next=N: the window's serialized state standing at article N, streamed to disk as sent
* (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
*/
async putSnapshot(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
const next = Number(ctx.request.qs().next)
if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
const raw = ctx.request.raw()
const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
}
async getSnapshot(ctx: HttpContext) {
const lane = this.lane(ctx)
if (!lane) return
try {
const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
ctx.response.header('Content-Type', 'application/octet-stream')
ctx.response.header('Content-Length', String(snapshot.size))
ctx.response.header('X-Snapshot-Next', String(snapshot.next))
return ctx.response.stream(snapshot.stream)
} catch (error) {
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
throw error
}
}
/**
* The range's chunks from its resume point: a stored range's from the corpus (a worker needs no mount; REBUILD.md
* design 1), else from the spool, waiting for those still being fetched.
*/
/** The range's chunks from its resume point, waiting for those still being fetched. */
async stream(ctx: HttpContext) {
const nodeId = this.node(ctx)
const rangeId = Number(ctx.params.id)
const range = await this.leased(ctx, rangeId, nodeId)
if (!range) return
const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
// The chunk from the corpus or the spool, whichever has it. Both are polled while waiting: the corpus fill
// claims a chunk ahead of the leased run (the prefetcher's claims are per range and chunk), stores it in the
// corpus only, and the run then leaves it there — a stream that looked at the corpus once, before that
// store, waited the whole hour on the spool (three groups on 2026-10-27, LOG.md (118)).
const stored = async (span: ChunkSpan) =>
corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
async function* frames() {
for (const span of spans) {
const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
const waitUntil = Date.now() + CHUNK_WAIT_MS
let quietSince = Date.now()
let data = await stored(span)
while (!data) {
if (await Spool.has(rangeId, span)) {
data = await Spool.read(rangeId, span)
break
}
while (!(await Spool.has(rangeId, span))) {
if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
if (Date.now() - quietSince >= KEEPALIVE_MS) {
quietSince = Date.now()
@@ -138,9 +49,8 @@ export default class NodeController {
await Leases.heartbeat(rangeId, nodeId)
}
await sleep(500)
data = await stored(span)
}
yield encodeFrame({ ...span, data })
yield encodeFrame({ ...span, data: await Spool.read(rangeId, span) })
// Streaming keeps the lease alive.
await Leases.heartbeat(rangeId, nodeId)
}
@@ -165,11 +75,7 @@ export default class NodeController {
}
async finish(ctx: HttpContext) {
return this.lifecycle(ctx, async (rangeId, nodeId) => {
const range = await Leases.leased(rangeId, nodeId)
await Leases.finish(rangeId, nodeId)
await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
})
return this.lifecycle(ctx, (rangeId, nodeId) => Leases.finish(rangeId, nodeId))
}
/** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */
app/controllers/profile_controller.ts
+12 -1
View File
@@ -1,8 +1,19 @@
import UserTransformer from '#transformers/user_transformer'
import type { HttpContext } from '@adonisjs/core/http'
import AccessService from '#services/access_service'
export default class ProfileController {
/** The user, their tier, Newznab key and what they used today. */
async show({ auth, serialize }: HttpContext) {
return serialize(UserTransformer.transform(auth.use('api').getUserOrFail()))
const user = auth.use('api').getUserOrFail()
return serialize({ user: UserTransformer.transform(user), access: await AccessService.summary(user) })
}
/** POST /api/v1/account/api-key: a new Newznab key (the old one stops working). */
async resetApiKey({ auth }: HttpContext) {
const user = auth.use('api').getUserOrFail()
user.apiKey = AccessService.newKey()
await user.save()
return { apiKey: user.apiKey }
}
}
app/controllers/releases_controller.ts
+24 -8
View File
@@ -1,5 +1,6 @@
import type { HttpContext } from '@adonisjs/core/http'
import NzbService from '#services/nzb_service'
import AccessService from '#services/access_service'
import SearchService, { type SortOrder } from '#services/search_service'
import { expandCategories } from '#lib/newznab/index'
@@ -14,13 +15,16 @@ function megabytes(value: unknown): number | undefined {
/** The catalog for the web interface (REBUILD.md, part 9). */
export default class ReleasesController {
/** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
async index({ request }: HttpContext) {
const qs = request.qs()
async index(ctx: HttpContext) {
const qs = ctx.request.qs()
const grant = await AccessService.authorize(ctx, qs.apikey)
await AccessService.charge(grant, 'api')
const categories = String(qs.cat ?? '')
.split(',')
.map((c) => Number(c.trim()))
.filter((c) => Number.isInteger(c) && c > 0)
return SearchService.search({
maxAgeDays: AccessService.maxAgeDays(grant),
q: typeof qs.q === 'string' ? qs.q : undefined,
categories: categories.length ? expandCategories(categories) : undefined,
readable: flag(qs.readable),
@@ -39,13 +43,20 @@ export default class ReleasesController {
}
/** GET /api/v1/releases/:id, with its files */
async show({ params, response }: HttpContext) {
const release = await SearchService.release(Number(params.id))
return release ?? response.notFound({ error: 'Release not found' })
async show(ctx: HttpContext) {
const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
if (!(await AccessService.checkRelease(grant, Number(ctx.params.id)))) return ctx.response.notFound({ error: 'Release not found' })
await AccessService.charge(grant, 'api')
const release = await SearchService.release(Number(ctx.params.id))
return release ?? ctx.response.notFound({ error: 'Release not found' })
}
/** GET /api/v1/releases/:id/nzb */
async nzb({ params, response }: HttpContext) {
/** GET /api/v1/releases/:id/nzb (?apikey= for a plain link, as the web interface's) */
async nzb(ctx: HttpContext) {
const { params, response } = ctx
const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
if (!(await AccessService.checkRelease(grant, Number(params.id)))) return response.notFound({ error: 'Release not found' })
await AccessService.charge(grant, 'grab')
const nzb = await NzbService.build(Number(params.id))
if (!nzb) return response.notFound({ error: 'Release not found' })
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
@@ -54,7 +65,12 @@ export default class ReleasesController {
return response.send(nzb.xml)
}
/** GET /api/v1/stats */
/** GET /api/v1/access: whether the web interface needs an account, and how one is made. */
async access() {
return { mode: AccessService.mode, signup: AccessService.signup }
}
/** GET /api/v1/stats (open: what the index holds, not what's in it) */
async stats() {
return SearchService.stats()
}
app/exceptions/handler.ts
+3
View File
@@ -8,6 +8,9 @@ export default class HttpExceptionHandler extends ExceptionHandler {
*/
protected debug = !app.inProduction
/** A refused request (AccessService: no key, a limit reached) is an answer, not a fault. */
protected ignoreCodes = ['E_ACCESS_DENIED']
/**
* The method is used for handling errors and returning
* response to the client
app/models/user.ts
+8
View File
@@ -1,4 +1,6 @@
import { randomBytes } from 'node:crypto'
import { UserSchema } from '#database/schema'
import { beforeCreate } from '@adonisjs/lucid/orm'
import hash from '@adonisjs/core/services/hash'
import { compose } from '@adonisjs/core/helpers'
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
@@ -8,6 +10,12 @@ export default class User extends compose(UserSchema, withAuthFinder(hash)) {
static accessTokens = DbAccessTokensProvider.forModel(User)
declare currentAccessToken?: AccessToken
/** The Newznab key (AccessService), known on the model right away rather than from the column's default. */
@beforeCreate()
static assignApiKey(user: User) {
user.apiKey ??= randomBytes(16).toString('hex')
}
get initials() {
const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
if (first && last) {
app/services/access_service.ts
+171
View File
@@ -0,0 +1,171 @@
import { randomBytes } from 'node:crypto'
import { DateTime } from 'luxon'
import db from '@adonisjs/lucid/services/db'
import { Exception } from '@adonisjs/core/exceptions'
import type { HttpContext } from '@adonisjs/core/http'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import User from '#models/user'
import { ACCESS, SIGNUP, TIERS, type TierLimits } from '#config/access'
export type Use = 'api' | 'grab'
/** Newznab's error codes for access (the catalog answers with the HTTP status). */
const STATUS = { 100: 401, 101: 403, 102: 403, 500: 429, 501: 429 } as const
/** Why a request is refused; the catalog's answer (Newznab catches it for its own). */
export class AccessDenied extends Exception {
constructor(
readonly newznabCode: keyof typeof STATUS,
message: string
) {
super(message, { status: STATUS[newznabCode], code: 'E_ACCESS_DENIED' })
}
async handle(error: this, ctx: HttpContext) {
ctx.response.status(error.status).send({ error: error.message, code: error.newznabCode })
}
}
export interface Usage {
apiHits: number
grabs: number
}
/** A caller let in: who, their tier as it is now, and what they used today. */
export interface Grant {
user: User
tier: string
limits: TierLimits
usage: Usage
}
const COLUMNS: Record<Use, string> = { api: 'api_hits', grab: 'grabs' }
const TODAY = `(now() AT TIME ZONE 'UTC')::date`
const DAY_MS = 86_400_000
/** Accounts, tiers and their daily limits (config/access.ts). */
export default class AccessService {
/** From config/access.ts; tests switch it. */
static mode: 'open' | 'accounts' = ACCESS
static signup: 'open' | 'invite' | 'closed' = SIGNUP
static get accounts() {
return this.mode === 'accounts'
}
static newKey() {
return randomBytes(16).toString('hex')
}
/** The user's tier as it is now: one that ran out (or isn't configured) is free. */
static tierOf(user: User): { tier: string; limits: TierLimits } {
const expired = user.tierUntil !== null && user.tierUntil.toMillis() <= Date.now()
const tier = !expired && TIERS[user.tier] ? user.tier : 'free'
return { tier, limits: TIERS[tier] }
}
/** The user a request names: the web interface's token, else an apikey (Newznab, the catalog's NZB links). */
static async caller(ctx: HttpContext, apikey?: unknown): Promise<User | null> {
const guard = ctx.auth.use('api')
if (guard.isAuthenticated || (await guard.check())) return guard.user!
if (typeof apikey === 'string' && /^[0-9a-f]{32}$/.test(apikey)) return User.findBy('apiKey', apikey)
return null
}
/** Lets a request in (counting nothing yet); null when access is open. */
static async authorize(ctx: HttpContext, apikey?: unknown): Promise<Grant | null> {
if (!this.accounts) return null
const user = await this.caller(ctx, apikey)
if (!user) throw new AccessDenied(100, 'Incorrect user credentials')
if (user.suspended) throw new AccessDenied(101, 'Account suspended')
return { user, ...this.tierOf(user), usage: await this.usage(user.id) }
}
/** Counts one use against the grant's tier, or refuses it once the day's limit is reached. */
static async charge(grant: Grant | null, use: Use) {
if (!grant) return
const usage = await this.count(grant.user.id, use, use === 'api' ? grant.limits.apiHits : grant.limits.grabs)
if (!usage) throw use === 'api' ? new AccessDenied(500, 'Request limit reached') : new AccessDenied(501, 'Download limit reached')
grant.usage = usage
}
/**
* Adds one use to today's row unless the limit is reached (null then): one
* statement, so concurrent requests can't both take the last one.
*/
static async count(userId: number, use: Use, limit: number | null): Promise<Usage | null> {
if (limit !== null && limit <= 0) return null
const column = COLUMNS[use]
const { rows } = await db.rawQuery(
`INSERT INTO usage (user_id, day, ${column}) VALUES (?, ${TODAY}, 1)
ON CONFLICT (user_id, day) DO UPDATE SET ${column} = usage.${column} + 1
${limit === null ? '' : `WHERE usage.${column} < ?`}
RETURNING api_hits, grabs`,
limit === null ? [userId] : [userId, limit]
)
return rows[0] ? { apiHits: rows[0].api_hits, grabs: rows[0].grabs } : null
}
static async usage(userId: number): Promise<Usage> {
const { rows } = await db.rawQuery(`SELECT api_hits, grabs FROM usage WHERE user_id = ? AND day = ${TODAY}`, [userId])
return { apiHits: rows[0]?.api_hits ?? 0, grabs: rows[0]?.grabs ?? 0 }
}
/** A search's age bound: the asked one within the tier's retention. */
static maxAgeDays(grant: Grant | null, requested?: number): number | undefined {
const retention = grant?.limits.retentionDays
if (!retention) return requested
return requested ? Math.min(requested, retention) : retention
}
/** Refuses a release posted before the tier's retention (one without a date is let through). */
static checkPosted(grant: Grant | null, postedAt: Date | string | null) {
const retention = grant?.limits.retentionDays
if (!retention || !postedAt) return
if (new Date(postedAt).getTime() < Date.now() - retention * DAY_MS) {
throw new AccessDenied(102, `Older than your tier's retention (${retention} days)`)
}
}
/** Whether a release exists, refusing it when it's older than the tier sees; asked before an NZB is counted. */
static async checkRelease(grant: Grant | null, id: number): Promise<boolean> {
const release = await db.from('releases').where('id', id).select('posted_at').first()
if (!release) return false
this.checkPosted(grant, release.posted_at)
return true
}
/** Takes an unused invite for a new user: what it gives, or null when it's unknown or used. */
static async redeem(code: string, userId: number, trx: TransactionClientContract) {
const { rows } = await trx.rawQuery(
`UPDATE invites SET used_by = ?, used_at = now() WHERE code = ? AND used_by IS NULL AND used_at IS NULL RETURNING tier, tier_days`,
[userId, code]
)
return (rows[0] as { tier: string | null; tier_days: number | null } | undefined) ?? null
}
/**
* Gives a user a tier for `days` (null: without end). The same tier still
* running is extended from where it ends, so a renewal paid early loses nothing.
*/
static grantTier(user: User, tier: string, days: number | null) {
if (!TIERS[tier]) throw new Error(`No such tier: ${tier} (config/access.ts has ${Object.keys(TIERS).join(', ')})`)
const now = DateTime.utc()
const running = user.tier === tier && user.tierUntil && user.tierUntil > now ? user.tierUntil : now
user.tier = tier
user.tierUntil = days === null ? null : running.plus({ days })
}
/** What a profile shows of the account's access. */
static async summary(user: User) {
const { tier, limits } = this.tierOf(user)
return {
mode: this.mode,
tier,
tierUntil: tier === user.tier ? user.tierUntil : null,
apiKey: user.apiKey,
limits,
usage: await this.usage(user.id),
}
}
}
app/services/batch_store.ts
+176 -296
View File
@@ -1,58 +1,24 @@
import db from '@adonisjs/lucid/services/db'
import { pruneSevenZipTails } from '#services/seven_zip_tails'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import ChunkPartitions from '#services/chunk_partitions'
import CatalogRefs, { groupSetKey, posterText } from '#services/catalog_refs'
import { decodeChunk } from '#lib/chunks/index'
import { SIDECAR } from '#services/sidecar_folder'
import { identityKey, releaseKey } from '#lib/release/keys'
import { sourceCode } from '#lib/release/source'
import { uploadTool } from '#lib/rules/index'
import { byEffort } from '#services/pending_ladder'
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
/**
* A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
* fold's candidates (releases.sidecar_only). Its file list decoded here, small as it is.
*/
function sidecarOnly(r: ReleaseChunk): boolean | null {
if (r.files > 3 || r.bytes >= 1_000_000) return null
const files = decodeChunk(r.payload)
return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
}
export class StaleBatch extends Error {}
export interface StoredBatch {
/** Release chunks in the batch. */
releases: number
/** Releases the batch started. */
newReleases: number
/** Provisional (tail) releases the batch finished. */
finalized: number
candidates: number
/** Unnamed releases written to the pending store instead of the catalog (decision f). */
pending: number
fragments: number
}
/** A chunk's slot in its batch: the key and the upload the edge numbered it with (ProbeCandidate points at it). */
const slot = (key: string, upload: number) => `${key}\n${upload}`
/**
* Stores one batch of a leased range (REBUILD.md, "Voyager 3: window indexing", 2 and 4) in one transaction: each
* release chunk is a whole release the edge's window finished, written once (its poster and group set by id, its
* keys as bytea, its name into release_texts), with one chunk of its files, its probe candidates, and the batch's
* rule counts and samples; then the range's resume point moves. A batch that doesn't start at the resume point was
* Stores one batch of a leased range (REBUILD.md, parts 3–7), in one
* transaction: release aggregates in the catalog, one chunk per release,
* probe candidates, rule counts and samples, edge fragments, and the
* range's resume point. A batch that doesn't start at the resume point was
* sent twice or too late, and is refused (StaleBatch), so the sums stay right.
*
* A tail release comes twice: provisionally (finished: false, settled_at null, its candidates waiting) and then
* finished, under the same release key (the identity, the group and its first article: lib/release/keys.ts), with
* the files not sent before (a chunk's counts are its own; the finish adds them up); the finish is the one update a
* release gets from the header pass. A finished release sent again is left as it is.
*
* Interim, until blocks are leased (the next step): chunks, counts and samples are keyed by the range's block
* (ranges.block_id), and a range without one gets a block of its own here.
*/
class BatchStore {
async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
@@ -61,9 +27,7 @@ class BatchStore {
// One batch of a range at a time, without holding the range's row for the whole transaction: the
// prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
// LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [
rangeId,
])
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [rangeId])
const range = await trx.from('ranges').where('id', rangeId).first()
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
@@ -71,48 +35,35 @@ class BatchStore {
throw new StaleBatch(
`Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
)
const blockId = await this.blockOf(trx, range)
const result: StoredBatch = {
releases: batch.releases.length,
newReleases: 0,
finalized: 0,
candidates: 0,
pending: 0,
fragments: batch.fragments.length,
}
// Nothing enters the catalog without a usable name (decision f): an unnamed release (no readable file name)
// goes to the pending store with its probe targets, the named ones to the catalog.
const named = batch.releases.filter((r) => r.readable)
const unnamed = batch.releases.filter((r) => !r.readable)
const written = await this.writeReleases(
trx,
{ groupId: Number(range.group_id), blockId },
rangeId,
batch.from,
named,
batch.releases,
batch.candidates
)
result.newReleases = written.newReleases
result.finalized = written.finalized
result.candidates = written.candidates
result.pending = await this.writePending(
trx,
Number(range.group_id),
unnamed,
batch.candidates
)
if (batch.counts.length > 0) {
await trx.rawQuery(
`INSERT INTO rule_counts (block_id, rule, key, articles, files)
`INSERT INTO rule_counts (range_id, rule, key, articles, files)
SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
ON CONFLICT (block_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
[blockId, JSON.stringify(batch.counts)]
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
[rangeId, JSON.stringify(batch.counts)]
)
}
for (const s of batch.samples) {
await trx.insertQuery().table('rule_samples').insert({
block_id: blockId,
range_id: rangeId,
rule: s.rule,
rule_key: s.ruleKey,
release_key: s.key,
@@ -123,6 +74,18 @@ class BatchStore {
})
}
for (let i = 0; i < batch.fragments.length; i += 2000) {
const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
range_id: rangeId,
file_key: f.fileKey,
// Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
segments: f.segments.length,
fragment: JSON.stringify(f),
}))
await trx.insertQuery().table('fragments').multiInsert(rows)
}
const moved = await trx
.from('ranges')
.where('id', rangeId)
@@ -132,162 +95,106 @@ class BatchStore {
.update({
next_id: batch.next,
headers: db.raw('headers + ?', [batch.headers]),
// The range's posting time span: the batches' trimmed spans, min and max (least and greatest skip nulls).
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [
batch.postedLow ?? null,
]),
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [
batch.postedHigh ?? null,
]),
// The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
heartbeat_at: new Date(),
})
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1))
throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
return result
})
}
/** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
if (range.block_id) return Number(range.block_id)
const [block] = await trx
.table('blocks')
.insert({
group_id: range.group_id,
start_id: range.start_id,
end_id: range.end_id,
next_id: range.next_id,
status: 'leased',
priority: range.priority ?? 0,
node_id: range.node_id,
leased_at: range.leased_at ?? new Date(),
heartbeat_at: new Date(),
})
.returning('id')
await trx.from('ranges').where('id', range.id).update({ block_id: block.id })
return Number(block.id)
}
/**
* The unnamed releases of a batch into pending_releases (migration 6300, the HDD): keys, counts, the upload
* tool and the probe targets, no chunk. The pending ladder (REBUILD.md decision f) probes them cheapest first,
* and a name moves one into the catalog; a provisional tail release sent again adds its counts up.
* An upload spread over groups (FTD: par2 and part1 to a.b.music.misc,
* part2 to a.b.sounds.music, each article in one group only; LOG.md
* 2026-10-11 (9)): a chunk without a release in its own group joins one of
* its key in another group whose window is within UPLOAD_GAP_MS, when none
* of its file names are in that release yet. A post of the same files to
* another group (other message-ids, the same names) stays a release of its
* own, or the NZB would hold every file twice. Reads the candidate's file
* names from its chunks; candidates are few.
*/
async writePending(
private async joinAcrossGroups(
trx: TransactionClientContract,
part: ReleaseChunk[],
windows: { idx: number; key: string; first: number | null | undefined; last: number | null | undefined }[],
groupId: number,
releases: ReleaseChunk[],
candidates: ProbeCandidate[]
): Promise<number> {
if (releases.length === 0) return 0
const targets = new Map<
string,
{ kind: string; messageId: string; file: string; bytes?: number }[]
>()
for (const c of candidates) {
const key = slot(c.releaseKey, c.upload ?? 0)
const list = targets.get(key) ?? []
list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
targets.set(key, list)
joined: Map<number, number>
) {
const open = windows.filter((w) => !joined.has(w.idx) && w.first)
if (open.length === 0) return
const { rows: candidates } = await trx.rawQuery(
`SELECT v.idx, r.id
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
JOIN releases r ON r.identity_key = v.key AND r.group_id <> ?
WHERE r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0)
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
[JSON.stringify(open), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
)
if (candidates.length === 0) return
const ids: number[] = [...new Set<number>(candidates.map((c: { id: string }) => Number(c.id)))]
const names = new Map<number, Set<string>>()
const { rows: chunks } = await trx.rawQuery(`SELECT release_id, payload FROM release_chunks WHERE release_id = ANY(?)`, [ids])
for (const c of chunks) {
const set = names.get(Number(c.release_id)) ?? new Set<string>()
for (const f of decodeChunk(c.payload)) set.add(f.name)
names.set(Number(c.release_id), set)
}
let written = 0
for (let i = 0; i < releases.length; i += 1000) {
const rows = releases.slice(i, i + 1000).map((r) => {
const probes = targets.get(slot(r.key, r.upload)) ?? []
return {
release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
identity_key: identityKey(r.key).toString('hex'),
name: r.name.slice(0, TEXT_MAX),
poster: r.poster || null,
first: r.postedAt,
last: r.postedLast ?? r.postedAt,
first_article: r.firstArticle,
last_article: r.lastArticle,
files: r.files,
segments: r.segments,
bytes: r.bytes,
partial: r.partial === true,
finished: r.finished !== false,
file_total: r.fileTotal,
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
candidates: probes,
// The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
target_bytes: byEffort(probes)[0]?.bytes ?? null,
}
})
const { rows: done } = await trx.rawQuery(
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes)
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates,
pending_class(v.name, v.bytes, v.partial), v.target_bytes
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb, target_bytes bigint)
ON CONFLICT (release_key) DO UPDATE SET
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END,
klass = pending_class(pending_releases.name, pending_releases.bytes + excluded.bytes, excluded.partial),
target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes)
WHERE NOT pending_releases.finished
RETURNING release_key, tool, klass, (xmax = 0) AS inserted`,
[groupId, JSON.stringify(rows)]
)
written += done.length
// Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
for (const d of done) {
const key = `${d.tool}\t${d.klass}`
const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
if (d.inserted) entry.waiting++
classes.set(key, entry)
}
if (classes.size > 0) {
await trx.rawQuery(
`INSERT INTO pending_classes (tool, group_id, klass, waiting)
SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
[groupId, JSON.stringify([...classes.values()])]
)
for (const c of candidates as { idx: number; id: string }[]) {
if (joined.has(c.idx)) continue
const have = names.get(Number(c.id)) ?? new Set<string>()
const incoming = decodeChunk(part[c.idx].payload).map((f) => f.name)
if (incoming.every((name) => !have.has(name))) {
joined.set(c.idx, Number(c.id))
for (const name of incoming) have.add(name)
names.set(Number(c.id), have)
}
}
return written
}
/**
* The releases of a batch into the catalog, one chunk each, and their probe candidates. Every chunk is one
* release: inserted, or (a provisional tail release sent again) finished.
* Release aggregates in the catalog, one write-once chunk per release, and
* probe candidates (a batch's, or files the fragment merger completed).
* A chunk joins the release of its key whose articles the range's group
* owns and whose posting window is within UPLOAD_GAP_MS of its own, else
* starts one: re-uploads are releases of their own (decision d). The
* group is locked, so concurrent writers don't both start one.
*/
async writeReleases(
trx: TransactionClientContract,
where: { groupId: number; blockId: number },
rangeId: number,
batchFrom: number,
releases: ReleaseChunk[],
candidates: ProbeCandidate[]
) {
const out = { newReleases: 0, finalized: 0, candidates: 0 }
if (releases.length === 0) return out
/** Release ids by slot, for the chunks and candidates; a slot not here was a finished release sent again. */
const out = { newReleases: 0, candidates: 0 }
const ids = new Map<string, number>()
const finalized = new Set<number>()
const posters = await CatalogRefs.posterIds(trx, releases.map((r) => r.poster).filter(Boolean))
const groupSets = await CatalogRefs.groupSetIds(
trx,
releases.map((r) => r.groups)
)
const slot = (key: string, upload: number) => `${key}\n${upload}`
const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
// One writer per group at a time: a chunk only matches releases of its group, and a lock per key
// ran out of shared memory (a picture group's batch has 50k keys; LOG.md 2026-10-09 (12)).
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('releases:' || ?::text, 0))`, [groupId])
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
for (let i = 0; i < sorted.length; i += 1000) {
const part = sorted.slice(i, i + 1000)
const rows = part.map((r, idx) => ({
idx,
release_key: releaseKey(r.key, where.groupId, r.firstArticle).toString('hex'),
identity_key: identityKey(r.key).toString('hex'),
const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
const { rows: matches } = await trx.rawQuery(
`SELECT DISTINCT ON (v.idx) v.idx, r.id
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
WHERE v.first IS NULL OR r.posted_at IS NULL
OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
[JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
)
const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
await this.joinAcrossGroups(trx, part, windows, groupId, joined)
const row = (r: ReleaseChunk) => ({
name: r.name,
source: sourceCode(r.source),
poster_id: r.poster ? (posters.get(posterText(r.poster)) ?? null) : null,
group_set_id: groupSets.get(groupSetKey(r.groups))!,
source: r.source,
poster: r.poster,
groups: r.groups,
first: r.postedAt,
last: r.postedLast ?? r.postedAt,
file_total: r.fileTotal,
@@ -298,114 +205,98 @@ class BatchStore {
last_article: r.lastArticle,
readable: r.readable,
has_par2: r.hasPar2,
finished: r.finished !== false,
edge: r.edge === true,
partial: r.partial === true,
sidecar_only: sidecarOnly(r),
}))
// Inserted, or a provisional release of the key finished (the chunk's files added up); a finished release
// sent again (no row returned) is left as it is.
const { rows: written } = await trx.rawQuery(
`INSERT INTO releases (release_key, identity_key, group_id, group_set_id, poster_id, name, source, posted_at, last_posted_at, file_total, files, segments, bytes, chunks,
first_article, last_article, readable, has_par2, complete, edge, sidecar_only, settled_at)
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.group_set_id, v.poster_id, v.name, v.source,
to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
v.first_article, v.last_article, v.readable, v.has_par2,
CASE WHEN v.partial THEN false WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END,
v.edge, v.sidecar_only, CASE WHEN v.finished THEN now() END
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, release_key text, identity_key text, name text, source int, poster_id int, group_set_id int, first bigint, last bigint,
file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean,
finished boolean, edge boolean, partial boolean, sidecar_only boolean)
ON CONFLICT (release_key) DO UPDATE SET
files = releases.files + excluded.files, segments = releases.segments + excluded.segments, bytes = releases.bytes + excluded.bytes,
chunks = releases.chunks + CASE WHEN excluded.files > 0 THEN 1 ELSE 0 END,
posted_at = least(releases.posted_at, excluded.posted_at), last_posted_at = greatest(releases.last_posted_at, excluded.last_posted_at),
file_total = coalesce(excluded.file_total, releases.file_total),
first_article = least(releases.first_article, excluded.first_article), last_article = greatest(releases.last_article, excluded.last_article),
readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2,
complete = CASE WHEN releases.complete IS FALSE AND releases.file_total IS NULL THEN false
WHEN coalesce(excluded.file_total, releases.file_total) IS NULL THEN releases.complete
ELSE releases.files + excluded.files >= coalesce(excluded.file_total, releases.file_total) END,
edge = excluded.edge, sidecar_only = coalesce(releases.sidecar_only, excluded.sidecar_only), settled_at = excluded.settled_at
WHERE releases.settled_at IS NULL
RETURNING id, encode(release_key, 'hex') AS release_key, (xmax = 0) AS inserted, settled_at`,
[where.groupId, JSON.stringify(rows)]
)
const byKey = new Map<string, { id: number; inserted: boolean; settled: boolean }>(
written.map(
(w: { id: string; release_key: string; inserted: boolean; settled_at: Date | null }) => [
w.release_key,
{ id: Number(w.id), inserted: w.inserted, settled: w.settled_at !== null },
]
)
)
const texts: { release_id: number; kind: number; text: string }[] = []
for (const row of rows) {
const w = byKey.get(row.release_key)
if (!w) continue
const r = part[row.idx]
ids.set(slot(r.key, r.upload), w.id)
if (w.inserted) out.newReleases++
else if (w.settled) {
out.finalized++
finalized.add(w.id)
}
texts.push({ release_id: w.id, kind: TEXT_KINDS.name, text: row.name.slice(0, TEXT_MAX) })
// An album's members stay findable by their own names (the first TEXTS_PER_KIND of them).
if (r.source === 'album' && r.files > 0) {
for (const text of textsToStore(decodeChunk(r.payload).map((f) => f.name))) {
texts.push({ release_id: w.id, kind: TEXT_KINDS.member, text })
}
}
}
// The name (and an album's members), searchable (release_texts; a text sent again is a no-op).
if (texts.length > 0) {
})
const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
if (updates.length > 0) {
await trx.rawQuery(
`INSERT INTO release_texts (release_id, kind, text)
SELECT v.release_id, v.kind, v.text FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind int, text text) WHERE v.text <> ''
ON CONFLICT DO NOTHING`,
[JSON.stringify(texts)]
`WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
FROM v GROUP BY id)
UPDATE releases r SET
files = r.files + a.files,
segments = r.segments + a.segments,
bytes = r.bytes + a.bytes,
chunks = r.chunks + a.chunks,
posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
file_total = coalesce(r.file_total, a.file_total),
first_article = least(r.first_article, a.first_article),
last_article = greatest(r.last_article, a.last_article),
readable = r.readable OR a.readable,
has_par2 = r.has_par2 OR a.has_par2,
-- As it stands (the counter's files all found); settling makes it final.
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
last_seen_at = now()
FROM a WHERE r.id = a.id`,
[JSON.stringify(updates)]
)
part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
}
const fresh = part.filter((_, idx) => !joined.has(idx))
if (fresh.length > 0) {
// The first upload of a key keeps the key; later ones are key:2, key:3, …
const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
const numbered = (r: ReleaseChunk) => {
const n = (taken.get(r.key) ?? 0) + 1
taken.set(r.key, n)
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
}
// Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
let inserts = fresh.map(numbered)
// Writers of other groups number the same key at the same time (a spam post crossposted to 67
// groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
for (let attempt = 0; inserts.length > 0; attempt++) {
if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
const { rows } = await trx.rawQuery(
`INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
ON CONFLICT (release_key) DO NOTHING
RETURNING id, release_key`,
[groupId, JSON.stringify(inserts)]
)
const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
const missed: typeof inserts = []
for (const r of inserts) {
const id = byReleaseKey.get(r.release_key)
if (id === undefined) missed.push(r)
else ids.set(slot(r.key, r.upload), id)
}
out.newReleases += rows.length
const chunks = new Map(fresh.map((r) => [slot(r.key, r.upload), r]))
inserts = missed.map((r) => numbered(chunks.get(slot(r.key, r.upload))!))
}
}
}
// One write-once chunk per release (a finish adds the files not sent before).
// One write-once chunk per release.
for (let i = 0; i < sorted.length; i += 500) {
const rows = sorted
.slice(i, i + 500)
.filter((r) => ids.has(slot(r.key, r.upload)) && r.files > 0)
.map((r) => ({
release_id: ids.get(slot(r.key, r.upload))!,
block_id: where.blockId,
batch_from: batchFrom,
files: r.files,
segments: r.segments,
payload: r.payload,
}))
const rows = sorted.slice(i, i + 500).map((r) => ({
release_id: ids.get(slot(r.key, r.upload))!,
range_id: rangeId,
batch_from: batchFrom,
files: r.files,
segments: r.segments,
payload: r.payload,
}))
if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
}
// A finished release's candidates are claimable at once; a provisional one's wait for its finish.
const settled = new Set<number>()
for (const r of sorted) {
const id = ids.get(slot(r.key, r.upload))
if (id !== undefined && r.finished !== false) settled.add(id)
}
const chunks = new Map(sorted.map((r) => [slot(r.key, r.upload), r]))
const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
for (let i = 0; i < kept.length; i += 2000) {
const rows = kept.slice(i, i + 2000).map((c) => {
const releaseId = ids.get(slot(c.releaseKey, c.upload ?? 0))!
const chunk = chunks.get(slot(c.releaseKey, c.upload ?? 0))!
return {
release_id: releaseId,
const rows = kept
.slice(i, i + 2000)
.map((c) => ({
release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
kind: c.kind,
message_id: c.messageId,
file: c.file,
ready: settled.has(releaseId),
unreadable: !chunk.readable,
}
})
}))
const added = await trx
.knexQuery()
.table('probe_candidates')
@@ -414,17 +305,6 @@ class BatchStore {
.ignore()
.returning('id')
out.candidates += added.length
// A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
}
// The finish of a provisional release: its waiting candidates become claimable.
if (finalized.size > 0) {
await trx.rawQuery(
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
WHERE pc.release_id = ANY(?) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
[[...finalized]]
)
}
return out
app/services/catalog_refs.ts
-93
View File
@@ -1,93 +0,0 @@
import type db from '@adonisjs/lucid/services/db'
type Client = Pick<typeof db, 'rawQuery'>
/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
export const POSTER_MAX = 200
/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
const CACHE_MAX = 200_000
const posterCache = new Map<string, number>()
const groupSetCache = new Map<string, number>()
/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
export function groupSetKey(groups: string[]): string {
return JSON.stringify([...new Set(groups)].sort())
}
/** A poster as posters stores it. */
export function posterText(poster: string): string {
return poster.slice(0, POSTER_MAX)
}
async function ids(
client: Client,
cache: Map<string, number>,
table: 'posters' | 'group_sets',
column: 'poster' | 'groups',
cast: 'text' | 'jsonb',
values: string[]
): Promise<Map<string, number>> {
const out = new Map<string, number>()
const missing: string[] = []
for (const v of values) {
const id = cache.get(v)
if (id === undefined) missing.push(v)
else out.set(v, id)
}
if (missing.length === 0) return out
// Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
await client.rawQuery(
`INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
[missing]
)
const { rows } = await client.rawQuery(
`SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
[missing]
)
if (cache.size + rows.length > CACHE_MAX) cache.clear()
for (const r of rows) {
// jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
cache.set(key, Number(r.id))
out.set(key, Number(r.id))
}
return out
}
/**
* Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
* values are inserted; known ones come from a cache per process (the batch store sees the same posters and
* group sets batch after batch).
*/
export default {
/** Ids by poster (posterText applied), for a batch of posters. */
async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
return ids(client, posterCache, 'posters', 'poster', 'text', [
...new Set(posters.map(posterText)),
])
},
async posterId(client: Client, poster: string): Promise<number> {
const found = await this.posterIds(client, [poster])
return found.get(posterText(poster))!
},
/** Ids by groupSetKey, for a batch of group lists. */
async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
...new Set(sets.map(groupSetKey)),
])
},
async groupSetId(client: Client, groups: string[]): Promise<number> {
const found = await this.groupSetIds(client, [groups])
return found.get(groupSetKey(groups))!
},
/** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
forget() {
posterCache.clear()
groupSetCache.clear()
},
}
app/services/corpus.ts
+15 -45
View File
@@ -1,24 +1,11 @@
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
import { unzip } from 'node:zlib'
import { promisify } from 'node:util'
import coordinatorConfig from '#config/coordinator'
import { filterOverviews } from '#lib/edge/index'
import type { ChunkSpan } from '#services/spool'
const inflate = promisify(unzip)
const deflate = promisify(deflateCb)
const gzip = promisify(gzipCb)
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
const MAX_FILTER_BYTES = 256 * 1024 * 1024
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
export interface CorpusFilter {
ruleVersion: number
priorities: Record<string, number>
}
/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
export const FILTERED = ['crosspost', 'obfuscated']
export interface CorpusRange {
group: string
@@ -32,23 +19,20 @@ export interface CorpusRange {
* every chunk is there, the layout edge-bench reads (edge/fixtures). The
* grouping sweeps replay them with each build's rules instead of waiting for
* a re-index; the prefetcher reads a stored chunk instead of fetching it
* again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
* (101)): the lines the edge worker only counts are left out, ~half of them,
* and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
* article numbers by rule; its range.json says `filtered`.
* again. Off without CORPUS_DIR.
*/
class Corpus {
get dir(): string | undefined {
return coordinatorConfig.corpus.dir
}
private rangeDir(range: CorpusRange, dir = this.dir!) {
return join(dir, range.group, `${range.start}-${range.end}`)
private rangeDir(range: CorpusRange) {
return join(this.dir!, range.group, `${range.start}-${range.end}`)
}
private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
private path(range: CorpusRange, span: ChunkSpan) {
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
}
/** Whether every chunk of the range is stored (its range.json). */
@@ -69,50 +53,36 @@ class Corpus {
)
}
/** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
if (!dir) return null
return readFile(this.path(range, span, dir)).catch(() => null)
/** A stored chunk, or null. */
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
if (!this.dir) return null
return readFile(this.path(range, span)).catch(() => null)
}
/**
* Stores a chunk that inflates (a reply cut short isn't kept), and the
* range.json once the range is complete (`total` chunks). With `filter`,
* without the lines the edge worker drops on their own, compressed again
* as the news server's (zlib), with its `.drops` written first.
* range.json once the range is complete (`total` chunks).
*/
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
if (!this.dir) return
let raw: Buffer
try {
raw = await inflate(data)
await inflate(data)
} catch {
return
}
const dir = this.rangeDir(range)
const file = this.path(range, span)
await mkdir(dir, { recursive: true })
let stored = data
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
if (filter && raw.length < MAX_FILTER_BYTES) {
// Latin-1 both ways: every byte back as it was.
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
await rename(`${file}.drops.part`, `${file}.drops`)
}
await writeFile(`${file}.part`, stored)
await writeFile(`${file}.part`, data)
await rename(`${file}.part`, file)
const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
}
/** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
async removeChunk(range: CorpusRange, span: ChunkSpan) {
if (!this.dir) return
await rm(this.path(range, span), { force: true })
await rm(`${this.path(range, span)}.drops`, { force: true })
await rm(join(this.rangeDir(range), 'range.json'), { force: true })
}
}
app/services/disk_space.ts
-40
View File
@@ -1,40 +0,0 @@
import { statfs } from 'node:fs/promises'
import logger from '@adonisjs/core/services/logger'
/** How often free space is measured. */
const CHECK_MS = 60_000
/** Resumed only this much above the floor: not paused and resumed at every flush. */
const MARGIN_GB = 1
/**
* Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a
* container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on.
* The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md
* 2026-10-18 (77)). Off when the floor is 0.
*/
export class DiskSpace {
private checkedAt = 0
private low = false
constructor(
private what: string,
private path: string,
private floorGb: number,
private free: (path: string) => Promise<number> = async (p) => {
const s = await statfs(p)
return s.bavail * s.bsize
},
private checkMs = CHECK_MS
) {}
async short(): Promise<boolean> {
if (!this.floorGb) return false
if (Date.now() - this.checkedAt < this.checkMs) return this.low
this.checkedAt = Date.now()
const gb = (await this.free(this.path)) / 1e9
const was = this.low
this.low = gb < this.floorGb + (was ? MARGIN_GB : 0)
if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`)
return this.low
}
}
app/services/fragment_merger.ts
+153
View File
@@ -0,0 +1,153 @@
import db from '@adonisjs/lucid/services/db'
import BatchStore from '#services/batch_store'
import ChunkPartitions from '#services/chunk_partitions'
import RuleSets from '#services/rule_sets'
import { RuleSet } from '#lib/rules/index'
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import type { AggregatedFile } from '#lib/files/index'
import type { EdgeSegment } from '#lib/edge/index'
export interface MergeResult {
files: number
completed: number
incomplete: number
waiting: number
/** This round reached the end of the file keys (the next one starts over). */
wrapped: boolean
}
/** File keys per round. */
const BATCH = 20_000
/**
* Files cut by a range's edge (REBUILD.md, fragments), put together once
* the ranges around them are done: a complete file is judged by the rules
* like any other and written as a release chunk (with probe candidates); a
* file still missing parts while every neighbouring range is done is
* incomplete, and dropped (counted on its ranges). Others wait.
*/
class FragmentMerger {
/** Where the next round goes on: waiting files must not take every round's places. */
private cursor = ''
async run(): Promise<MergeResult> {
const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
const { rows: keys } = await db.rawQuery(
`SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
[this.cursor, BATCH]
)
// At the end of the keys: the next round starts over.
result.wrapped = keys.length < BATCH
this.cursor = result.wrapped ? '' : keys.at(-1).file_key
if (keys.length === 0) return result
const rules = new RuleSet(await RuleSets.current())
const rows = await db
.from('fragments as f')
.join('ranges as r', 'r.id', 'f.range_id')
.whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
.select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
const byKey = new Map<string, typeof rows>()
for (const row of rows) {
const list = byKey.get(row.file_key)
if (list) list.push(row)
else byKey.set(row.file_key, [row])
}
const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
const neighbours = await this.neighbourStatus([...byKey.values()])
for (const list of byKey.values()) {
result.files++
const file = mergeFragments(list.map((r) => r.fragment as Fragment))
const ids = list.map((r) => Number(r.id))
const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
if (file) {
completed.push({ file, rangeId, ids })
continue
}
if (list.every((r) => r.status === 'done') && neighbours(list)) {
incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
} else {
result.waiting++
}
}
// Completed files, grouped into their releases, judged and written like a batch's.
const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
for (const { file, rangeId } of completed) {
const key = releaseKeyOf(file.segments[0])
const entry = byRelease.get(key)
if (entry) entry.files.push(file)
else byRelease.set(key, { files: [file], rangeId })
}
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
for (const [key, { files, rangeId }] of byRelease) {
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
if (verdict.rule !== null) continue
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
}
await ChunkPartitions.ensure()
await db.transaction(async (trx) => {
// In key order (batches lock their group; the merger only ever waits on one batch at a time).
releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
for (const { rangeId, segments } of incomplete) {
await trx.rawQuery(
`INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
[rangeId, segments]
)
}
})
result.completed = completed.length
result.incomplete = incomplete.length
return result
}
/**
* For each file's fragments: whether the ranges right before and after
* their ranges (where a missing part could be) are done, read in one
* query for the whole round (per file it was three, ~2,000 files a
* minute). A neighbour not planned yet counts as not done (newer articles
* come later), except before the group's oldest article.
*/
private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
const span = (list: (typeof lists)[number]) => ({
group: Number(list[0].group_id),
first: Math.min(...list.map((r) => Number(r.start_id))),
last: Math.max(...list.map((r) => Number(r.end_id))),
})
const spans = lists.map(span)
const groups = [...new Set(spans.map((s) => s.group))]
// The range holding the article before and the one after, each by the (group_id, start_id) index:
// "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
const { rows } = await db.rawQuery(
`SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
ORDER BY r.start_id DESC LIMIT 1) b ON true
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
ORDER BY r.start_id DESC LIMIT 1) a ON true`,
[spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
)
const byEnd = new Map<string, string>()
const byStart = new Map<string, string>()
for (const r of rows) {
if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
}
const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
return (list: (typeof lists)[number]) => {
const { group, first, last } = span(list)
const before = byEnd.get(`${group}:${first - 1}`)
const low = lows.get(group)
const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
}
}
}
export default new FragmentMerger()
app/services/group_dates.ts
-138
View File
@@ -1,138 +0,0 @@
import type { Overview, XoverClient } from '#lib/nntp/index'
/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
export interface DateSample {
article: number
postedAt: Date
}
/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
const OLDEST = Date.UTC(1995, 0, 1)
const DAY = 86_400_000
/**
* The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
* `min` of them; null otherwise (an expired or empty slice).
*/
export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
const latest = now.getTime() + DAY
const times = overviews
.map(([, fields]) => Date.parse(fields.date ?? ''))
.filter((t) => t >= OLDEST && t <= latest)
.sort((a, b) => a - b)
if (times.length < min) return null
return new Date(times[Math.floor(times.length / 2)])
}
/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
export function samplePoints(low: number, high: number, count: number, size: number): number[] {
const last = Math.max(low, high - size + 1)
if (count <= 1 || last <= low) return [low]
const points = new Set<number>()
for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
return [...points].sort((a, b) => a - b)
}
/**
* The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
* (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
* in number order, bar stragglers and old crossposts).
*/
export function monotone(samples: DateSample[]): DateSample[] {
const sorted = [...samples].sort((a, b) => a.article - b.article)
const kept = sorted.filter((s, i) => {
const next = sorted.slice(i + 1, i + 3)
return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
})
let floor = Number.NEGATIVE_INFINITY
return kept.map((s) => {
floor = Math.max(floor, s.postedAt.getTime())
return { article: s.article, postedAt: new Date(floor) }
})
}
/**
* When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
* it, the first or last sample's date beyond them; null without samples.
*/
export function estimateDate(samples: DateSample[], article: number): Date | null {
if (samples.length === 0) return null
if (article <= samples[0].article) return samples[0].postedAt
const last = samples[samples.length - 1]
if (article >= last.article) return last.postedAt
let lo = 0
let hi = samples.length - 1
while (hi - lo > 1) {
const mid = (lo + hi) >> 1
if (samples[mid].article <= article) lo = mid
else hi = mid
}
const a = samples[lo]
const b = samples[hi]
const f = (article - a.article) / (b.article - a.article)
return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
}
/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
export function sampleCount(
low: number,
high: number,
every: number,
min = 16,
max = 1000
): number {
return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
}
/**
* Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
* with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
* server's span.
*/
export async function sampleGroup(
use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
name: string,
options: { every: number; size: number; maxFailures?: number },
now = new Date()
): Promise<{
samples: DateSample[]
low: number
high: number
commands: number
failed: string[]
}> {
const info = await use((client) => client.group(name))
const samples: DateSample[] = []
const failed: string[] = []
let commands = 1
// Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
// minutes; 16 groups never got through on one connection) costs that point, not the group.
for (const point of samplePoints(
info.low,
info.high,
sampleCount(info.low, info.high, options.every),
options.size
)) {
if (failed.length >= (options.maxFailures ?? 3)) break
try {
const date = await use(async (client) => {
await client.group(name)
commands += 2
const slice = medianDate(
await client.xover(point, Math.min(point + options.size - 1, info.high)),
now
)
if (slice) return slice
commands++
return medianDate(
await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
now
)
})
if (date) samples.push({ article: point, postedAt: date })
} catch (error: any) {
failed.push(`${point}: ${error.message}`)
}
}
return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
}
app/services/group_leases.ts
-280
View File
@@ -1,280 +0,0 @@
import db from '@adonisjs/lucid/services/db'
import { createReadStream, createWriteStream } from 'node:fs'
import { mkdir, rename, stat } from 'node:fs/promises'
import { join } from 'node:path'
import type { Readable } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import coordinatorConfig from '#config/coordinator'
import RuleSets from '#services/rule_sets'
import CatalogSpace from '#services/catalog_space'
import { LeaseLost } from '#services/leases'
import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
export const LANES: Lane[] = ['history', 'tail']
/**
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
* next range each time it finished one, so a group's ranges are indexed in article order through one carried
* window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
* works and the snapshot its window stands at.
*
* - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
* LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
* the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
* group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
* the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
* reaches the tail lane's first range (its ranges are done: the history lane is caught up).
* - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
* mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
* big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
* is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
* lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
* (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
* idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
* - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
* when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
* planned more.
* - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
* `<group>-tail.gz`), given to the next holder of the lane.
*/
class GroupLeases {
get dir(): string {
return coordinatorConfig.snapshotDir
}
/**
* The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
* above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
*/
private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
const tail = coordinatorConfig.tail.priority
if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
return {
sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
bindings: [tail, min, max],
}
}
async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
// The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
if (lane === 'history' && (await CatalogSpace.full())) return null
const rules = await RuleSets.current()
const ttl = coordinatorConfig.leaseTtlSeconds
const todo = this.laneFilter(lane, true)
const all = this.laneFilter(lane, false)
return db.transaction(async (trx) => {
// One lease at a time: the choice reads several tables.
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
// The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
// last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
// most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
// the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
// win).
const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
const { rows } = await trx.rawQuery(
`WITH todo AS (
SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
FROM ranges r
WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
AND r.next_id <= r.end_id AND ${todo.sql}
GROUP BY r.group_id
), free AS (
SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
todo.priority, todo.articles
FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
), turns AS (
SELECT free.*,
least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
FROM free
)
SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
[ttl, ...todo.bindings, lane, ttl, first, second]
)
const row = rows[0]
if (!row) return null
const groupId = Number(row.id)
// What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
// after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
if (row.snapshot_next !== null) {
await trx.rawQuery(
`UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
[
row.snapshot_next,
groupId,
row.snapshot_next,
...all.bindings,
row.snapshot_at ?? new Date(0),
row.snapshot_next,
]
)
} else {
await trx.rawQuery(
`UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
[groupId, ...all.bindings]
)
}
await trx.rawQuery(
`INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
VALUES (?, ?, ?, now(), now(), NULL)
ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
[groupId, lane, nodeId]
)
await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
return {
group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
lane,
rules,
snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
}
})
}
/** The group's lease row of the lane, when this node holds it. */
async held(groupId: number, nodeId: number, lane: Lane = 'history') {
const lease = await db
.from('group_leases')
.where('group_id', groupId)
.where('lane', lane)
.first()
if (!lease || Number(lease.node_id) !== nodeId)
throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
return lease
}
/**
* The group's next range of the lane in article order (or the one this node has), leased to the node; null when
* the lane is caught up.
*/
async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
await this.held(groupId, nodeId, lane)
const rules = await RuleSets.current()
const todo = this.laneFilter(lane, true)
const all = this.laneFilter(lane, false)
const mine = (
await db.rawQuery(
`SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
[groupId, nodeId, ...all.bindings]
)
).rows[0]
const row =
mine ??
(
await db.rawQuery(
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
WHERE r.id = (
SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
RETURNING r.id, r.start_id, r.end_id, r.next_id`,
[nodeId, rules.version, groupId, ...todo.bindings]
)
).rows[0]
await db
.from('group_leases')
.where('group_id', groupId)
.where('lane', lane)
.update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
if (!row) return null
return {
range: {
id: Number(row.id),
start: Number(row.start_id),
end: Number(row.end_id),
next: Number(row.next_id),
},
}
}
async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
await this.held(groupId, nodeId, lane)
await db
.from('group_leases')
.where('group_id', groupId)
.where('lane', lane)
.update({ heartbeat_at: new Date() })
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
}
/** The lane a range belongs to, by its priority. */
laneOf(priority: number): Lane {
return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
}
/** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
await db.rawQuery(
`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
[headers, groupId, lane]
)
}
/** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
async release(groupId: number, nodeId: number, lane: Lane = 'history') {
await this.held(groupId, nodeId, lane)
const all = this.laneFilter(lane, false)
await db.rawQuery(
`UPDATE ranges r SET status = 'planned', node_id = NULL
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
[groupId, nodeId, ...all.bindings]
)
await db
.from('group_leases')
.where('group_id', groupId)
.where('lane', lane)
.update({ node_id: null, range_id: null })
}
snapshotPath(groupId: number, lane: Lane = 'history') {
return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
}
/**
* The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
* gigabytes) and written whole before it replaces the old one.
*/
async putSnapshot(
groupId: number,
nodeId: number,
next: number,
body: Readable,
lane: Lane = 'history'
) {
await this.held(groupId, nodeId, lane)
await mkdir(this.dir, { recursive: true })
const path = this.snapshotPath(groupId, lane)
await pipeline(body, createWriteStream(`${path}.part`))
await rename(`${path}.part`, path)
await db
.from('group_leases')
.where('group_id', groupId)
.where('lane', lane)
.update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
}
/** The snapshot's stream and size, or null when there is none. */
async getSnapshot(
groupId: number,
nodeId: number,
lane: Lane = 'history'
): Promise<{ stream: Readable; size: number; next: number } | null> {
const lease = await this.held(groupId, nodeId, lane)
if (lease.snapshot_next === null) return null
const path = this.snapshotPath(groupId, lane)
try {
const { size } = await stat(path)
return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
} catch {
return null
}
}
}
export default new GroupLeases()
app/services/hash_twins.ts
-27
View File
@@ -1,27 +0,0 @@
import db from '@adonisjs/lucid/services/db'
/**
* Twins by content (LOG.md (130)): a probed file's first 16 KiB and size (file_hashes, what the par2 merge and the
* repeated verdict match) in a catalog release that has a name, readable or found inside. The same film posted
* under scrambles by other tools gets that name without another fetch. Releases with a junk verdict don't count.
*/
export async function twinNames(
pairs: { hash16k: string; size: number }[],
trx: { rawQuery: typeof db.rawQuery } = db
): Promise<Map<string, string>> {
const out = new Map<string, string>()
if (pairs.length === 0) return out
const { rows } = await trx.rawQuery(
`SELECT DISTINCT ON (h.hash16k, h.size) h.hash16k, h.size, coalesce(r.inner_name, r.name) AS name
FROM jsonb_to_recordset(?::jsonb) AS v(hash16k text, size bigint)
JOIN file_hashes h ON h.hash16k = v.hash16k AND h.size = v.size
JOIN releases r ON r.id = h.release_id AND (r.readable OR r.inner_name IS NOT NULL)
WHERE NOT EXISTS (SELECT 1 FROM release_verdicts x WHERE x.release_id = r.id)
ORDER BY h.hash16k, h.size, r.readable DESC, r.id`,
[JSON.stringify(pairs)]
)
for (const r of rows) out.set(`${r.hash16k}\t${r.size}`, r.name)
return out
}
export const twinKey = (hash16k: string, size: number) => `${hash16k}\t${size}`
app/services/leases.ts
+9 -63
View File
@@ -41,12 +41,7 @@ class Leases {
AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
[
nodeId,
rules.version,
coordinatorConfig.leaseTtlSeconds,
coordinatorConfig.leaseTtlSeconds,
]
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
)
let { rows } = await take(true)
if (rows.length === 0) ({ rows } = await take(false))
@@ -54,38 +49,12 @@ class Leases {
if (!row) return null
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
return {
range: {
id: Number(row.id),
group: group.name,
start: Number(row.start_id),
end: Number(row.end_id),
next: Number(row.next_id),
},
rules,
}
}
/**
* Ranges read to their end whose worker went quiet before finishing them: done. The lease query skips a range past
* its end, so one whose finish never came stayed leased for good (a gap range since 2026-10-16 14:03, during the lock
* pile-up; LOG.md 2026-10-17 (72)). Its batches are all stored; its spooled chunks go.
*/
async finishAbandoned(): Promise<number> {
const { rows } = await db.rawQuery(
`UPDATE ranges SET status = 'done', done_at = now()
WHERE status = 'leased' AND next_id > end_id AND heartbeat_at < now() - make_interval(secs => ?)
RETURNING id`,
[coordinatorConfig.leaseTtlSeconds]
)
for (const { id } of rows) await Spool.remove(Number(id))
return rows.length
return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
}
async leased(rangeId: number, nodeId: number) {
const range = await db.from('ranges').where('id', rangeId).first()
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
return range
}
@@ -98,14 +67,8 @@ class Leases {
/** The worker reached the range's end: done, and its spooled chunks go. */
async finish(rangeId: number, nodeId: number) {
const range = await this.leased(rangeId, nodeId)
if (Number(range.next_id) <= Number(range.end_id))
throw new LeaseLost(
`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`
)
await db
.from('ranges')
.where('id', rangeId)
.update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
await Spool.remove(rangeId)
}
@@ -116,19 +79,12 @@ class Leases {
*/
async corrupt(rangeId: number, nodeId: number, from: number) {
const range = await this.leased(rangeId, nodeId)
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find(
(s) => s.from === from
)
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
if (span) {
await Spool.removeChunk(rangeId, span)
if (range.capture) {
const group = await db.from('groups').where('id', range.group_id).select('name').first()
await Corpus.removeChunk(
{ group: group.name, start: Number(range.start_id), end: Number(range.end_id) },
span
)
// Not complete in the corpus any more: the corpus fill fetches the chunk again.
await db.from('ranges').where('id', rangeId).update({ stored_at: null })
await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
}
}
const plain: number[] = (range.plain_chunks ?? []).map(Number)
@@ -136,21 +92,11 @@ class Leases {
// refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
if (plain.filter((p) => p === from).length >= 2) {
// Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
await db
.from('ranges')
.where('id', rangeId)
.update({
status: 'failed',
node_id: null,
error: `chunk ${from} doesn't inflate, also fetched plain`,
})
await db.from('ranges').where('id', rangeId).update({ status: 'failed', node_id: null, error: `chunk ${from} doesn't inflate, also fetched plain` })
return
}
// The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
await db.rawQuery(
`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`,
[from, rangeId]
)
await db.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`, [from, rangeId])
}
/** A stopping worker gives its range back; the next one resumes it. */
app/services/nntp_pool.ts
+13 -93
View File
@@ -2,18 +2,12 @@ import env from '#start/env'
import logger from '@adonisjs/core/services/logger'
import coordinatorConfig from '#config/coordinator'
import { XoverClient } from '#lib/nntp/index'
import type { NntpShare } from '#services/nntp_shares'
/** Connections wanted beyond the most in use at once since the last sync (the planner's, a hedge's). */
const SHARE_MARGIN = 2
/**
* The coordinator's header connections (REBUILD.md, decision e: workers
* have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
* that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
* replaced on the next use. Sharing the account (share()), at most its
* share: over it, connections close as they come free. One left free for
* NNTP_IDLE_CLOSE_SECONDS is closed on the next sync.
* replaced on the next use.
*/
class NntpPool {
private idle: XoverClient[] = []
@@ -23,43 +17,9 @@ class NntpPool {
private openedAt = new Map<XoverClient, number>()
/** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
private retireAt = new Map<XoverClient, number>()
/** When an idle connection came free. */
private idleSince = new Map<XoverClient, number>()
private limit = coordinatorConfig.nntpConnections
/** Since the last sync: the most connections in use at once, and whether work waited for one (starved()). */
private peak = 0
private short = false
get size() {
return this.limit
}
/** Work that could use another connection didn't get one (the prefetcher with all its loops running). */
starved() {
this.short = true
}
/**
* Tells the share what the pool could use and has open, and takes the share's grant as its size. After work waited:
* twice its size (history going on gets its maximum in a few syncs; a tail round, done in seconds, doesn't take
* the prober's connections for one); else the most in use at once since the last sync and a margin.
*/
async share(share: NntpShare) {
const want = this.short || this.waiting.length > 0 ? Math.max(this.limit * 2, this.limit + SHARE_MARGIN) : this.peak + SHARE_MARGIN
this.short = false
this.peak = this.busy.size
const granted = await share.sync(want, this.open)
const was = this.limit
this.limit = granted
// Over the share: the idle ones close now, the busy ones once free (release()).
while (this.open > this.limit && this.idle.length > 0) this.retire(this.idle.pop()!)
this.closeIdle()
// Under it with work waiting: connections for it.
while (this.waiting.length > 0 && this.open < this.limit) {
const next = this.waiting.shift()!
this.acquire().then(next, () => this.waiting.unshift(next))
}
return { was, granted }
return coordinatorConfig.nntpConnections
}
async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
@@ -84,7 +44,6 @@ class NntpPool {
private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
this.busy.add(client)
this.peak = Math.max(this.peak, this.busy.size)
let broken = false
try {
return await work(client)
@@ -97,28 +56,9 @@ class NntpPool {
}
}
/** Closes the connections free for nntpIdleCloseMs, the plain one too. */
private closeIdle() {
const idleMs = coordinatorConfig.nntpIdleCloseMs
if (idleMs <= 0) return
const before = Date.now() - idleMs
for (const client of this.idle.filter((c) => (this.idleSince.get(c) ?? Infinity) <= before)) {
this.idle.splice(this.idle.indexOf(client), 1)
this.retire(client)
}
if (this.plain && this.plainPending === 0 && this.plainUsedAt <= before) {
void this.plain.then((c) => c.close()).catch(() => {})
this.plain = null
}
}
private async acquire(): Promise<XoverClient> {
// The one used last: the others stay free long enough to close.
const ready = this.idle.pop()
if (ready) {
this.idleSince.delete(ready)
return ready
}
if (ready) return ready
if (this.open < this.size) {
this.open++
try {
@@ -144,12 +84,15 @@ class NntpPool {
}
private release(client: XoverClient, broken: boolean) {
if (this.open > this.limit) {
this.retire(client)
return
}
if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
this.retire(client)
this.open--
this.openedAt.delete(client)
this.retireAt.delete(client)
try {
client.close()
} catch (error: any) {
logger.debug(`Closing a failed NNTP connection: ${error.message}`)
}
// Someone waiting gets a fresh connection.
const next = this.waiting.shift()
if (next) this.acquire().then(next, () => this.waiting.unshift(next))
@@ -157,30 +100,12 @@ class NntpPool {
}
const next = this.waiting.shift()
if (next) next(client)
else {
this.idle.push(client)
this.idleSince.set(client, Date.now())
}
}
private retire(client: XoverClient) {
this.open--
this.openedAt.delete(client)
this.retireAt.delete(client)
this.idleSince.delete(client)
try {
client.close()
} catch (error: any) {
logger.debug(`Closing an NNTP connection: ${error.message}`)
}
else this.idle.push(client)
}
/** Closes every connection, also those in use: their commands fail. */
private plain: Promise<XoverClient> | null = null
private plainQueue: Promise<unknown> = Promise.resolve()
/** Commands queued or running on the plain connection, and when the last one ended. */
private plainPending = 0
private plainUsedAt = 0
/**
* One connection without XFEATURE COMPRESS GZIP, opened when first needed,
@@ -188,7 +113,6 @@ class NntpPool {
* cuts short (Leases.corrupt). Its replies are deflated here.
*/
async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
this.plainPending++
const run = this.plainQueue.then(async () => {
this.plain ??= XoverClient.connect({
host: env.get('NNTP_HOST')!,
@@ -208,10 +132,7 @@ class NntpPool {
throw error
}
})
this.plainQueue = run.catch(() => {}).finally(() => {
this.plainPending--
this.plainUsedAt = Date.now()
})
this.plainQueue = run.catch(() => {})
return run
}
@@ -222,7 +143,6 @@ class NntpPool {
this.busy.clear()
this.openedAt.clear()
this.retireAt.clear()
this.idleSince.clear()
this.open = 0
}
}
app/services/nntp_shares.ts
-90
View File
@@ -1,90 +0,0 @@
import db from '@adonisjs/lucid/services/db'
import coordinatorConfig from '#config/coordinator'
/** A share whose process hasn't synced for this long is gone: its connections closed with it. */
const STALE_SECONDS = 60
/** Syncs one at a time (pg_advisory_xact_lock). */
const LOCK = 2_026_101_875
export interface Share {
name: string
priority: number
/** Kept whatever the others want (as far as the total goes, by priority). */
min: number
/** What it could use now. */
want: number
/** Connections it has open (or may have: a lane without one yet counts). */
open: number
}
/** What each share should have: its minimum first, then by priority what it wants, up to the total. */
export function targets(total: number, shares: Share[]): Map<string, number> {
const ordered = [...shares].sort((a, b) => b.priority - a.priority || a.name.localeCompare(b.name))
const target = new Map<string, number>()
let left = total
for (const s of ordered) {
const t = Math.min(s.min, left)
target.set(s.name, t)
left -= t
}
for (const s of ordered) {
const more = Math.min(Math.max(s.want - target.get(s.name)!, 0), left)
target.set(s.name, target.get(s.name)! + more)
left -= more
}
return target
}
/**
* The connections `self` may have: its target, but no more than the others leave over. Another share holds what it
* has open or was granted, whichever is more (it may open up to its grant until its next sync tells it otherwise),
* so the account never goes over the total: a share grows only into what the others closed or gave up.
*/
export function grant(total: number, self: Share, others: (Share & { granted: number })[]): number {
const target = targets(total, [self, ...others]).get(self.name)!
const held = others.reduce((sum, o) => sum + Math.max(o.open, o.granted), 0)
return Math.max(0, Math.min(target, total - held))
}
/**
* A process's share of the NNTP account's connections (LOG.md 2026-10-18 (75)): the coordinator and the prober had a
* fixed split, and with history paused the coordinator's 38 stood idle while the prober's 8 were its limit. Each
* syncs every nntpShares.everyMs (what it wants and has open) and gets what it may open; one wanting less closes
* connections as they come free, and the other grows as those show up closed.
*/
export class NntpShare {
granted = 0
constructor(
readonly name: string,
private priority: number,
readonly min: number,
readonly max: number
) {}
async sync(want: number, open: number): Promise<number> {
const self: Share = { name: this.name, priority: this.priority, min: this.min, want: Math.min(Math.max(want, this.min), this.max), open }
const total = coordinatorConfig.nntpShares.total
this.granted = await db.transaction(async (trx) => {
await trx.rawQuery(`SELECT pg_advisory_xact_lock(?)`, [LOCK])
const { rows } = await trx.rawQuery(
`SELECT name, priority, min, want, open, granted FROM nntp_shares WHERE name <> ? AND heartbeat_at > now() - make_interval(secs => ?)`,
[this.name, STALE_SECONDS]
)
const granted = grant(total, self, rows)
await trx.rawQuery(
`INSERT INTO nntp_shares (name, priority, min, want, open, granted, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, now())
ON CONFLICT (name) DO UPDATE SET priority = excluded.priority, min = excluded.min, want = excluded.want,
open = excluded.open, granted = excluded.granted, heartbeat_at = now()`,
[this.name, this.priority, this.min, self.want, open, granted]
)
return granted
})
return this.granted
}
/** Stopping: its connections are closed, the others needn't wait for the row to go stale. */
async leave() {
await db.from('nntp_shares').where('name', this.name).delete()
}
}
app/services/nzb_service.ts
+6 -27
View File
@@ -2,28 +2,16 @@ import { gunzipSync } from 'node:zlib'
import db from '@adonisjs/lucid/services/db'
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
import { buildNzb } from '#lib/nzb/index'
import { SOURCES } from '#lib/release/source'
/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
/**
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
* chunk store: every chunk decoded, its files in name order. A file stored
* twice (the same name and first article, e.g. a range read again) is taken
* once. A known password (release_passwords) goes into the NZB's meta, where
* SABnzbd and NZBGet take it for unpacking.
* once.
*/
class NzbService {
/** The release's known passwords, the most trusted first. */
async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
}
async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
async files(releaseId: number): Promise<ChunkFile[]> {
const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
const files = new Map<string, ChunkFile>()
for (const chunk of chunks) {
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
@@ -34,11 +22,10 @@ class NzbService {
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
if (!release) return null
const password = (await this.passwords(releaseId))[0]?.password
// A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
if (Number(release.source) === SOURCES.spot) {
if (release.source === 'spot') {
const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
}
const xml = buildNzb(
(await this.files(releaseId)).map((f) => ({
@@ -48,18 +35,10 @@ class NzbService {
groups: f.groups,
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
})),
{ title: release.name, password }
{ title: release.name }
)
return { name: release.name, xml }
}
}
/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
export function withPassword(xml: string, password: string | undefined): string {
if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
const meta = `<meta type="password">${password.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')}</meta>`
if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
}
export default new NzbService()
app/services/par2_completeness.ts
-53
View File
@@ -1,53 +0,0 @@
import NzbService from '#services/nzb_service'
/** A file a par2 set describes (par2_files). */
export interface SetFile {
name: string
length: number
hash16k: string
}
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
/**
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
* many bytes (its articles' bytes, a little over the files' with yEnc).
*/
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
const names = new Set(files.map((f) => base(f.name)))
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
const byName = set.filter((f) => names.has(base(f.name))).length
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
if (byName > 0) return false
const data = files.filter((f) => !/\.par2$/i.test(f.name))
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
}
/**
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
* a transaction that moved chunks (a merge) reads them as moved.
*/
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
if (releaseIds.length === 0) return 0
const { rows: releases } = await client.rawQuery(
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
[releaseIds.map(String)]
)
let decided = 0
for (const { id } of releases) {
const { rows: set } = await client.rawQuery(
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
[id]
)
if (set.length === 0) continue
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
decided++
}
return decided
}
app/services/par2_merger.ts
+60 -243
View File
@@ -2,12 +2,8 @@ import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import { par2ReleaseName } from '#lib/release/index'
import { looksRandom } from '#lib/release/random'
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
import { randomPoster } from '#lib/release/identity'
import NzbService from '#services/nzb_service'
import { SidecarFolder } from '#services/sidecar_folder'
import { Par2Pruner } from '#services/par2_pruner'
import { pruneSevenZipTails } from '#services/seven_zip_tails'
import { decidePar2Completeness } from '#services/par2_completeness'
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
const MAX_SPREAD_MS = 48 * 3600 * 1000
@@ -19,45 +15,21 @@ const MAX_HOLDERS = 4
/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
const MAX_FILES_PER_SET_FILE = 3
const SLACK_FILES = 10
/**
* A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
* it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
*/
const COMMON_FILE = 20
/**
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
* NAME_CANDIDATES of them (a poster's uploads of a day).
*/
const NAME_SPREAD_MS = 12 * 3600 * 1000
const NAME_CANDIDATES = 200
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
/** Between rounds. */
const ROUND_MS = 60_000
/** A window looks back this far past the previous one's end (rows committed after it was read). */
/** A round looks back this far past the previous one's start (rows committed while it ran). */
const OVERLAP_MS = 60_000
/**
* A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
* two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
*/
const WINDOW_MS = 60 * 60_000
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
const FOLD_EVERY_MS = 5 * 60_000
/** Between par2_files prunes (complete and judged sets' file lists). */
const PRUNE_EVERY_MS = 60 * 60_000
/** A prune's share of the merger's time: the rest goes on in the next one. */
const PRUNE_BUDGET_MS = 2 * 60_000
/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
const CURSOR = 'merger'
export interface MergeRound {
sets: number
merges: number
releases: number
/** The name pass: merges, releases in them. */
nameMerges: number
nameReleases: number
/** The inner-name pass: stems checked, merges, releases in them. */
stems: number
stemMerges: number
@@ -72,33 +44,13 @@ export interface MergeRound {
*/
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
/**
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
*/
export function fileKey(name: string): string {
let n = name.replace(/^.*[\\/]/, '')
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
const read = Buffer.from(n, 'latin1').toString('utf8')
if (!read.includes('\ufffd')) n = read
}
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
}
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
const HOLDERS = `SELECT c.release_id FROM (
SELECT l.release_id, count(*) OVER () AS holders
FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
) c WHERE c.holders <= ${COMMON_FILE}`
interface Member {
id: number
release_key: string
group_id: number
files: number
posted_at: Date | null
readable: boolean
poster_id?: number | null
/** The poster's text (joined from posters) where a query needs it. */
poster?: string | null
file_total?: number | null
}
@@ -143,12 +95,11 @@ export class Par2Merger {
// First, at start too: a prune needn't wait for a long round.
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
prunedAt = Date.now()
const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
const pruned = await new Par2Pruner().round()
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
}
const round = await this.round()
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
} catch (error: any) {
@@ -158,18 +109,14 @@ export class Par2Merger {
}
}
/**
* The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
* lone inspection), a window at a time.
*/
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
async round(): Promise<MergeRound> {
// A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
const end = new Date(now[0].at)
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
if (!this.since) {
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
this.since = saved ? new Date(saved.at) : new Date(0)
}
const since = new Date(this.since.getTime() - OVERLAP_MS)
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
// par2 pass (hours when that starts over).
const folded = await this.folder.round()
@@ -183,170 +130,57 @@ export class Par2Merger {
foldedAt = Date.now()
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
}
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
while (this.since < end) {
const { rows } = await db.rawQuery(
`SELECT set_id FROM par2_sets WHERE created_at >= ?
UNION
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
[since, since]
)
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
for (const { set_id: setId } of rows) {
if (this.stopped) return round
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
if (from) {
const { touched, sets } = await this.setsToMerge(from, until)
round.sets += touched
for (const setId of sets) {
if (this.stopped) return round
await foldAgain()
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
const merged = await this.mergeSet(setId.trim()).catch((error) => {
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
return 0
})
if (merged > 0) {
round.merges++
round.releases += merged + 1
}
}
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
// 2026-10-23 (97)).
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
for (const { set_id: setId } of found) {
if (this.stopped) return round
const merged = await this.mergeByName(setId.trim()).catch((error) => {
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
return 0
})
if (merged > 0) {
round.nameMerges++
round.nameReleases += merged + 1
}
}
const { rows: stems } = await db.rawQuery(
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
[from, until]
)
for (const { stem } of stems) {
if (this.stopped) return round
if (!looksRandom(stem)) continue
round.stems++
const merged = await this.mergeStem(stem).catch((error) => {
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
return 0
})
if (merged > 0) {
round.stemMerges++
round.stemReleases += merged + 1
}
}
await foldAgain()
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
const merged = await this.mergeSet(setId.trim()).catch((error) => {
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
return 0
})
if (merged > 0) {
round.merges++
round.releases += merged + 1
}
// Only after a whole window: a restart halfway goes on from the window before.
this.since = until
await db.rawQuery(
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
[CURSOR, this.since]
)
}
const { rows: stems } = await db.rawQuery(
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i WHERE i.kind = 'lone' AND i.created_at >= ? AND ${INNER_STEM} IS NOT NULL`,
[since]
)
for (const { stem } of stems) {
if (this.stopped) return round
if (!looksRandom(stem)) continue
round.stems++
const merged = await this.mergeStem(stem).catch((error) => {
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
return 0
})
if (merged > 0) {
round.stemMerges++
round.stemReleases += merged + 1
}
}
// Only after a whole round: a restart halfway goes on from the round before.
this.since = now[0].at
await db.rawQuery(
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
[CURSOR, this.since]
)
return round
}
/** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
private async firstNew(from: Date, end: Date): Promise<Date | null> {
const { rows } = await db.rawQuery(
`SELECT least(
(SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
(SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
(SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
[from, from, from]
)
const at = rows[0].at ? new Date(rows[0].at) : null
return at && at <= end ? at : null
}
/**
* The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
* change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
* hour's 147k): mergeSet needn't look at them.
*/
private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
const { rows } = await db.rawQuery(
`WITH touched AS MATERIALIZED (
SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
UNION
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
WHERE h.created_at >= ? AND h.created_at <= ?
)
SELECT (SELECT count(*) FROM touched) AS touched, array(
SELECT t.set_id FROM touched t
WHERE EXISTS (
SELECT 1 FROM (
SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
UNION ALL
SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
) u HAVING count(DISTINCT u.release_id) >= 2)
) AS sets`,
[from, until, from, until]
)
return { touched: Number(rows[0].touched), sets: rows[0].sets }
}
/**
* The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
* and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
* sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
* again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
*/
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
const { rows: all } = await trx.rawQuery(
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
[setId]
)
const anchor: Member | undefined = all[0]
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
const holders = all.filter(
(h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
)
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
if (described.length === 0) return null
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
const at = new Date(anchor.posted_at).getTime()
const { rows: candidates } = await trx.rawQuery(
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
AND r.files <= ?
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
// More files than the set describes, with its sidecars, can't all be named in it.
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
)
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
for (const c of candidates) {
const own = await NzbService.files(Number(c.id), trx)
const content = own.filter((f) => !besideSet(f.name))
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
found.set(String(c.id), c)
files += Number(c.files)
}
if (found.size < 2) return null
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
return found
}
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
async mergeByName(setId: string): Promise<number> {
return db.transaction(async (trx) => {
const found = await this.nameMatches(trx, setId)
return found ? this.merge(trx, found, setId, 'name') : 0
})
}
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
async mergeStem(stem: string): Promise<number> {
return db.transaction(async (trx) => {
const { rows } = await trx.rawQuery(
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
`SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
FROM release_inspections i JOIN releases r ON r.id = i.release_id
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
[stem]
@@ -358,7 +192,7 @@ export class Par2Merger {
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
const near = (r: Member) =>
r.group_id === anchor.group_id &&
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
(r.poster ?? '') === (anchor.poster ?? '') &&
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
const found = new Map<string, Member>()
@@ -376,7 +210,7 @@ export class Par2Merger {
async mergeSet(setId: string): Promise<number> {
return db.transaction(async (trx) => {
const { rows: holders } = await trx.rawQuery(
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
WHERE s.set_id = ? ORDER BY r.id`,
[setId]
)
@@ -385,8 +219,8 @@ export class Par2Merger {
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
const { rows: matches } = await trx.rawQuery(
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
`SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
WHERE f.set_id = ?`,
[setId]
)
@@ -416,7 +250,7 @@ export class Par2Merger {
* Merges the found releases into the one with the most files (named by the par2 set's files when none
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
*/
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
{
// Locked in id order (the batch store and other rounds take the same rows).
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
@@ -428,28 +262,12 @@ export class Par2Merger {
await trx.rawQuery(
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
[target.id, key, via, others]
)
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
// isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
await trx.rawQuery(
`DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
[others, target.id, others]
)
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
}
// An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
await trx.rawQuery(
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
[target.id]
)
// The members' 7z tails together: those below the highest volume go.
await pruneSevenZipTails(trx, [target.id])
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
await trx.rawQuery(
@@ -464,26 +282,25 @@ export class Par2Merger {
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
first_seen_at = least(t.first_seen_at, s.first_seen_at),
first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
name = coalesce(nullif(?, ''), t.name)
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
max(file_total) AS file_total, max(inner_name) AS inner_name
FROM releases WHERE id = ANY(?)) s
WHERE t.id = ?`,
[name ?? '', others, target.id]
)
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
// Completeness as the settler decides it, once settled.
await trx.rawQuery(
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
[target.id]
)
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
await decidePar2Completeness(trx, [target.id])
return others.length
}
}
app/services/par2_pruner.ts
+43 -75
View File
@@ -1,107 +1,75 @@
import db from '@adonisjs/lucid/services/db'
import { decidePar2Completeness } from '#services/par2_completeness'
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
const SETTLED_MS = 2 * 86400_000
/** Sets per statement. */
const BATCH = 20_000
/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
const CURSOR = 'par2-prune'
/** The merger's row in worker_cursors: the end of its last whole window. */
const MERGER = 'merger'
/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
const MERGER_OVERLAP_MS = 60_000
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
const JUNK = ['dead', 'fake', 'repeated']
/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
const MERGE_KINDS = ['lone', 'par2']
export interface PruneRound {
complete: number
judged: number
final: number
rows: number
}
/**
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
* matches file hashes to releases and names an unreadable upload once; it's done when
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
* 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
* done when
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
* set (all of it merged into one);
* - judged: every release holding it has a junk verdict (dead, fake, repeated);
* - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
* it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
* and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
* describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
* - judged: every release holding it has a junk verdict (dead, fake, repeated).
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
* The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
* of a run, its ranges' probes not done yet) is looked at again on the next pass.
*/
export class Par2Pruner {
/** Sets per batch (BATCH; smaller in the tests). */
constructor(private batch = BATCH) {}
/**
* Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
* it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
* 2026-10-17 (72)).
*/
async round(budgetMs = Infinity): Promise<PruneRound> {
const deadline = Date.now() + budgetMs
const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
let after: string = cursors[0]?.key ?? ''
// Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
// still to come. Hashes since the merger's last window: it hasn't matched them yet.
const { rows: marks } = await db.rawQuery(
`SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
now() - make_interval(secs => ?) AS settled`,
[MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
)
const { probed, merged, settled } = marks[0]
async round(): Promise<PruneRound> {
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
let since: Date = saved ? new Date(saved.at) : new Date(0)
const until = new Date(Date.now() - SETTLED_MS)
for (;;) {
const { rows: sets } = await db.rawQuery(
`WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
SELECT b.set_id, CASE
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
r.settled_at IS NULL
OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
THEN 'final' END AS reason
FROM batch b ORDER BY b.set_id`,
[after, this.batch, probed, merged, settled, JUNK, settled]
`SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
[since, until, BATCH]
)
const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
for (const reason of ['complete', 'judged', 'final'] as const) {
const ids = done(reason)
round[reason] += ids.length
// The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
if (reason !== 'judged' && ids.length > 0) {
const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
}
round.rows += await this.drop(ids)
}
// Past the last set: the next batch starts the next pass.
after = sets.length < this.batch ? '' : sets.at(-1).set_id
if (sets.length === 0) break
const ids = sets.map((s: { set_id: string }) => s.set_id)
const { rows: done } = await db.rawQuery(
`SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
AND NOT EXISTS (
SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
[ids]
)
round.complete += done.length
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
since = sets.at(-1).at
await db.rawQuery(
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
[CURSOR, after]
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
[CURSOR, since]
)
if (after === '' || Date.now() > deadline) break
if (sets.length < BATCH) break
}
// Judged: no cursor (verdicts come and go); only sets with rows left.
const { rows: judged } = await db.rawQuery(
`SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
WHERE v.rule = ANY(?)
AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
AND NOT EXISTS (
SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
[JUNK, JUNK]
)
round.judged = judged.length
const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
return round
}
app/services/pending_ladder.ts
-423
View File
@@ -1,423 +0,0 @@
import db from '@adonisjs/lucid/services/db'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import coordinatorConfig from '#config/coordinator'
import { explored } from '#lib/rules/index'
import { usableName, type Inspection } from '#lib/probe/index'
import RuleSets from '#services/rule_sets'
import { twinKey, twinNames } from '#services/hash_twins'
/** pending_releases.status (migration 6400). */
export const PENDING = {
waiting: 0,
named: 1,
discarded: 2,
exhausted: 3,
naming: 4,
taken: 9,
} as const
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
export interface PendingTarget {
kind: string
messageId: string
file: string
/** The target article's bytes (candidates from 2026-11-02 on). */
bytes?: number
}
/** One probe the ladder hands the prober: the next target of a pending release. */
export interface PendingProbe {
/** pending_releases.release_key as hex. */
releaseKey: string
kind: string
messageId: string
file: string
tool: string
groupId: number
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
klass: number
poster: string
/** Targets probed before this one. */
tries: number
targets: number
}
export type PendingOutcome = {
probe: PendingProbe
status: 'done' | 'missing' | 'error' | 'waiting'
inspection?: Inspection
}
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
const rank = (kind: string) => {
const i = (EFFORT as readonly string[]).indexOf(kind)
return i < 0 ? EFFORT.length : i
}
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
}
/** A class's record (pending_classes), for the claim's order. */
interface ClassRow {
tool: string
groupId: number
klass: number
probed: number
named: number
waiting: number
}
/** The record is read again this often. */
const CLASSES_MS = 60_000
/** A class found short of a claim's share is left alone this long. */
const EMPTY_MS = 5 * 60_000
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
const PRIOR_RATE = 0.15
const PRIOR_WEIGHT = 10
/** One class takes at most this share of a claim: the rest explores the classes after it. */
const CLASS_SHARE = 0.25
/** The expected naming rate of a class: its record shrunk towards the prior. */
export function classScore(c: { probed: number; named: number }): number {
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
}
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
function claimed(rows: any[]): any[] {
const bytes = (r: any) =>
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
return [...rows].sort(
(a, b) =>
bytes(a) - bytes(b) ||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
Number(a.first_article) - Number(b.first_article)
)
}
function toProbe(r: any): PendingProbe {
const targets = byEffort(r.candidates as PendingTarget[])
const tries = Number(r.tries)
const target = targets[Math.min(tries, targets.length - 1)]
return {
releaseKey: r.release_key,
kind: target.kind,
messageId: target.messageId,
file: target.file,
tool: r.tool,
groupId: Number(r.group_id),
klass: Number(r.klass ?? 0),
poster: r.poster,
tries,
targets: targets.length,
}
}
/**
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
* classes with the best record first (classScore), the cheapest target first within one.
*/
class PendingLadder {
/** A previous prober's claims back in the queue. */
async release(): Promise<number> {
const { rowCount } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ?`,
[PENDING.waiting, PENDING.taken]
)
return rowCount ?? 0
}
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
private classes: ClassRow[] = []
private classesAt = 0
private empty = new Map<string, number>()
/** The record read again on the next claim (the tests). */
forget() {
this.classesAt = 0
this.empty.clear()
}
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
private async classRows(): Promise<ClassRow[]> {
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
const { rows } = await db.rawQuery(
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
)
this.classes = rows
.map((r: any) => ({
tool: r.tool as string,
groupId: Number(r.group_id),
klass: Number(r.klass),
probed: Number(r.probed),
named: Number(r.named),
waiting: Number(r.waiting),
}))
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
this.classesAt = Date.now()
return this.classes
}
/**
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
* classes, status 0 of a class without a row).
*/
async claim(n: number): Promise<PendingProbe[]> {
if (n <= 0) return []
const out: PendingProbe[] = []
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
const now = Date.now()
for (const c of await this.classRows()) {
if (out.length >= n) break
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
if ((this.empty.get(key) ?? 0) > now) continue
const want = Math.min(perClass, n - out.length)
const { rows } = await db.rawQuery(
`UPDATE pending_releases p SET status = ? FROM (
SELECT release_key FROM pending_releases
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
WHERE p.release_key = c.release_key
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
)
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
if (rows.length > 0) {
await db.rawQuery(
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
[rows.length, c.tool, c.groupId, c.klass]
)
}
out.push(...claimed(rows).map(toProbe))
}
if (out.length < n) {
const { rows } = await db.rawQuery(
`UPDATE pending_releases p SET status = ? FROM (
SELECT release_key FROM pending_releases
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
WHERE p.release_key = c.release_key
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
[PENDING.taken, PENDING.waiting, n - out.length]
)
out.push(...claimed(rows).map(toProbe))
}
return out
}
/**
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
* end of the ladder (status 3); the finished climbs counted on their signatures.
*/
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
if (outcomes.length === 0) return
// A twin by content names what the probe itself didn't (hash_twins.ts).
const twins = await twinNames(
outcomes
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
trx
)
const rows = outcomes.map((o) => {
const p = o.probe
if (o.status === 'waiting') {
return {
release_key: p.releaseKey,
status: PENDING.waiting,
tries: p.tries,
named: null,
inspection: null,
final: null,
}
}
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
const twin =
o.status === 'done' && o.inspection?.hash16k
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
: undefined
const name = probed ?? twin ?? null
const tries = p.tries + 1
const inspection = {
kind: p.kind,
status: o.status,
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
}
if (name)
return {
release_key: p.releaseKey,
status: PENDING.naming,
tries: 0,
named: name,
inspection,
final: 'named',
}
if (tries >= p.targets)
return {
release_key: p.releaseKey,
status: PENDING.exhausted,
tries,
named: null,
inspection,
final: 'exhausted',
}
return {
release_key: p.releaseKey,
status: PENDING.waiting,
tries,
named: null,
inspection,
final: null,
}
})
await trx.rawQuery(
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
WHERE p.release_key = decode(v.release_key, 'hex')`,
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
)
// The signatures: the poster's own and the tool's and group's together.
const counts = new Map<
string,
{ tool: string; group_id: number; poster: string; probed: number; named: number }
>()
for (const [i, o] of outcomes.entries()) {
const final = rows[i].final
if (!final) continue
for (const poster of [o.probe.poster, '']) {
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
const entry = counts.get(key) ?? {
tool: o.probe.tool,
group_id: o.probe.groupId,
poster,
probed: 0,
named: 0,
}
entry.probed++
if (final === 'named') entry.named++
counts.set(key, entry)
}
}
if (counts.size === 0) return
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
await trx.rawQuery(
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
[JSON.stringify([...counts.values()])]
)
await trx.rawQuery(
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
[
pendingMinProbes,
pendingMaxNamed,
JSON.stringify(
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
),
]
)
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
const classes = new Map<
string,
{
tool: string
group_id: number
klass: number
probed: number
named: number
waiting: number
}
>()
for (const [i, o] of outcomes.entries()) {
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
const entry = classes.get(key) ?? {
tool: o.probe.tool,
group_id: o.probe.groupId,
klass: o.probe.klass,
probed: 0,
named: 0,
waiting: 0,
}
const final = rows[i].final
if (final) entry.probed++
if (final === 'named') entry.named++
if (o.status === 'waiting') entry.waiting++
classes.set(key, entry)
}
await trx.rawQuery(
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
[JSON.stringify([...classes.values()])]
)
await trx.rawQuery(
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
[
pendingMinProbes,
pendingMaxNamed,
JSON.stringify(
[...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
),
]
)
}
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
async discard(limit = 10_000): Promise<number> {
const { samplePercent } = await RuleSets.current()
const { rows } = await db.rawQuery(
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
WHERE p.status = ? AND (EXISTS (
SELECT 1 FROM pending_signatures s
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
LIMIT ?`,
[PENDING.waiting, limit]
)
const keys = rows
.map((r: any) => r.release_key as string)
.filter((key: string) => !explored(key, samplePercent))
if (keys.length === 0) return 0
const { rows: gone } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
RETURNING tool, group_id, klass`,
[PENDING.discarded, PENDING.waiting, keys]
)
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
for (const g of gone) {
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
const entry = counts.get(key) ?? {
tool: g.tool,
group_id: Number(g.group_id),
klass: Number(g.klass),
n: 0,
}
entry.n++
counts.set(key, entry)
}
if (counts.size > 0) {
await db.rawQuery(
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
[JSON.stringify([...counts.values()])]
)
}
return gone.length
}
}
export default new PendingLadder()
app/services/pending_namer.ts
-229
View File
@@ -1,229 +0,0 @@
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import { setTimeout as sleep } from 'node:timers/promises'
import { promisify } from 'node:util'
import { unzip } from 'node:zlib'
import env from '#start/env'
import coordinatorConfig from '#config/coordinator'
import { XoverClient } from '#lib/nntp/index'
import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
import { identityKey } from '#lib/release/keys'
import BatchStore from '#services/batch_store'
import ChunkPartitions from '#services/chunk_partitions'
import Corpus from '#services/corpus'
import RuleSets from '#services/rule_sets'
import { PENDING } from '#services/pending_ladder'
const inflate = promisify(unzip)
interface NamedRow {
release_key: Buffer
identity_key: Buffer
group_id: number
named: string
first_article: number
last_article: number
file_total: number | null
tries: number
}
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
const MAX_MOVES = 3
/**
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
* connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
* release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
* candidates, which the prober answers from probe_cache. Runs inside the prober.
*/
export class PendingNamer {
private client: XoverClient | null = null
private usedAt = 0
constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
async run(stopped: () => boolean) {
while (!stopped()) {
let rows: NamedRow[] = []
try {
rows = await this.take(20)
for (const row of rows) {
if (stopped()) break
try {
await this.move(row)
} catch (error: any) {
logger.warn(
`Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
)
await db.rawQuery(
`UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
[row.release_key]
)
}
}
} catch (error: any) {
logger.error(`Naming: ${error.message}`)
}
if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
this.client.close()
this.client = null
}
if (rows.length === 0)
for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
}
this.client?.close()
this.client = null
}
/** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
async take(n: number): Promise<NamedRow[]> {
const { rows } = await db.rawQuery(
`SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
[PENDING.naming, MAX_MOVES, n]
)
return rows.map((r: any) => ({
...r,
group_id: Number(r.group_id),
first_article: Number(r.first_article),
last_article: Number(r.last_article),
tries: Number(r.tries),
}))
}
/** One named release into the catalog. */
async move(row: NamedRow): Promise<void> {
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
const range = await db
.from('ranges')
.where('group_id', row.group_id)
.where('start_id', '<=', row.first_article)
.where('end_id', '>=', row.first_article)
.orderBy('id')
.first()
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
const text = await this.overviewText(group.name, row)
const rules = await RuleSets.current()
const ix = new RangeIndexer({
group: group.name,
start: row.first_article,
end: row.last_article,
rules: { ...rules, samplePercent: 0 },
idleChunks: 0,
edgeArticles: 0,
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
partialFiles: true,
})
ix.addText(text, row.last_article + 1)
ix.finish()
const batch = ix.take()
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
if (!found)
throw new Error(
`its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
)
// The catalog row under the pending key (the identity, the group, the first article the edge saw).
const chunk: ReleaseChunk = {
...found,
name: row.named,
source: 'weak-stem',
readable: true,
finished: true,
edge: false,
firstArticle: row.first_article,
fileTotal: row.file_total ?? found.fileTotal,
tool: undefined,
}
const candidates = batch.candidates
.filter((c) => c.releaseKey === found.key)
.map((c) => ({ ...c, upload: found.upload }))
// The chunk's month partition, as the batch store makes sure of before its writes.
await ChunkPartitions.ensure()
await db.transaction(async (trx) => {
const blockId = await BatchStore.blockOf(trx, range)
await BatchStore.writeReleases(
trx,
{ groupId: row.group_id, blockId },
row.first_article,
[chunk],
candidates
)
await trx.rawQuery(
`UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
[row.named.slice(0, 255), row.release_key]
)
await trx.rawQuery(
`UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
[PENDING.named, row.release_key]
)
})
}
/** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
private async overviewText(groupName: string, row: NamedRow): Promise<string> {
if (this.corpusDir) {
const stored = await db
.from('ranges')
.where('group_id', row.group_id)
.whereNotNull('stored_at')
.where('start_id', '<=', row.last_article)
.where('end_id', '>=', row.first_article)
.orderBy('start_id')
const parts: string[] = []
let covered = true
for (const r of stored) {
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
const size = coordinatorConfig.chunkArticles
for (
let from =
range.start +
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
from <= Math.min(row.last_article, range.end);
from += size
) {
const raw = await Corpus.read(
range,
{ from, to: Math.min(from + size - 1, range.end) },
this.corpusDir
)
if (!raw) {
covered = false
break
}
parts.push((await inflate(raw)).toString('latin1'))
}
if (!covered) break
}
if (
covered &&
parts.length > 0 &&
stored.length > 0 &&
Number(stored[0].start_id) <= row.first_article &&
Number(stored[stored.length - 1].end_id) >= row.last_article
) {
return parts.join('')
}
}
this.client ??= await XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: true,
timeoutMs: coordinatorConfig.nntpTimeoutMs,
})
this.usedAt = Date.now()
try {
await this.client.group(groupName)
// One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
const raw = await this.client.xoverRaw(row.first_article, row.last_article)
return (await inflate(raw)).toString('latin1')
} catch (error) {
this.client.close()
this.client = null
throw error
}
}
}
app/services/planner.ts
+31 -177
View File
@@ -35,8 +35,20 @@ class Planner {
async planGroup(group: { id: number; name: string }): Promise<number> {
const info = await NntpPool.use((client) => client.group(group.name))
const size = coordinatorConfig.rangeArticles
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
const rows = historyRows(group.id, info, Number(last?.end ?? 0), coordinatorConfig.rangeArticles)
const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
const span = Math.max(1, info.high - info.low)
const rows = []
for (let start = from; start <= info.high; start += size) {
rows.push({
group_id: group.id,
start_id: start,
end_id: Math.min(start + size - 1, info.high),
next_id: start,
priority: Math.floor((1000 * (start - info.low)) / span),
})
}
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
return rows.length
@@ -63,99 +75,32 @@ class Planner {
* numbers come from LIST ACTIVE, one command per name prefix (GROUP per
* group took minutes for 1000 groups).
*/
async planTail(now = new Date()): Promise<TailPlanned> {
async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
const { rows: ends } = await db.rawQuery(
`SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
(status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
FROM ranges ORDER BY group_id, start_id DESC`,
[coordinatorConfig.tail.priority]
)
const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
)
if (groups.length === 0) return { groups: 0, ranges: 0 }
const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
const highs = new Map<string, number>()
await NntpPool.use(async (client) => {
for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
}
})
const plan = tailRanges(
groups.map((g) => {
const l = last.get(Number(g.id))
return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
}),
const rows = tailRanges(
groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
now,
{ rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
)
return this.applyTail(plan, now)
}
/** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
const rows = [...plan.rows]
let extended = 0
if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
if (rows.length === 0) return { groups: 0, ranges: 0 }
await db.transaction(async (trx) => {
for (const e of plan.extended) {
const { rows: done } = await trx.rawQuery(
`UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
[e.end_id, e.id, e.from - 1]
)
if (done.length > 0) extended++
else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
await trx.table('ranges').multiInsert(rows)
for (const id of new Set(rows.map((r) => r.group_id))) {
const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
await trx.from('groups').where('id', id).update({ high, planned_at: now })
}
if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
const highs = new Map<number, number>()
for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
})
return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
}
/**
* Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
* RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
* (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
*/
async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
const { rows: candidates } = await db.rawQuery(
`SELECT id, group_id, start_id, end_id, priority FROM ranges
WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
ORDER BY group_id, start_id`
)
const merges = tailMerges(
candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
coordinatorConfig.rangeArticles
)
const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
await db.transaction(async (trx) => {
// The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
const { rows: gone } = await trx.rawQuery(
`DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
[merges.flatMap((m) => m.drop)]
)
if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
const { rows: grown } = await trx.rawQuery(
`UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
[merges.map((m) => m.id), merges.map((m) => m.end_id)]
)
if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
})
return { candidates: candidates.length, merged: merges.length, dropped }
}
}
export interface TailPlanned {
groups: number
/** Rows added. */
ranges: number
/** Open ranges grown. */
extended: number
}
/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
@@ -172,119 +117,28 @@ export interface TailGroup {
plannedAt: Date
/** The server's newest article (null: not listed). */
high: number | null
/**
* Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
* nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
*/
open?: { id: number; start: number } | null
}
export interface TailRow {
group_id: number
start_id: number
end_id: number
next_id: number
priority: number
}
export interface TailPlan {
/** New ranges. */
rows: TailRow[]
/** Open ranges grown to end_id (from: the first new article, the old end + 1). */
extended: { id: number; group_id: number; from: number; end_id: number }[]
}
/**
* The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
* the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
* tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
* The tail's new ranges: per group, the articles after its last range, in
* ranges of at most `rangeArticles`, once there are `minArticles` of them or
* the group's last planning is `maxDelayMinutes` ago.
*/
export function tailRanges(
groups: TailGroup[],
now: Date,
options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
): TailPlan {
const rows: TailRow[] = []
const extended: TailPlan['extended'] = []
) {
const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
for (const g of groups) {
if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
const waiting = g.high - g.plannedTo
const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
let start = g.plannedTo + 1
if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
start = end + 1
}
for (; start <= g.high; start += options.rangeArticles) {
for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
}
}
return { rows, extended }
}
export interface TailMerge {
/** The run's first range, grown to end_id. */
id: number
end_id: number
/** The ranges it absorbed. */
drop: number[]
}
/**
* Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
* `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
*/
export function tailMerges(
ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
rangeArticles: number
): TailMerge[] {
const merges: TailMerge[] = []
let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
const flush = () => {
if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
}
for (const r of ranges) {
if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
cur.end_id = r.end_id
cur.drop.push(r.id)
continue
}
flush()
cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
}
flush()
return merges
}
/**
* The rows a group's history is cut into, from `lastEnd + 1` (0: the group has no ranges yet) up to the high
* watermark: RANGE_ARTICLES slices at the position priority. A group planned for the first time also gets one
* "ancient" range, articles 1 … low − 1: what had expired on the server before it was planned. Status
* 'ancient', never leased or fetched; it keeps the tiling complete from article 1 (migration 6700 added it for
* the groups planned before 2026-10-28).
*/
export function historyRows(
groupId: number,
info: { low: number; high: number },
lastEnd: number,
size: number
): Record<string, unknown>[] {
const rows: Record<string, unknown>[] = []
if (lastEnd === 0 && info.low > 1)
rows.push({ group_id: groupId, start_id: 1, end_id: info.low - 1, next_id: 1, priority: 0, status: 'ancient' })
const from = Math.max(info.low, lastEnd + 1)
const span = Math.max(1, info.high - info.low)
for (let start = from; start <= info.high; start += size) {
rows.push({
group_id: groupId,
start_id: start,
end_id: Math.min(start + size - 1, info.high),
next_id: start,
priority: Math.floor((1000 * (start - info.low)) / span),
})
}
return rows
}
app/services/prefetcher.ts
+28 -199
View File
@@ -3,8 +3,7 @@ import logger from '@adonisjs/core/services/logger'
import coordinatorConfig from '#config/coordinator'
import NntpPool from '#services/nntp_pool'
import Spool, { type ChunkSpan } from '#services/spool'
import Corpus, { type CorpusFilter } from '#services/corpus'
import RuleSets from '#services/rule_sets'
import Corpus from '#services/corpus'
import CatalogSpace from '#services/catalog_space'
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
@@ -73,13 +72,7 @@ export class Prefetcher {
private writingNow = 0
private sampler?: NodeJS.Timeout
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
private inFlight = new Set<{
rangeId: number
from: number
to: number
since: number
client: XoverClient
}>()
private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
/** Capture ranges found complete in the corpus (not looked at again). */
private stored = new Set<number>()
@@ -138,8 +131,6 @@ export class Prefetcher {
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
.finally(() => this.running--)
}
// Work left with every loop running: the pool's share could be bigger.
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
}
stop() {
@@ -154,9 +145,6 @@ export class Prefetcher {
/** One connection's loop: chunks until there is nothing left to fetch. */
private async work() {
while (!this.stopped) {
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
if (this.running > this.loops) return
const next = this.take()
if (!next) return
const { target, span } = next
@@ -169,14 +157,10 @@ export class Prefetcher {
}
const corpusRange = { group: target.group, start: target.start, end: target.end }
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
let data =
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
const stored = data !== null
try {
if (!data)
data = target.plain.has(span.from)
? await NntpPool.usePlain(fetch)
: await this.fetchHedged(target.id, target.group, span)
if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
} finally {
this.fetchingNow--
}
@@ -186,8 +170,7 @@ export class Prefetcher {
this.stats.chunks++
this.stats.articles += span.to - span.from + 1
this.stats.fetchMs += Date.now() - fetchedAt
if (target.capture)
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
}
if (target.corpusOnly) {
this.stats.captured++
@@ -204,8 +187,7 @@ export class Prefetcher {
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
[target.id]
)
if (Number(rows[0]?.spooled_chunks) >= target.total)
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
} finally {
this.writingNow--
this.stats.writeMs += Date.now() - writtenAt
@@ -227,10 +209,7 @@ export class Prefetcher {
*/
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
let age: number | undefined
const request = (
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
first: boolean
): Request<Buffer> | null => {
const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
let client: XoverClient | undefined
let cancelled = false
const done = start(async (c) => {
@@ -262,9 +241,7 @@ export class Prefetcher {
data = await c.xoverRaw(span.from, span.to)
} catch (error) {
if (!late) throw error
throw new Error(
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
)
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
} finally {
clearInterval(watch)
this.inFlight.delete(entry)
@@ -307,8 +284,7 @@ export class Prefetcher {
/** The longest-running request: range, chunk, how long, how much came in. */
private oldest() {
let oldest:
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
if (!oldest) return 'none in flight'
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
@@ -333,12 +309,7 @@ export class Prefetcher {
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
if (chunks > 0) {
const ages = byAge
.map((b, i) =>
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
'– min',
'+ min'
)
)
.map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
.join(', ')
logger.info(
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
@@ -364,18 +335,13 @@ export class Prefetcher {
}
private peek(): boolean {
return this.targets.some(
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
)
return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
}
/** Whether the target is under its range's and its group's limit of connections. */
private open(target: Target) {
const { perRange, perGroup } = coordinatorConfig.spool
return (
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
)
return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
}
private count(target: Target, by: number) {
@@ -406,203 +372,66 @@ export class Prefetcher {
this.sizedAt = Date.now()
this.spoolBytes = await Spool.size()
}
const mine = (q: any) =>
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
const rows = (q: any) =>
mine(
q
.from('ranges as r')
.join('groups as g', 'g.id', 'r.group_id')
.select(
'r.id',
'g.name as group',
'r.start_id',
'r.end_id',
'r.next_id',
'r.plain_chunks',
'r.capture'
)
)
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
// The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
// cursors (a group's history and tail lanes are two cursors).
const held: { group: number; lane: string }[] = (
await db
.from('group_leases')
.whereNotNull('node_id')
.where(
'heartbeat_at',
'>=',
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
)
.select('group_id', 'lane')
).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
const ahead = Math.max(
coordinatorConfig.spool.minAhead,
coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
)
const started = await rows(db.query())
.where('r.status', 'planned')
.where('r.spooled_chunks', '>', 0)
.whereNull('r.spooled_at')
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
const [{ ready }] = await mine(
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
).count('* as ready')
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
const { perRange } = coordinatorConfig.spool
const short =
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
const want = Math.max(ahead - Number(ready), short)
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
const full = await CatalogSpace.full()
// Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
// leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
// at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
// window worker running) nothing is fetched ahead for workers.
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
const tailPriority = coordinatorConfig.tail.priority
const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
const fresh =
room && want > 0 && heldHistory.length + heldTail.length > 0
? await rows(db.query())
.where('r.status', 'planned')
.where('r.spooled_chunks', 0)
.whereRaw('r.next_id <= r.end_id')
.where((q: any) =>
q
.where((h: any) =>
h
.whereIn('r.group_id', heldHistory)
.where('r.priority', '>', tailPriority)
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
.where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
)
.orWhere((t: any) =>
t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
)
)
.orderByRaw(
'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
[tailPriority]
)
.limit(want)
: []
const fresh = room && want > 0
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
: []
const targets: Target[] = []
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
const start = Number(row.start_id)
const end = Number(row.end_id)
const total = Spool.chunksOf({ start, end }).length
const missing: ChunkSpan[] = []
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
continue
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
missing.push(span)
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
}
if (missing.length > 0) {
targets.push({
id: Number(row.id),
group: row.group,
plain: new Set((row.plain_chunks ?? []).map(Number)),
missing,
total,
start,
end,
capture: row.capture === true,
})
targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
}
}
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
targets.push(target)
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
return targets
}
private filterRules: { at: number; filter: CorpusFilter } | null = null
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
private async corpusFilter(): Promise<CorpusFilter | undefined> {
if (!coordinatorConfig.corpus.filter) return undefined
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
const rules = await RuleSets.current()
this.filterRules = {
at: Date.now(),
filter: { ruleVersion: rules.version, priorities: rules.priorities },
}
}
return this.filterRules.filter
}
/**
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
* gets stored_at and isn't read again.
*/
/** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
const { dir, ahead } = coordinatorConfig.corpus
if (!dir || ahead <= 0) return []
const { perRange, perGroup } = coordinatorConfig.spool
const ofGroup =
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
const groups = new Map<string, number>()
const candidates = await rows(db.query())
.where('r.capture', true)
// A group taken off the list (groups.active) isn't worth the connections.
.where('g.active', true)
.whereNull('r.stored_at')
.where((q: any) =>
q
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
.orWhere('r.status', 'done')
)
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
.limit(ahead * 16)
const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
const targets: Target[] = []
for (const row of candidates) {
if (targets.length >= ahead) break
const id = Number(row.id)
if (taken.has(id) || this.stored.has(id)) continue
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
const start = Number(row.start_id)
const end = Number(row.end_id)
const range = { group: row.group, start, end }
if (await Corpus.complete(range)) {
this.stored.add(id)
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
continue
}
const chunks = Spool.chunksOf({ start, end })
const missing: ChunkSpan[] = []
for (const span of chunks)
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
missing.push(span)
for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
if (missing.length > 0) {
targets.push({
id,
group: row.group,
plain: new Set((row.plain_chunks ?? []).map(Number)),
missing,
total: chunks.length,
start,
end,
capture: true,
corpusOnly: true,
})
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
}
}
return targets
app/services/prober.ts
+74 -234
View File
@@ -6,28 +6,19 @@ import coordinatorConfig from '#config/coordinator'
import { XoverClient } from '#lib/nntp/index'
import { YencError } from '#lib/yenc/index'
import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
import { NntpShare } from '#services/nntp_shares'
import { DiskSpace } from '#services/disk_space'
import { decidePar2Completeness } from '#services/par2_completeness'
import { twinKey, twinNames } from '#services/hash_twins'
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
import { PendingNamer } from '#services/pending_namer'
interface Candidate {
id: number
releaseId: number
kind: string
messageId: string
/** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
pending?: PendingProbe
}
type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
/**
* The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
* then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
*/
const TIERS: { kind: string; unreadable: boolean }[] = [
{ kind: 'lone', unreadable: false },
@@ -35,35 +26,8 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
{ kind: 'archive', unreadable: true },
{ kind: 'par2', unreadable: false },
{ kind: 'archive', unreadable: false },
{ kind: '7z-tail', unreadable: true },
{ kind: '7z-tail', unreadable: false },
]
const FLUSH_MS = 2000
/** Between sweeps that discard the waiting releases of dropped signatures. */
const SWEEP_MS = 5 * 60_000
/**
* JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
* and the whole flush failed with it (LOG.md 2026-10-18 (78)).
*/
function jsonb(value: unknown): string {
return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
}
/** Rounds of every lane's pipeline still queued when the next claim starts. */
const LOW_ROUNDS = 8
/**
* The candidate queue's levels: the next claim starts below `low`, LOW_ROUNDS rounds of every lane's pipeline (at
* ~0.2 s an article and a pipeline of 8, ~13 s of work), and fills it to `target`, twice that. A claim takes 1.5–4 s
* and more (random slices of an 11 M-row table); refilling at half of four rounds left 3–6 s, and at 20–24 lanes the
* lanes waited for candidates up to 19% of their time (LOG.md 2026-10-28 (120)). At least `minBatch` either way.
*/
export function queueLevels(lanes: number, pipeline: number, minBatch: number): { low: number; target: number } {
const low = Math.max(minBatch, lanes * pipeline * LOW_ROUNDS)
return { low, target: 2 * low }
}
/** Slices a claim takes from random points of the queue. */
const SLICES = 20
const REPORT_MS = 60_000
@@ -82,31 +46,18 @@ export class Prober {
private refilling: Promise<void> | null = null
private stopped = false
private stats = Prober.freshStats()
/** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
private lanes = new Set<Promise<void>>()
private running = 0
/** The last lane to look found nothing to claim: the prober wants its minimum only. */
private dry = false
/** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */
private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb)
private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections)
/** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
private static freshStats() {
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
}
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
private namer = new PendingNamer()
private sweptAt = 0
async run() {
// A previous run's claims: one prober at a time.
const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
const released = await PendingLadder.release()
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
const connections = coordinatorConfig.probe.connections
logger.info(`Probing with ${connections} connections.`)
const flusher = (async () => {
while (!this.stopped || this.outcomes.length > 0) {
await sleep(FLUSH_MS)
@@ -116,27 +67,7 @@ export class Prober {
this.report()
}
})()
// Lanes as the share allows: the coordinator's history comes first (app/services/nntp_shares.ts).
while (!this.stopped) {
try {
const was = this.share.granted
const granted = await this.share.sync(this.dry ? 0 : this.share.max, this.running)
if (granted !== was) logger.info(`Probing: ${granted} connections (was ${was}).`)
while (this.running < granted) {
this.running++
const lane: Promise<void> = this.lane()
.catch((error) => logger.error(`Probing: a lane failed: ${error.message}`))
.finally(() => this.lanes.delete(lane))
this.lanes.add(lane)
}
} catch (error: any) {
logger.error(`Probing: NNTP share failed: ${error.message}`)
}
for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
}
await Promise.all(this.lanes)
await namer
await this.share.leave().catch(() => {})
await Promise.all(Array.from({ length: connections }, () => this.lane()))
// What was claimed and not probed goes back.
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
this.stopped = true
@@ -147,95 +78,68 @@ export class Prober {
this.stopped = true
}
/**
* One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure; done when
* there are more lanes than the share (its connection closed, the coordinator takes it on its next sync).
*/
/** One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure. */
private async lane() {
let client: XoverClient | null = null
let usedAt = Date.now()
try {
while (!this.stopped) {
if (this.running > this.share.granted) break
const batch: Candidate[] = []
const waited = Date.now()
while (batch.length < coordinatorConfig.probe.pipeline) {
const candidate = await this.next()
if (!candidate) break
batch.push(candidate)
}
this.stats.waitMs += Date.now() - waited
this.dry = batch.length === 0
if (batch.length === 0) {
// Nothing to probe for a while: the connection closes (nntpIdleCloseMs), and opens again with work.
const idleMs = coordinatorConfig.nntpIdleCloseMs
if (client && idleMs > 0 && Date.now() - usedAt >= idleMs) {
client.close()
client = null
while (!this.stopped) {
const batch: Candidate[] = []
const waited = Date.now()
while (batch.length < coordinatorConfig.probe.pipeline) {
const candidate = await this.next()
if (!candidate) break
batch.push(candidate)
}
this.stats.waitMs += Date.now() - waited
if (batch.length === 0) {
this.stats.idleMs += 30_000
await sleep(30_000)
continue
}
try {
client ??= await XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: false,
timeoutMs: coordinatorConfig.nntpTimeoutMs,
})
const fetched = Date.now()
const bodies = await client.bodies(batch.map((c) => c.messageId))
this.stats.fetchMs += Date.now() - fetched
const inspected = Date.now()
for (const [i, candidate] of batch.entries()) {
const body = bodies[i]
this.stats.articles++
if (body === null) {
// Taken down or expired: final.
this.outcomes.push({ candidate, status: 'missing' })
continue
}
this.stats.idleMs += 30_000
await sleep(30_000)
continue
}
try {
client ??= await XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: false,
timeoutMs: coordinatorConfig.nntpTimeoutMs,
})
const fetched = Date.now()
const bodies = await client.bodies(batch.map((c) => c.messageId))
this.stats.fetchMs += Date.now() - fetched
const inspected = Date.now()
for (const [i, candidate] of batch.entries()) {
const body = bodies[i]
this.stats.articles++
if (body === null) {
// Taken down or expired: final.
this.outcomes.push({ candidate, status: 'missing' })
continue
}
this.stats.bytes += body.length
try {
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
} catch (error: any) {
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
this.outcomes.push({ candidate, status: 'error' })
}
this.stats.bytes += body.length
try {
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
} catch (error: any) {
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
this.outcomes.push({ candidate, status: 'error' })
}
this.stats.inspectMs += Date.now() - inspected
usedAt = Date.now()
} catch (error: any) {
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
client?.close()
client = null
await sleep(5000)
}
this.stats.inspectMs += Date.now() - inspected
} catch (error: any) {
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
client?.close()
client = null
await sleep(5000)
}
} finally {
client?.close()
this.running--
}
}
/** The queue's low-water mark and the level a claim fills it to (queueLevels). */
private get levels() {
return queueLevels(this.running, coordinatorConfig.probe.pipeline, coordinatorConfig.probe.batch)
}
/** Claims to the queue's target; the claim after starts when it's below its low-water mark (queueLevels). */
private get batch() {
return this.levels.target
client?.close()
}
private async next(): Promise<Candidate | undefined> {
if (this.queue.length < this.levels.low) {
if (this.queue.length < coordinatorConfig.probe.batch / 2) {
this.refilling ??= this.claim().finally(() => (this.refilling = null))
if (this.queue.length === 0) await this.refilling
}
@@ -258,55 +162,13 @@ export class Prober {
}
private async claimTiers() {
// The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
if (await this.space.short()) return
await this.claimCatalog()
await this.claimPending()
}
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
private async claimPending() {
if (Date.now() - this.sweptAt > SWEEP_MS) {
this.sweptAt = Date.now()
this.stats.discarded += await PendingLadder.discard()
}
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
const probes = await PendingLadder.claim(want)
if (probes.length === 0) return
const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
probes.map((p) => p.messageId),
PROBE_VERSION,
])
const known = new Map(cached.map((c: any) => [c.message_id, c]))
for (const probe of probes) {
const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
const hit: any = known.get(probe.messageId)
if (hit) {
this.stats.cached++
this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
} else {
this.queue.push(candidate)
}
}
}
private async claimCatalog() {
for (const { kind, unreadable } of TIERS) {
const want = this.batch - this.queue.length
const want = coordinatorConfig.probe.batch - this.queue.length
if (want <= 0) return
// The starting points chosen here: in the query Postgres drew random() once for all slices.
// Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
// Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting
// entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)).
const values = unreadable ? [true] : [false, true]
const ends = values
.map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`)
.join(' UNION ALL ')
const [{ lo, hi }] = (
await db.rawQuery(
`SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`,
values.flatMap((v) => [kind, v])
)
await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
).rows
if (lo === null) continue
const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
@@ -341,31 +203,22 @@ export class Prober {
/** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
private async flush() {
const taken = this.outcomes.splice(0)
if (taken.length === 0) return
for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
// The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
const pending = taken.filter((o) => o.candidate.pending)
const outcomes = taken.filter((o) => !o.candidate.pending)
this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
const outcomes = this.outcomes.splice(0)
if (outcomes.length === 0) return
for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
await db.transaction(async (trx) => {
if (outcomes.length > 0) {
await trx.rawQuery(
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
[jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
)
}
await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
await trx.rawQuery(
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
[JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
)
// Final answers read from the news server, kept by message-id for the next catalog (a reset).
// One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
// row twice in one statement.
const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
const fresh = outcomes.filter((o) => !o.cached && o.status !== 'waiting')
if (fresh.length > 0) {
await trx.rawQuery(
`INSERT INTO probe_cache (message_id, status, result, version)
SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
[PROBE_VERSION, jsonb(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
[PROBE_VERSION, JSON.stringify(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
)
}
const done = outcomes.filter((o) => o.inspection)
@@ -375,11 +228,9 @@ export class Prober {
SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
// A par2 set's file list goes to par2_files, not into every inspection.
[jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
[JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
)
await this.storeForMerge(trx, done)
// A par2 index read: the release's completeness by its set, when it has no counter.
await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
// Each touched release from all its inspections (earlier rounds may have stored others).
const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
const { rows } = await trx.rawQuery(
@@ -393,25 +244,15 @@ export class Prober {
entry.all.push({ kind: r.kind, result: r.result })
byRelease.set(id, entry)
}
// A twin by content names a release its own inspections didn't (hash_twins.ts; LOG.md (130)).
const twins = await twinNames(
done.filter((o) => o.inspection!.hash16k && o.inspection!.size).map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
trx
)
const twinOf = new Map<number, string>()
for (const o of done) {
const twin = o.inspection!.hash16k ? twins.get(twinKey(o.inspection!.hash16k, o.inspection!.size)) : undefined
if (twin && twin !== byRelease.get(o.candidate.releaseId)?.name) twinOf.set(o.candidate.releaseId, twin)
}
const updates = [...byRelease].map(([id, { name, all }]) => ({
id,
passworded: all.some((i) => i.result.passworded === true),
inner_name: innerName(all, name) ?? twinOf.get(id) ?? null,
inner_name: innerName(all, name),
}))
await trx.rawQuery(
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
[jsonb(updates)]
[JSON.stringify(updates)]
)
})
}
@@ -424,7 +265,7 @@ export class Prober {
`INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
ON CONFLICT (candidate_id) DO NOTHING`,
[jsonb(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
[JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
)
}
for (const o of done) {
@@ -434,7 +275,7 @@ export class Prober {
`INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
ON CONFLICT DO NOTHING`,
[set.setId, jsonb(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
[set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
)
await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
}
@@ -446,8 +287,7 @@ export class Prober {
const { articles, bytes, done, missing, error } = this.stats
logger.info(
`Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
`the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
`Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
`idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
`claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
app/services/range_blocks.ts
+31
View File
@@ -0,0 +1,31 @@
import db from '@adonisjs/lucid/services/db'
/**
* range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
* release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
* range that isn't done or doesn't start where the one before ended.
*/
class RangeBlocks {
async rebuild(): Promise<number> {
return db.transaction(async (trx) => {
await trx.rawQuery(`DELETE FROM range_blocks`)
const result = await trx.rawQuery(
`WITH r AS (
SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
min(start_id) OVER (PARTITION BY group_id) AS group_first,
lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
runs AS (
SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
OVER (PARTITION BY group_id ORDER BY start_id) AS run
FROM r)
INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
FROM runs WHERE done GROUP BY group_id, run`
)
return result.rowCount ?? 0
})
}
}
export default new RangeBlocks()
app/services/release_judge.ts
+53 -184
View File
@@ -1,19 +1,10 @@
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import { SOURCES } from '#lib/release/source'
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
/** Raised when a rule changes: verdicts of an older version are judged again. */
export const JUDGE_VERSION = 1
/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
const ROUND_MS = 2 * 60 * 1000
/**
* A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
* (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
* after a merge (repeated).
*/
const FULL_MS = 24 * 60 * 60 * 1000
/** Changed releases judged at a time. */
const CHUNK = 50_000
/** Between rounds: each rule is a scan of its table. */
const ROUND_MS = 60 * 60 * 1000
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
const REPEATED_RELEASES = 20
@@ -33,15 +24,10 @@ export interface Rule {
name: string
/** Why, in a few words (the log, the web interface). */
says: string
/** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
sql: (scoped: boolean) => string
/** SELECT of the release ids the rule judges. */
sql: string
}
/** A join to the releases in scope (an incremental round), on the given release id column. */
const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
/**
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
* the header pass, the prober, the merges and the spots know by now.
@@ -50,33 +36,26 @@ export const RULES: Rule[] = [
{
name: 'dead',
says: 'every probed first article is gone from the server (taken down or expired)',
sql: (scoped) =>
`SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
},
{
name: 'fake',
says: 'named as a film, episode or music, but the content is a program',
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
// an executable by name or by its first bytes, under a media name.
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
UNION
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
},
{
name: 'repeated',
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
// Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
// (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
sql: (scoped) =>
scoped
? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
: `SELECT DISTINCT h.release_id FROM file_hashes h
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
},
{
@@ -85,28 +64,21 @@ export const RULES: Rule[] = [
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
sql: (scoped) =>
scoped
? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
: `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
sql: `SELECT r.id FROM releases r
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
},
{
name: 'locked',
says: 'passworded, and no spot, NZB or title gives the password',
// releases_passworded_idx: a scan of all of releases was 30 s a round.
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
},
{
name: 'unusable',
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
// Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
sql: `SELECT r.id FROM releases r
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
AND r.settled_at IS NOT NULL
AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
@@ -114,18 +86,15 @@ export const RULES: Rule[] = [
]
export interface JudgeRound {
/** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
releases?: number
/** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
/** Per rule: releases it judges now, verdicts added, verdicts taken back. */
rules: Record<string, { matched: number; added: number; removed: number }>
}
/**
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
* minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
* daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
* verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
* round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
* matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
*/
export class ReleaseJudge {
private stopped = false
@@ -138,13 +107,9 @@ export class ReleaseJudge {
while (!this.stopped) {
const started = Date.now()
try {
if (await this.fullDue()) {
const round = await this.full()
if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
} else {
const round = await this.changed()
if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
}
const round = await this.round()
const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
logger.info(`Judged: ${parts.join(', ')}.`)
} catch (error: any) {
logger.error(`Judge round failed: ${error.message}`)
}
@@ -152,139 +117,43 @@ export class ReleaseJudge {
}
}
/** A full round is due: a day after the last, or the last was by another rule set. */
async fullDue(): Promise<boolean> {
const state = await db.from('judge_state').first()
return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
}
/**
* Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
* half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
*/
async full(dry = false): Promise<JudgeRound> {
const started = new Date()
/** One round over every rule; with dry, only counts. */
async round(dry = false): Promise<JudgeRound> {
const round: JudgeRound = { rules: {} }
// Verdicts of an older rule set are judged again from scratch.
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
await db.rawQuery(`TRUNCATE judge_full`)
const judged = new Set<string>()
for (const rule of RULES) {
if (this.stopped) return round
const { rowCount } = await db.rawQuery(
`INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
[rule.name]
)
round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
}
if (dry || this.stopped) return round
await db.rawQuery(`ANALYZE judge_full`)
await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
await db.rawQuery(
`INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
[started, JUDGE_VERSION]
)
return round
}
/**
* The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
* A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
* already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
* file in that many without them had its releases judged before.
*/
async changed(): Promise<JudgeRound> {
const round: JudgeRound = { releases: 0, rules: {} }
for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
let more = true
while (more && !this.stopped) {
await db.transaction(async (trx) => {
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
const { rows } = await trx.rawQuery(
`WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
SELECT (SELECT count(*) FROM taken) AS taken`
)
const taken = Number(rows[0].taken)
more = taken === CHUNK
if (taken === 0) return
// The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
await trx.rawQuery(
`CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
)
await trx.rawQuery(
`INSERT INTO judge_tipped
SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
CROSS JOIN LATERAL (
SELECT count(DISTINCT f.release_id) AS total,
count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
WHERE c.total >= ${REPEATED_RELEASES}`
)
await trx.rawQuery(
`INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
ON CONFLICT DO NOTHING`
)
await trx.rawQuery(`ANALYZE judge_scope`)
const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
round.releases! += Number(scope[0].n)
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
for (const rule of RULES) {
const { rowCount } = await trx.rawQuery(
`INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
if (this.stopped) break
const { rows } = await db.rawQuery(rule.sql)
const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
for (const id of ids) judged.add(id)
const stats = { matched: ids.length, added: 0, removed: 0 }
if (!dry) {
await db.transaction(async (trx) => {
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
for (let i = 0; i < ids.length; i += 50_000) {
await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
}
const removed = await trx.rawQuery(
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
[rule.name]
)
round.rules[rule.name].matched += rowCount ?? 0
}
await trx.rawQuery(`ANALYZE judged`)
await this.apply(trx, 'judged', true, round)
})
const added = await trx.rawQuery(
`INSERT INTO release_verdicts (release_id, rule, rule_version)
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
WHERE release_verdicts.rule <> excluded.rule`,
[rule.name, JUDGE_VERSION]
)
stats.removed = removed.rowCount ?? 0
stats.added = added.rowCount ?? 0
})
}
round.rules[rule.name] = stats
}
return round
}
/**
* The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
* more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
* insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
*/
private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
for (const { rule, n } of rows) {
round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
round.rules[rule][field] += Number(n)
}
}
const { rows: removed } = await trx.rawQuery(
`WITH gone AS (DELETE FROM release_verdicts v
WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
RETURNING v.rule)
SELECT rule, count(*) AS n FROM gone GROUP BY 1`
)
tally(removed, 'removed')
const { rows: added } = await trx.rawQuery(
`WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
WHERE release_verdicts.rule <> excluded.rule
RETURNING rule)
SELECT rule, count(*) AS n FROM written GROUP BY 1`,
[JUDGE_VERSION]
)
tally(added, 'added')
}
}
/** "dead 1,157,018 (+1,136 −712), …" for the log. */
function describe(round: JudgeRound): string {
return Object.entries(round.rules)
.map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
.join(', ')
}
app/services/search_service.ts
+33 -65
View File
@@ -1,7 +1,6 @@
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import NzbService from '#services/nzb_service'
import { SOURCES, sourceName } from '#lib/release/source'
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
@@ -52,14 +51,12 @@ export interface ReleaseSummary {
size: number
readable: boolean
complete: boolean | null
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
final: boolean
category: number
settled: boolean
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
passworded: boolean | null
/** A password is known (release_passwords): it's in the NZB. */
passwordKnown: boolean
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
innerName: string | null
}
@@ -85,19 +82,6 @@ function contains(word: string): string {
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
}
/**
* A search word's condition on release_texts: anywhere in a text (ILIKE), or with a leading "^" at a word's start, the
* text's or after a character that's no letter or digit ("^zed" finds "Zed.Dead", "The_Zed", not "Facialized"). Both
* use the trigram index.
*/
function wordMatch(word: string): { sql: string; binding: string } {
if (word.startsWith('^') && word.length > 1) {
const rest = word.slice(1).replace(/[\\^$.|?*+()[\]{}]/g, (c) => `\\${c}`)
return { sql: 't.text ~* ?', binding: `(^|[^[:alnum:]])${rest}` }
}
return { sql: 't.text ILIKE ?', binding: contains(word) }
}
/** Searching and browsing the catalog, for the web interface and Newznab. */
class SearchService {
async search(params: SearchParams) {
@@ -105,34 +89,27 @@ class SearchService {
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
const bindings: unknown[] = []
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike;
// "^zed" only at a word's start.
// Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
const match = wordMatch(word)
where.push(`r.id IN (SELECT t.release_id FROM release_texts t WHERE ${match.sql})`)
bindings.push(match.binding)
where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
bindings.push(contains(word), contains(word))
}
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
if (params.complete) where.push('r.complete')
// Incomplete only when final: a provisional tail release may yet get its missing files.
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
if (params.source) {
const code = (SOURCES as Record<string, number>)[params.source]
if (code === undefined) where.push('false')
else {
where.push('r.source = ?')
bindings.push(code)
}
where.push('r.source = ?')
bindings.push(params.source)
}
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
let posters: number[] | null = null
let posters: string[] | null = null
if (params.poster) {
const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
posters = matched.map((m: { id: string }) => Number(m.id))
const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
posters = matched.map((m: { poster: string }) => m.poster)
}
if (params.categories?.length) {
where.push('r.category = ANY(?)')
@@ -164,14 +141,14 @@ class SearchService {
const whereSql = where.join(' AND ')
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
// Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
// in the order asked; then merged.
// Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
// bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
const source =
posters === null
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
: {
from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
where: 'true',
bindings: [posters] as unknown[],
}
@@ -179,17 +156,15 @@ class SearchService {
posters === null
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
: db.rawQuery(
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
[posters, ...bindings]
)
const [{ rows }, { rows: counted }] = await Promise.all([
db.rawQuery(
`SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
r.settled_at IS NOT NULL AS final
FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
[...source.bindings, ...bindings, perPage, offset]
),
counting,
@@ -204,9 +179,9 @@ class SearchService {
releases: (rows as any[]).map((r): ReleaseSummary => ({
id: Number(r.id),
name: r.name,
source: sourceName(r.source),
poster: r.poster ?? null,
groups: r.groups ?? [],
source: r.source,
poster: r.poster,
groups: r.groups,
postedAt: r.posted_at,
fileTotal: r.file_total,
fileCount: Number(r.files),
@@ -217,7 +192,6 @@ class SearchService {
category: r.category,
settled: r.settled_at !== null,
passworded: r.passworded,
passwordKnown: r.password_known,
innerName: r.inner_name,
})),
}
@@ -225,24 +199,19 @@ class SearchService {
/** A release with its files, read from its chunks. */
async release(id: number) {
const {
rows: [r],
} = await db.rawQuery(
`SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
[id]
)
const r = await db.from('releases').where('id', id).first()
if (!r) return null
const [files, { rows: inspections }, passwords] = await Promise.all([
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
NzbService.files(id),
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
NzbService.passwords(id),
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
])
return {
id: Number(r.id),
name: r.name,
source: sourceName(r.source),
poster: r.poster ?? null,
groups: r.groups ?? [],
source: r.source,
poster: r.poster,
groups: r.groups,
postedAt: r.posted_at,
fileTotal: r.file_total,
fileCount: Number(r.files),
@@ -250,14 +219,13 @@ class SearchService {
segments: Number(r.segments),
readable: r.readable,
complete: r.complete,
final: r.settled_at !== null,
final: finality[0].final,
hasPar2: r.has_par2,
category: r.category,
firstSeenAt: r.first_seen_at,
lastSeenAt: r.last_seen_at,
settledAt: r.settled_at,
passworded: r.passworded,
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
passwords,
innerName: r.inner_name,
inspectedAt: r.inspected_at,
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
@@ -319,9 +287,9 @@ class SearchService {
// History by articles; the pass: the history ranges before those deferred behind them (priority
// 2000 apart; e.g. groups added later that go first).
const { rows: ranges } = await db.rawQuery(
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
app/services/settler.ts
+36 -25
View File
@@ -1,35 +1,46 @@
import db from '@adonisjs/lucid/services/db'
/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
export const SETTLE_COMPLETE_HOURS = 6
/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
const QUIET_HOURS = 6
/**
* History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
* neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
* Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
* each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
*/
const QUIET_HISTORY_MINUTES = 30
const BATCH = 50_000
/**
* Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
* lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
* never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
* hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
* announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
* settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
* candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
* Settles releases (REBUILD.md, part 8): once nothing has been added for
* QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
* [i/N] counter announced was found, unknown without a counter (phase 2
* reads par2 sets for those). Phase 2 picks its work from settled releases.
*/
class Settler {
async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
return db.transaction(async (trx) => {
const { rows } = await trx.rawQuery(
`UPDATE releases SET settled_at = now()
WHERE settled_at IS NULL AND complete
AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
RETURNING id`,
[quietHours, quietHours]
/** Up to `batches` batches of BATCH (all there are without). */
async run(batches = Infinity): Promise<number> {
let settled = 0
for (let batch = 0; batch < batches; batch++) {
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
const { rows } = await db.rawQuery(
`WITH settled AS (
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
RETURNING id, readable, files, bytes, poster),
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
-- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
ON CONFLICT DO NOTHING)
SELECT count(*)::int AS n FROM settled`,
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
)
if (rows.length > 0)
await trx.rawQuery(
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
[rows.map((r: { id: string }) => Number(r.id))]
)
return rows.length
})
const n = rows[0].n
settled += n
if (n < BATCH) return settled
}
return settled
}
}
app/services/seven_zip_tails.ts
-22
View File
@@ -1,22 +0,0 @@
/**
* 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
* segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
* its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
* it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
* .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
* waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
* a piece that saw only part of it queued a segment that isn't its last.
*/
export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
if (releaseIds.length === 0) return 0
const result = await trx.rawQuery(
`DELETE FROM probe_candidates pc USING (
SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
) s
WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
[releaseIds.map(String)]
)
return result.rowCount ?? 0
}
app/services/sidecar_folder.ts
+12 -10
View File
@@ -6,7 +6,7 @@ const BATCH = 2000
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
const WINDOW_MINUTES = 10
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
/** The words of a name worth comparing: 4+ letters or digits. */
function words(text: string): Set<string> {
@@ -53,27 +53,29 @@ export class SidecarFolder {
async round(): Promise<FoldRound> {
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
// A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
// queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
const { rows: queued } = await db.rawQuery(
`SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
`SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
[BATCH]
)
const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
const { rows: stale } = await db.rawQuery(
`SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
[BATCH]
)
const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
for (const c of queued) {
if (c.id === null) continue
round.looked++
done.push(Number(c.id))
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
round.sidecars++
if (!c.posted_at) continue
const { rows: siblings } = await db.rawQuery(
`SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
`SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
)
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
if (!sibling) continue
app/services/spot_inspector.ts
+8 -30
View File
@@ -9,9 +9,6 @@ import { decodeSpotBinary } from '#lib/spotnet/index'
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
import { parseSubject } from '#lib/headers/index'
import { releaseIdentity } from '#lib/release/index'
import { identityKey, releaseKey } from '#lib/release/keys'
import { SOURCES } from '#lib/release/source'
import CatalogRefs from '#services/catalog_refs'
/** Spots claimed per round, newest first. */
const BATCH = 20
@@ -105,12 +102,6 @@ export class SpotInspector {
this.client = undefined
}
if (!round || round.inspected === 0) {
// Nothing to inspect for a while: closed (nntpIdleCloseMs), opened again with work.
const idleMs = coordinatorConfig.nntpIdleCloseMs
if (idleMs > 0 && IDLE_MS >= idleMs) {
this.client?.close()
this.client = undefined
}
for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
}
}
@@ -254,8 +245,8 @@ export class SpotInspector {
}
if (wanted.length === 0) return 0
const { rows } = await db.rawQuery(
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
`SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
[[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
)
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
for (const r of rows) {
@@ -278,7 +269,7 @@ export class SpotInspector {
await db.transaction(async (trx) => {
// A release of the spot's own goes now that the header pass built its files' releases.
if (spot.release_id) {
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
if (own) {
await trx.from('release_spots').where('release_id', own.id).delete()
await trx.from('release_passwords').where('release_id', own.id).delete()
@@ -297,12 +288,6 @@ export class SpotInspector {
.ignore()
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
}
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
spot.category,
JSON.stringify(spot.subcats ?? {}),
[...perRelease.keys()],
])
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
@@ -338,19 +323,16 @@ export class SpotInspector {
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
const groupSetId = await CatalogRefs.groupSetId(db, groups)
const [row] = await db
.table('releases')
.insert({
release_key: releaseKey(key, groupId, null),
identity_key: identityKey(key),
release_key: `spot:${key}`,
identity_key: key,
group_id: groupId,
group_set_id: groupSetId,
poster_id: posterId,
name: spot.title.slice(0, 1000),
source: SOURCES.spot,
source: 'spot',
poster: spot.poster,
groups: JSON.stringify(groups),
posted_at: spot.posted_at,
last_posted_at: spot.posted_at,
files: files.length,
@@ -359,8 +341,6 @@ export class SpotInspector {
readable: true,
passworded: spot.password ? true : null,
settled_at: new Date(),
// Kept when its title says nothing (the category trigger, migration 5200).
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
})
.onConflict('release_key')
.ignore()
@@ -369,8 +349,6 @@ export class SpotInspector {
const releaseId = Number(row.id)
await db.transaction(async (trx) => {
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
// Its title, searchable (release_texts).
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
if (spot.password) {
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
}
app/services/spotter.ts
+1 -8
View File
@@ -1,6 +1,5 @@
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import { SOURCES } from '#lib/release/source'
import env from '#start/env'
import coordinatorConfig from '#config/coordinator'
import { MissingArticle, XoverClient } from '#lib/nntp/index'
@@ -61,12 +60,6 @@ export class Spotter {
this.client = undefined
}
const wait = coordinatorConfig.spots.everySeconds * 1000
// Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
const idleMs = coordinatorConfig.nntpIdleCloseMs
if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
this.client?.close()
this.client = undefined
}
for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
}
}
@@ -184,7 +177,7 @@ export class Spotter {
await trx.from('release_spots').where('spot_id', targetId).delete()
await trx.from('release_passwords').where('spot_id', targetId).delete()
// A release the spot made of its own goes with it.
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete()
})
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
}
app/services/status.ts
-343
View File
@@ -1,343 +0,0 @@
import db from '@adonisjs/lucid/services/db'
import { readdir, stat, statfs } from 'node:fs/promises'
import { join } from 'node:path'
import coordinatorConfig from '#config/coordinator'
import GroupLeases from '#services/group_leases'
import type { Lane } from '#lib/edge/index'
export interface RangeBand {
band: string
planned: number
plannedArticles: number
leased: number
done: number
doneArticles: number
failed: number
}
export interface HeldGroup {
node: string
group: string
lane: Lane
range: { start: number; end: number; next: number } | null
heartbeatSeconds: number
snapshotNext: number | null
snapshotSeconds: number | null
snapshotMb: number | null
rangesDone: number
headers: number
}
export interface SnapshotFile {
group: string
lane: Lane
mb: number
hours: number
}
export interface Status {
at: string
catalog: {
tablespace: string
gb: number
capGb: number
full: boolean
freeGb: number
}
bands: RangeBand[]
lastHour: {
indexedArticles: number
doneRanges: number
storedArticles: number
storedRanges: number
}
corpus: { pendingRanges: number; pendingArticles: number }
spool: { ranges: number; chunks: number }
tail: { groups: number; ranges: number; articles: number }
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
ancient: { groups: number; articles: number }
held: HeldGroup[]
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
releases: { estimated: number; provisional: number }
/** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
judge: { dirty: number }
nodes: { name: string; lastSeenSeconds: number | null }[]
}
const seconds = (since: Date | string | null) =>
since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
/**
* What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
* NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
* with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
* filesystem; nothing is kept between calls.
*/
class StatusService {
async collect(): Promise<Status> {
const tail = coordinatorConfig.tail.priority
const cap = coordinatorConfig.catalogMaxNvmeGb
// The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
// chunk partitions and the probe cache on the HDD, minutes on homelab.
const { rows: space } = await db.rawQuery(
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
)
const tablespaceGb = Number(space[0].bytes) / 1e9
const fs = await statfs('/').catch(() => null)
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
// The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
// capture too, at their history priorities), the history by its thousand.
const { rows: bands } = await db.rawQuery(
`SELECT CASE WHEN priority <= ? THEN 'tail'
WHEN priority < 0 THEN 'raised'
WHEN capture AND priority <= 2 THEN 'window ' || priority
WHEN priority < 1000 THEN 'history 0-999'
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
count(*) FILTER (WHERE status = 'planned') AS planned,
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
count(*) FILTER (WHERE status = 'leased') AS leased,
count(*) FILTER (WHERE status = 'done') AS done,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
count(*) FILTER (WHERE status = 'failed') AS failed
FROM ranges GROUP BY 1
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
[tail, tail]
)
const { rows: hour } = await db.rawQuery(
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
FROM ranges`,
[tail, tail, tail]
)
const h = hour[0]
const ttl = coordinatorConfig.leaseTtlSeconds
const { rows: held } = await db.rawQuery(
`SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
r.start_id, r.end_id, r.next_id, l.group_id
FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
LEFT JOIN ranges r ON r.id = l.range_id
WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
ORDER BY l.lane, g.name`,
[ttl]
)
const snapshotSizes = new Map<string, number>()
const files: SnapshotFile[] = []
const names = new Map<number, string>(
(await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
)
for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
const m = /^(\d+)(-tail)?\.gz$/.exec(file)
if (!m) continue
const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
if (!s) continue
const lane: Lane = m[2] ? 'tail' : 'history'
snapshotSizes.set(`${m[1]}:${lane}`, s.size)
files.push({
group: names.get(Number(m[1])) ?? m[1],
lane,
mb: s.size / 1e6,
hours: (Date.now() - s.mtimeMs) / 3600_000,
})
}
files.sort((a, b) => b.mb - a.mb)
// The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
// homelab); the rest are small partial indexes.
const { rows: releases } = await db.rawQuery(
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
(SELECT count(*) FROM judge_dirty) AS dirty`
)
const r = releases[0]
const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
return {
at: new Date().toISOString(),
catalog: {
tablespace: space[0].tablespace,
gb: tablespaceGb,
capGb: cap,
full: cap > 0 && tablespaceGb > cap,
freeGb,
},
bands: bands.map((b: any) => ({
band: b.band,
planned: Number(b.planned),
plannedArticles: Number(b.planned_articles),
leased: Number(b.leased),
done: Number(b.done),
doneArticles: Number(b.done_articles),
failed: Number(b.failed),
})),
lastHour: {
indexedArticles: Number(h.indexed),
doneRanges: Number(h.done_ranges),
storedArticles: Number(h.stored),
storedRanges: Number(h.stored_ranges),
},
corpus: {
pendingRanges: Number(h.corpus_pending),
pendingArticles: Number(h.corpus_pending_articles),
},
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
tail: {
groups: Number(h.tail_groups),
ranges: Number(h.tail_ranges),
articles: Number(h.tail_articles),
},
held: held.map((l: any) => {
const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
return {
node: l.node,
group: l.group,
lane: l.lane,
range:
l.start_id !== null
? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
: null,
heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
snapshotSeconds: seconds(l.snapshot_at),
snapshotMb: size === undefined ? null : size / 1e6,
rangesDone: Number(l.ranges_done),
headers: Number(l.headers),
}
}),
snapshots: {
count: files.length,
gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
largest: files.slice(0, 5),
},
releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
probes: {
ready: Number(r.ready),
waiting: Number(r.waiting),
pendingWaiting: Number(r.pending_waiting),
pendingNamed: Number(r.pending_named),
},
judge: { dirty: Number(r.dirty) },
nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
}
}
/** The status as a text panel. */
render(s: Status): string {
const n = (v: number) => Math.round(v).toLocaleString('en-US')
const b = (v: number) =>
v >= 1e9
? `${(v / 1e9).toFixed(2)} B`
: v >= 1e6
? `${(v / 1e6).toFixed(1)} M`
: v >= 1e3
? `${(v / 1e3).toFixed(0)} k`
: String(v)
const age = (sec: number | null) =>
sec === null
? '-'
: sec < 90
? `${sec} s`
: sec < 5400
? `${Math.round(sec / 60)} min`
: `${(sec / 3600).toFixed(1)} h`
const lines: string[] = []
lines.push(`Status at ${s.at}`)
lines.push('')
lines.push(
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
)
lines.push(
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
)
lines.push(
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
)
lines.push('')
const table = (header: string[], rows: string[][]) => {
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
const line = (r: string[]) =>
r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(' ')
lines.push(line(header))
for (const r of rows) lines.push(line(r))
}
lines.push('Ranges by band:')
table(
['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
s.bands.map((x) => [
x.band,
n(x.planned),
b(x.plannedArticles),
n(x.leased),
n(x.done),
b(x.doneArticles),
n(x.failed),
])
)
lines.push('')
lines.push(`Held groups (${s.held.length}):`)
if (s.held.length > 0)
table(
[
'group',
'lane',
'node',
'range',
'at',
'heartbeat',
'snapshot',
'size',
'done',
'headers',
],
s.held.map((x) => [
x.group,
x.lane,
x.node,
x.range ? `${x.range.start}-${x.range.end}` : '-',
x.range ? String(x.range.next) : '-',
age(x.heartbeatSeconds),
x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
n(x.rangesDone),
b(x.headers),
])
)
lines.push('')
lines.push(
`Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
)
lines.push(
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
)
lines.push(
`Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
)
lines.push(
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
)
return lines.join('\n')
}
}
export default new StatusService()
app/validators/user.ts
+2
View File
@@ -14,6 +14,8 @@ export const signupValidator = vine.create({
email: email().unique({ table: 'users', column: 'email' }),
password: password(),
passwordConfirmation: password().sameAs('password'),
// Needed when SIGNUP is `invite` (config/access.ts).
invite: vine.string().trim().maxLength(32).optional(),
})
/**
commands/access_invite.ts
+37
View File
@@ -0,0 +1,37 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import AccessService from '#services/access_service'
import { TIERS } from '#config/access'
/** Makes invite codes (SIGNUP=invite), optionally with a tier for the new account, e.g. a sold VIP year. */
export default class AccessInvite extends BaseCommand {
static commandName = 'access:invite'
static description = 'Makes invite codes, optionally giving a tier (e.g. --tier=vip --days=365)'
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'How many codes', default: 1 })
declare count: number
@flags.string({ description: `Tier the new account gets (${Object.keys(TIERS).join(', ')})` })
declare tier?: string
@flags.number({ description: 'For how many days the tier holds (without: no end)' })
declare days?: number
@flags.string({ description: 'A note, e.g. who it is for or the payment' })
declare note?: string
async run() {
if (this.tier && !TIERS[this.tier]) {
this.logger.error(`No such tier: ${this.tier}`)
this.exitCode = 1
return
}
const codes = Array.from({ length: this.count }, () => AccessService.newKey())
await db.table('invites').multiInsert(
codes.map((code) => ({ code, tier: this.tier ?? null, tier_days: this.tier ? (this.days ?? null) : null, note: this.note ?? null }))
)
for (const code of codes) this.logger.log(code)
}
}
commands/access_suspend.ts
+28
View File
@@ -0,0 +1,28 @@
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import User from '#models/user'
/** Suspends a user (Newznab error 101, the catalog 403), e.g. for a shared key; --undo lifts it. */
export default class AccessSuspend extends BaseCommand {
static commandName = 'access:suspend'
static description = 'Suspends a user (or lifts it with --undo)'
static options: CommandOptions = { startApp: true }
@args.string({ description: 'The user’s email' })
declare email: string
@flags.boolean({ description: 'Lift the suspension' })
declare undo: boolean
async run() {
const user = await User.findBy('email', this.email)
if (!user) {
this.logger.error(`No user ${this.email}`)
this.exitCode = 1
return
}
user.suspended = !this.undo
await user.save()
this.logger.info(`${user.email}: ${user.suspended ? 'suspended' : 'active'}`)
}
}
commands/access_tier.ts
+32
View File
@@ -0,0 +1,32 @@
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import User from '#models/user'
import AccessService from '#services/access_service'
/** Gives a user a tier (a payment, by hand until one is taken on the site); a running one is extended. */
export default class AccessTier extends BaseCommand {
static commandName = 'access:tier'
static description = "Sets a user's tier, e.g. access:tier someone@example.com vip --days=365"
static options: CommandOptions = { startApp: true }
@args.string({ description: 'The user’s email' })
declare email: string
@args.string({ description: 'Tier (config/access.ts)' })
declare tier: string
@flags.number({ description: 'For how many days (without: no end); the same tier still running is extended' })
declare days?: number
async run() {
const user = await User.findBy('email', this.email)
if (!user) {
this.logger.error(`No user ${this.email}`)
this.exitCode = 1
return
}
AccessService.grantTier(user, this.tier, this.days ?? null)
await user.save()
this.logger.info(`${user.email}: ${user.tier} ${user.tierUntil ? `until ${user.tierUntil.toISO()}` : 'without end'}`)
}
}
commands/coordinator_run.ts
+57 -72
View File
@@ -3,46 +3,45 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
import { setTimeout as sleep } from 'node:timers/promises'
import { Prefetcher } from '#services/prefetcher'
import Planner from '#services/planner'
import Leases from '#services/leases'
import FragmentMerger from '#services/fragment_merger'
import Settler from '#services/settler'
import RangeBlocks from '#services/range_blocks'
import NntpPool from '#services/nntp_pool'
import { NntpShare } from '#services/nntp_shares'
import coordinatorConfig from '#config/coordinator'
const TICK_MS = 2000
const PLAN_EVERY_MS = 60 * 60 * 1000
const MERGE_EVERY_MS = 60 * 1000
const MERGE_BUDGET_MS = 30 * 1000
const STOP_WAIT_MS = 15 * 1000
/** Leases whose worker went quiet after reading everything are finished every few minutes. */
const ABANDONED_EVERY_MS = 5 * 60 * 1000
/**
* Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
* rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
* (LOG.md 2026-10-16 (67)).
*/
const SETTLE_EVERY_MS = 5 * 60 * 1000
const SETTLE_BATCHES = 4
/** range_blocks (release_final) after ranges finish. */
const BLOCKS_EVERY_MS = 5 * 60 * 1000
/**
* The coordinator's background work (REBUILD.md, part 1): the prefetcher, the tail's ranges every few minutes,
* new groups' history every hour, and abandoned leases. The API (leases, stream, batches) runs in the web server.
* Voyager 3: the settler and the fragment merger are gone (the window finishes a release once; the block merger
* joins edge releases).
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
* merging fragments of finished ranges every minute, and every hour
* planning ranges for new articles and settling quiet releases. The API (leases, stream,
* batches) runs in the web server.
*/
export default class CoordinatorRun extends BaseCommand {
static commandName = 'coordinator:run'
static description = 'Prefetches ranges into the spool and plans new ones'
static options: CommandOptions = { startApp: true, staysAlive: true }
@flags.string({
description:
'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles',
})
@flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
declare shard?: string
async run() {
const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
if (!(count >= 1 && index >= 1 && index <= count))
throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
const prefetcher = new Prefetcher({ index: index - 1, count })
const share = new NntpShare(
count > 1 ? `coordinator ${index}/${count}` : 'coordinator',
coordinatorConfig.nntpShares.coordinatorPriority,
coordinatorConfig.nntpMinConnections,
coordinatorConfig.nntpConnections
)
const main = index === 1
let stopping = false
let stopped: () => void
@@ -54,33 +53,9 @@ export default class CoordinatorRun extends BaseCommand {
prefetcher.stop()
await done
})
this.logger.info(
count > 1
? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).`
: 'Coordinator started.'
)
// The pool's size: its share of the account's connections, the prober the rest (app/services/nntp_shares.ts).
// Before the first connection: until then it's NNTP_CONNECTIONS.
const first = await NntpPool.share(share)
this.logger.info(`NNTP share: ${first.granted} connections.`)
const sharing = (async () => {
while (!stopping) {
try {
const { was, granted } = await NntpPool.share(share)
if (granted !== was) this.logger.info(`NNTP share: ${granted} connections (was ${was}).`)
} catch (error: any) {
this.logger.error(`NNTP share failed: ${error.message}`)
}
for (
let waited = 0;
waited < coordinatorConfig.nntpShares.everyMs && !stopping;
waited += TICK_MS
)
await sleep(TICK_MS)
}
})()
// The prefetcher's ticks in a loop of their own: planning doesn't hold them (a merge round of up to 30 s once
// left the connections idle once their targets were fetched).
this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
// Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
// of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
const prefetching = (async () => {
while (!stopping) {
try {
@@ -91,31 +66,22 @@ export default class CoordinatorRun extends BaseCommand {
await sleep(TICK_MS)
}
})()
const abandoned = (async () => {
// range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
const blocks = (async () => {
while (main && !stopping) {
try {
const finished = await Leases.finishAbandoned()
if (finished > 0)
this.logger.info(
`Finished ${finished} ranges read to their end whose worker went quiet.`
)
await RangeBlocks.rebuild()
} catch (error: any) {
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
this.logger.error(`Range blocks failed: ${error.message}`)
}
try {
const settled = await Settler.settleComplete()
if (settled > 0)
this.logger.info(`Settled ${settled} complete provisional releases quiet for hours.`)
} catch (error: any) {
this.logger.error(`Settling complete releases failed: ${error.message}`)
}
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
await sleep(TICK_MS)
for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
}
})()
// History for new groups hourly; the tail every few minutes.
let plannedAt = 0
let tailedAt = 0
let mergedAt = 0
let settledAt = 0
while (!stopping) {
try {
if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
@@ -123,27 +89,46 @@ export default class CoordinatorRun extends BaseCommand {
const { groups, ranges } = await Planner.planAll()
if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
}
if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
settledAt = Date.now()
const settled = await Settler.run(SETTLE_BATCHES)
if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
}
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
tailedAt = Date.now()
const { groups, ranges, extended } = await Planner.planTail()
if (ranges + extended > 0)
this.logger.info(
`Tail: ${ranges} ranges added, ${extended} grown, for ${groups} groups.`
)
const { groups, ranges } = await Planner.planTail()
if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
}
if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
mergedAt = Date.now()
// Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
// (Going on while only waiting files were left kept Postgres busy all the time.)
const totals = { completed: 0, incomplete: 0, waiting: 0 }
let decidedThisPass = 0
while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
const merged = await FragmentMerger.run()
totals.completed += merged.completed
totals.incomplete += merged.incomplete
totals.waiting = merged.waiting
decidedThisPass += merged.completed + merged.incomplete
if (merged.wrapped) {
if (decidedThisPass === 0) break
decidedThisPass = 0
}
}
if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
}
} catch (error: any) {
this.logger.error(`Coordinator round failed: ${error.message}`)
}
await sleep(TICK_MS)
}
await Promise.all([prefetching, abandoned, sharing])
await Promise.all([prefetching, blocks])
// The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
const waitUntil = Date.now() + STOP_WAIT_MS
while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
if (prefetcher.active > 0)
this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
await NntpPool.close()
await share.leave().catch(() => {})
this.logger.info('Coordinator stopped.')
stopped!()
await this.terminate()
commands/coordinator_status.ts
-22
View File
@@ -1,22 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import StatusService from '#services/status'
/**
* What the indexer is doing (app/services/status.ts): the catalog's room, the ranges by band, the last hour's
* rates, the held groups and their snapshots, the phase-2 backlogs. Reads only.
*/
export default class CoordinatorStatus extends BaseCommand {
static commandName = 'coordinator:status'
static description =
'Prints what the indexer is doing: catalog room, ranges by band, rates, held groups, backlogs'
static options: CommandOptions = { startApp: true }
@flags.boolean({ description: 'JSON instead of the text panel' })
declare json: boolean
async run() {
const status = await StatusService.collect()
this.logger.log(this.json ? JSON.stringify(status, null, 2) : StatusService.render(status))
}
}
commands/corpus_filter.ts
-54
View File
@@ -1,54 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { access, readFile, readdir, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import db from '@adonisjs/lucid/services/db'
import Corpus, { FILTERED } from '#services/corpus'
import RuleSets from '#services/rule_sets'
import coordinatorConfig from '#config/coordinator'
/**
* Filters corpus chunks stored as the news server sent them (before CORPUS_FILTER, 2026-10-23): each chunk without a
* `.drops` of the planned capture ranges (--all: of every range in the corpus) written again as Corpus.write would.
*/
export default class CorpusFilterCommand extends BaseCommand {
static commandName = 'corpus:filter'
static description = 'Filters stored corpus chunks (planned capture ranges; --all: every range) as CORPUS_FILTER does'
static options: CommandOptions = { startApp: true }
@flags.boolean({ description: 'Every range in the corpus, also done ones (the grouping sweeps read them raw)' })
declare all: boolean
async run() {
const dir = coordinatorConfig.corpus.dir
if (!dir) return this.logger.error('CORPUS_DIR is not set')
const rules = await RuleSets.current()
const filter = { ruleVersion: rules.version, priorities: rules.priorities }
const query = db.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').where('r.capture', true).select('g.name as group', 'r.start_id', 'r.end_id')
const ranges = await (this.all ? query : query.where('r.status', 'planned'))
let chunks = 0
let before = 0
let after = 0
for (const r of ranges) {
const range = { group: r.group, start: Number(r.start_id), end: Number(r.end_id) }
const rangeDir = join(dir, range.group, `${range.start}-${range.end}`)
const files = await readdir(rangeDir).catch(() => [] as string[])
const gz = files.filter((f) => f.endsWith('.gz'))
for (const name of gz) {
if (files.includes(`${name}.drops`)) continue
const m = /^\d{4}-(\d+)-(\d+)\.gz$/.exec(name)
if (!m) continue
const data = await readFile(join(rangeDir, name))
await Corpus.write(range, { from: Number(m[1]), to: Number(m[2]) }, data, Number.POSITIVE_INFINITY, filter)
before += data.length
after += (await readFile(join(rangeDir, name))).length
chunks++
}
// A complete range's range.json says it's filtered now (Corpus.write only writes it at the last chunk).
const json = join(rangeDir, 'range.json')
if (gz.length > 0 && (await access(json).then(() => true, () => false))) await writeFile(json, JSON.stringify({ ...range, filtered: FILTERED }))
if (chunks > 0 && chunks % 1000 < gz.length) this.logger.info(`${chunks} chunks filtered: ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
}
this.logger.info(`Done: ${chunks} chunks of ${ranges.length} ranges filtered, ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
}
}
commands/edge_bench.ts
+3 -2
View File
@@ -108,7 +108,7 @@ export default class EdgeBench extends BaseCommand {
let text = 0
let peakRss = 0
let peakDetail = ''
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, samples: 0 }
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
const counts = new Map<string, number>()
const keys = new Set<string>()
let batches = 0
@@ -164,6 +164,7 @@ export default class EdgeBench extends BaseCommand {
out.payload += r.payload.length
}
out.candidates += batch.candidates.length
out.fragments += batch.fragments.length
out.samples += batch.samples.length
for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
const m = process.memoryUsage()
@@ -181,7 +182,7 @@ export default class EdgeBench extends BaseCommand {
this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
this.logger.info(` inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
this.logger.info(` ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.samples} samples`)
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(` dropped ${k}: ${v.toLocaleString('en')} articles`)
if (this.store) this.logger.info(` stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
}
commands/nntp_probe.ts
-117
View File
@@ -1,117 +0,0 @@
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import env from '#start/env'
import { XoverClient } from '#lib/nntp/index'
/**
* What the news server does with XOVER spans around an article that made a request hang (ranges:dates: one point
* per group timed out after 2 minutes, again on a fresh connection; LOG.md 2026-10-25 (105)). Each probe on a
* connection of its own with a short timeout: the span asked for, the reply, how many lines, how long. First the
* span itself and wider ones around it, then halves of it down to single articles, then a span past `high`.
*/
export default class NntpProbe extends BaseCommand {
static commandName = 'nntp:probe'
static description =
'Probes XOVER spans around an article that hangs, bisecting to the articles at fault'
static options: CommandOptions = { startApp: true }
@args.string({ description: 'Group' })
declare group: string
@args.string({ description: 'First article of the span that hung' })
declare article: string
@flags.number({ description: 'Articles in the span (default 32)' })
declare size?: number
@flags.number({ description: 'Seconds without data before a probe is given up (default 20)' })
declare timeout?: number
@flags.boolean({ description: 'Only the span itself and the wider ones, no bisection' })
declare noBisect: boolean
@flags.boolean({ description: 'Without XFEATURE COMPRESS GZIP' })
declare plain: boolean
@flags.string({
description: 'Only spans from the article of these sizes, e.g. "8,16,24,32,48,64"',
})
declare sizes?: string
private timeoutMs = 20_000
private info = { count: 0, low: 0, high: 0 }
async run() {
this.timeoutMs = (this.timeout ?? 20) * 1000
const a = Number(this.article)
const size = this.size ?? 32
const client = await this.connect()
this.info = await client.group(this.group)
client.close()
this.logger.info(
`${this.group}: ${this.info.count} articles, ${this.info.low}–${this.info.high}${this.plain ? '; plain' : '; compressed'}.`
)
const span = (from: number, to: number) =>
[Math.max(this.info.low, from), Math.min(this.info.high, to)] as const
if (this.sizes) {
for (const n of this.sizes.split(',').map(Number)) await this.probe(...span(a, a + n - 1))
return
}
this.logger.info('The span and wider ones:')
await this.probe(...span(a, a + size - 1))
await this.probe(...span(a, a + size * 32 - 1))
await this.probe(...span(a - size * 32, a + size - 1))
await this.probe(...span(a - 50_000, a + 49_999))
this.logger.info('Past the end, and an empty span before it:')
await this.probe(this.info.high + 100, this.info.high + 131)
await this.probe(this.info.high + 1, this.info.high + 1)
if (this.noBisect) return
this.logger.info('Halves of the span:')
await this.bisect(a, Math.min(this.info.high, a + size - 1))
}
/** Halves of a span that hangs, down to single articles; a half that answers isn't split further. */
private async bisect(from: number, to: number) {
const hung = await this.probe(from, to)
if (!hung || from === to) return
const mid = Math.floor((from + to) / 2)
await this.bisect(from, mid)
await this.bisect(mid + 1, to)
}
/** One XOVER on a fresh connection; true when it timed out. */
private async probe(from: number, to: number): Promise<boolean> {
const startedAt = Date.now()
let client: XoverClient | null = null
try {
client = await this.connect()
await client.group(this.group)
const rows = await client.xover(from, to)
const t = client.lastTiming
this.logger.info(
` XOVER ${from}-${to}: ${rows.length} lines in ${Date.now() - startedAt} ms (reply ${t.replyMs.toFixed(0)}, body ${t.bodyMs.toFixed(0)})${rows.length > 0 ? `; first ${rows[0][0]}, last ${rows[rows.length - 1][0]}` : ''}`
)
return false
} catch (error: any) {
const timedOut = /timed out/i.test(error.message)
this.logger.warning(
` XOVER ${from}-${to}: ${error.message} after ${Date.now() - startedAt} ms${client ? `, ${client.buffered} bytes buffered: ${JSON.stringify(client.bufferedHead().toString('latin1'))} (${client.bufferedHead().toString('hex')})` : ''}`
)
return timedOut
} finally {
client?.close()
}
}
private connect() {
return XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: !this.plain,
timeoutMs: this.timeoutMs,
})
}
}
commands/par2_prune.ts
-17
View File
@@ -1,17 +0,0 @@
import { BaseCommand } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { Par2Pruner } from '#services/par2_pruner'
/** The par2 prune (Par2Pruner) in one go: a whole pass from the first set, instead of the merger's 2 minutes an hour. */
export default class Par2Prune extends BaseCommand {
static commandName = 'par2:prune'
static description = 'Drops the par2 file lists the merge can no longer use: one whole pass over every set, then exits'
static options: CommandOptions = { startApp: true }
async run() {
await db.from('worker_cursors').where('name', 'par2-prune').update({ key: '' })
const round = await new Par2Pruner().round()
this.logger.info(`Pruned ${round.rows} par2 file rows: ${round.complete} complete sets, ${round.judged} judged, ${round.final} final.`)
}
}
commands/pending_classes.ts
-97
View File
@@ -1,97 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import coordinatorConfig from '#config/coordinator'
import { describeClass } from '#lib/release/index'
/**
* The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and
* `target_bytes` (the first target's bytes, when its candidates carry them) set in windows of created_at; the
* finished rows among them (named, exhausted, naming) seed each class's record, which gets its verdict (dropped) as
* the ladder gives it; then pending_classes' `waiting` counted afresh from the rows. --report shows the record.
*/
export default class PendingClasses extends BaseCommand {
static commandName = 'pending:classes'
static description =
'Classes the pending releases from before the classes and recounts pending_classes; --report shows the record'
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'Hours of created_at per window' })
declare hours: number
@flags.boolean({ description: 'Show the classes with the best and worst records, write nothing' })
declare report: boolean
async run() {
if (this.report) return this.showReport()
const hours = this.hours ?? 6
const { rows: span } = await db.rawQuery(
`SELECT min(created_at) AS lo, max(created_at) AS hi FROM pending_releases WHERE klass = 0`
)
if (!span[0].lo) {
this.logger.info('No rows without a class.')
} else {
let from = new Date(span[0].lo)
const hi = new Date(span[0].hi)
let classed = 0
const started = Date.now()
while (from <= hi) {
const to = new Date(from.getTime() + hours * 3600_000)
// Rows of the window classed; the finished ones among them (named, exhausted, naming) seed the record.
const { rows } = await db.rawQuery(
`WITH classed AS (
UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
WHERE klass = 0 AND created_at >= ? AND created_at < ?
RETURNING tool, group_id, klass, status),
seeded AS (
INSERT INTO pending_classes (tool, group_id, klass, probed, named)
SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed WHERE status IN (1, 3, 4) GROUP BY 1, 2, 3
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named
RETURNING 1)
SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM seeded) AS seeded`,
[from.toISOString(), to.toISOString()]
)
classed += Number(rows[0].classed)
from = to
this.logger.info(
`Up to ${to.toISOString().slice(0, 16)}: ${classed} rows classed (${Math.round((Date.now() - started) / 1000)} s).`
)
}
// The record's verdicts on the seeded classes.
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
await db.rawQuery(
`UPDATE pending_classes SET dropped = true WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`,
[pendingMinProbes, pendingMaxNamed]
)
}
// The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here).
await db.rawQuery(`
INSERT INTO pending_classes (tool, group_id, klass, waiting)
SELECT tool, group_id, klass, count(*) FROM pending_releases WHERE status = 0 AND finished GROUP BY 1, 2, 3
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = excluded.waiting`)
await db.rawQuery(
`UPDATE pending_classes c SET waiting = 0 WHERE waiting > 0 AND NOT EXISTS (
SELECT 1 FROM pending_releases p WHERE p.status = 0 AND p.finished AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass)`
)
const { rows: totals } = await db.rawQuery(
`SELECT count(*) AS classes, coalesce(sum(waiting), 0) AS waiting, count(*) FILTER (WHERE dropped) AS dropped FROM pending_classes`
)
this.logger.info(
`Done: ${totals[0].classes} classes, ${totals[0].waiting} waiting, ${totals[0].dropped} dropped.`
)
}
private async showReport() {
const { rows } = await db.rawQuery(
`SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named, c.waiting, c.dropped FROM pending_classes c JOIN groups g ON g.id = c.group_id
WHERE c.probed >= 20 ORDER BY c.named::float / c.probed DESC, c.probed DESC`
)
const line = (r: any) =>
` ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} probed ${String(r.probed).padStart(7)} named ${String(Math.round((100 * r.named) / r.probed)).padStart(3)}% waiting ${String(r.waiting).padStart(8)}${r.dropped ? ' dropped' : ''}`
this.logger.info(`${rows.length} classes with 20+ climbs; the best:`)
for (const r of rows.slice(0, 25)) this.logger.info(line(r))
this.logger.info('The worst:')
for (const r of rows.slice(-25)) this.logger.info(line(r))
}
}
commands/ranges_capture.ts
-80
View File
@@ -1,80 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
const BYTES_PER_ARTICLE = 18.5
/**
* Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
* corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
* ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
* fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
* missing) and reported.
*/
export default class RangesCapture extends BaseCommand {
static commandName = 'ranges:capture'
static description =
'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
static options: CommandOptions = { startApp: true }
@flags.string({ description: 'First day of the window (ISO date)', required: true })
declare from: string
@flags.string({ description: 'Day after the window (ISO date)', required: true })
declare to: string
@flags.number({
description: 'Priority for the planned ranges marked (default 0: before the history)',
})
declare priority?: number
@flags.boolean({ description: 'Only report what the window holds' })
declare dryRun: boolean
async run() {
const from = new Date(this.from)
const to = new Date(this.to)
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
return this.logger.error('--from and --to: ISO dates, from before to')
const priority = this.priority ?? 0
// A range covers est_date up to the next range's; the last range of a group up to now.
const hits = `
WITH spans AS (
SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
const { rows } = await db.rawQuery(
`${hits}
SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
FROM hit GROUP BY 1 ORDER BY 1`,
[to, from]
)
const fetch = rows
.filter((r: any) => r.kind === 'planned' || r.kind === 'done')
.reduce((sum: number, r: any) => sum + Number(r.articles), 0)
this.logger.info(`${this.from} to ${this.to}:`)
for (const r of rows)
this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
this.logger.info(
`To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
)
if (this.dryRun) return
const { rows: marked } = await db.rawQuery(
`${hits}
UPDATE ranges r SET capture = true,
priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
FROM hit WHERE r.id = hit.id AND NOT hit.capture
AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
RETURNING r.status`,
[to, from, priority]
)
const planned = marked.filter((r: any) => r.status === 'planned').length
this.logger.info(
`Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
)
}
}
commands/ranges_dates.ts
-216
View File
@@ -1,216 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import env from '#start/env'
import coordinatorConfig from '#config/coordinator'
import { XoverClient } from '#lib/nntp/index'
import { NntpShare } from '#services/nntp_shares'
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
/**
* When each active group's articles were posted (group_dates, sampled from the news server) and from that every
* range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
* posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
* articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
*/
export default class RangesDates extends BaseCommand {
static commandName = 'ranges:dates'
static description =
"Samples when each active group's articles were posted and estimates every range's date"
static options: CommandOptions = { startApp: true }
@flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
declare group?: string
@flags.number({ description: 'Articles between two samples of a group (default 500M)' })
declare every?: number
@flags.number({
description:
'Connections, a share of the account beside the coordinator and the prober (default 6)',
})
declare connections?: number
@flags.boolean({ description: 'Only groups without samples yet' })
declare missing: boolean
@flags.boolean({
description: 'No sampling: estimate the ranges from the samples stored already',
})
declare estimateOnly: boolean
async run() {
if (!this.estimateOnly) await this.sample()
await this.estimate()
await this.report()
}
private groups(missing = this.missing) {
const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
if (this.group) q.where('name', 'like', this.group)
if (missing)
q.whereNotExists((sub) =>
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
)
return q
}
private async sample() {
const every = this.every ?? 500_000_000
const connections = this.connections ?? 6
const groups = await this.groups()
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
// room for them), as many as asked for.
const workers = Math.max(1, Math.min(connections, groups.length))
const share = new NntpShare(
'dates',
coordinatorConfig.nntpShares.coordinatorPriority + 1,
workers,
workers
)
let open = 0
const sync = () =>
share
.sync(workers, open)
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
await sync()
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
this.logger.info(
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
)
const startedAt = Date.now()
let done = 0
let commands = 0
let samples = 0
const queue = [...groups]
const worker = async () => {
let client: XoverClient | null = null
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
if (!client) {
client = await XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: false,
timeoutMs: coordinatorConfig.nntpTimeoutMs,
})
open++
}
try {
return await work(client)
} catch (error) {
client.close()
client = null
open--
throw error
}
}
for (let g = queue.shift(); g; g = queue.shift()) {
try {
const result = await sampleGroup(use, g.name, { every, size: 32 })
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
await db.transaction(async (trx) => {
await trx.from('group_dates').where('group_id', g.id).delete()
if (result.samples.length > 0)
await trx.table('group_dates').multiInsert(
result.samples.map((s) => ({
group_id: g.id,
article: s.article,
posted_at: s.postedAt,
}))
)
})
commands += result.commands
samples += result.samples.length
if (result.samples.length < 2)
this.logger.warning(
`${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
)
} catch (error: any) {
this.logger.warning(`${g.name}: ${error.message}`)
}
done++
if (done % 50 === 0)
this.logger.info(
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
)
}
if (client) {
;(client as XoverClient).close()
open--
}
}
try {
await Promise.all(Array.from({ length: workers }, worker))
} finally {
clearInterval(sharing)
await share.sync(0, 0).catch(() => {})
}
this.logger.info(
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
)
}
/** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
private async estimate() {
const groups = await this.groups(false).whereExists((sub) =>
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
)
let updated = 0
for (const g of groups) {
const rows = await db
.from('group_dates')
.where('group_id', g.id)
.orderBy('article')
.select('article', 'posted_at')
const samples: DateSample[] = rows.map((r) => ({
article: Number(r.article),
postedAt: new Date(r.posted_at),
}))
const ranges = await db
.from('ranges')
.where('group_id', g.id)
.orderBy('start_id')
.select('id', 'start_id')
for (let i = 0; i < ranges.length; i += 1000) {
const batch = ranges.slice(i, i + 1000)
const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
const bindings = batch.flatMap((r) => [
Number(r.id),
estimateDate(samples, Number(r.start_id))!.toISOString(),
])
await db.rawQuery(
`UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
bindings
)
updated += batch.length
}
}
this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
}
/** The planned history by posting year: what a window would take. */
private async report() {
const { rows } = await db.rawQuery(`
SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
const undatedRows = await db.rawQuery(
`SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
)
const undated = undatedRows.rows[0].undated
this.logger.info(
`By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
)
for (const r of rows)
this.logger.info(
` ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
)
}
}
commands/ranges_merge_tail.ts
-23
View File
@@ -1,23 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import Planner from '#services/planner'
/**
* Merges adjacent planned tail ranges (Planner.mergeTail): the tail planner added a sliver a group every round
* while nothing leased the tail; now it grows the group's open range instead, and this packs what is there.
*/
export default class RangesMergeTail extends BaseCommand {
static commandName = 'ranges:merge-tail'
static description = 'Merges adjacent planned tail ranges into ranges of RANGE_ARTICLES'
static options: CommandOptions = { startApp: true }
@flags.boolean({ description: 'Only count what would be merged' })
declare dryRun: boolean
async run() {
const { candidates, merged, dropped } = await Planner.mergeTail(this.dryRun)
this.logger.info(
`${candidates} planned tail ranges: ${dropped} ${this.dryRun ? 'would merge' : 'merged'} into ${merged} grown ranges (${candidates - dropped} left).`
)
}
}
commands/ranges_plan.ts
+2 -7
View File
@@ -14,13 +14,8 @@ export default class RangesPlan extends BaseCommand {
async run() {
try {
if (this.tail) {
const { groups, ranges, extended } = await Planner.planTail()
this.logger.info(`Tail: ${ranges} ranges added, ${extended} grown, over ${groups} groups.`)
} else {
const { groups, ranges } = await Planner.planAll()
this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
}
const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
} finally {
await NntpPool.close()
}
commands/releases_judge.ts
+3 -3
View File
@@ -5,10 +5,10 @@ import { ReleaseJudge, RULES } from '#services/release_judge'
/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
export default class ReleasesJudge extends BaseCommand {
static commandName = 'releases:judge'
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable): the changed ones every few minutes, all daily'
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable), hourly'
static options: CommandOptions = { startApp: true, staysAlive: true }
@flags.boolean({ description: 'One full round, then exit' })
@flags.boolean({ description: 'One round, then exit' })
declare once: boolean
@flags.boolean({ description: 'With --once: count only, write no verdicts' })
@@ -17,7 +17,7 @@ export default class ReleasesJudge extends BaseCommand {
async run() {
const judge = new ReleaseJudge()
if (this.once) {
const round = await judge.full(this.dry)
const round = await judge.round(this.dry)
for (const rule of RULES) {
const r = round.rules[rule.name]
if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
commands/releases_merge_names.ts
-65
View File
@@ -1,65 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { Par2Merger } from '#services/par2_merger'
/**
* The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
* first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
*/
export default class ReleasesMergeNames extends BaseCommand {
static commandName = 'releases:merge-names'
static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'Sets to look at (newest first); all without' })
declare sets: number
@flags.boolean({ description: 'Count and show, merge nothing' })
declare dryRun: boolean
async run() {
const merger = new Par2Merger()
// Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
// so microseconds survive the round trip.
let before = ['9999-12-31', '']
let looked = 0
let merges = 0
let releases = 0
const examples: string[] = []
while (this.sets === undefined || looked < this.sets) {
const { rows } = await db.rawQuery(
`SELECT set_id, at::text AS at FROM (
SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
GROUP BY s.set_id) x
WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
before
)
if (rows.length === 0) break
for (const { set_id: setId } of rows) {
looked++
if (this.dryRun) {
const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
if (found) {
merges++
releases += found.size
if (examples.length < 15) {
const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
}
}
} else {
const merged = await merger.mergeByName(setId.trim())
if (merged > 0) {
merges++
releases += merged + 1
}
}
}
before = [rows.at(-1).at, rows.at(-1).set_id]
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
}
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
for (const e of examples) this.logger.info(` ${e.slice(0, 300)}`)
}
}
commands/releases_par2_complete.ts
-38
View File
@@ -1,38 +0,0 @@
import { BaseCommand } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { decidePar2Completeness } from '#services/par2_completeness'
/** Releases per batch. */
const BATCH = 1000
/**
* Decides completeness by par2 set (app/services/par2_completeness.ts) for the releases that had their par2 read
* before it was built (2026-10-23): every holder of a set whose file list is still there, by release id. Once.
*/
export default class ReleasesPar2Complete extends BaseCommand {
static commandName = 'releases:par2-complete'
static description = 'Decides completeness by their par2 sets for settled releases without a counter, then exits'
static options: CommandOptions = { startApp: true }
async run() {
let after = 0
let looked = 0
let decided = 0
for (;;) {
const { rows } = await db.rawQuery(
`SELECT DISTINCT s.release_id FROM par2_sets s
WHERE s.release_id > ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
ORDER BY s.release_id LIMIT ?`,
[after, BATCH]
)
if (rows.length === 0) break
const ids = rows.map((r: { release_id: string }) => Number(r.release_id))
decided += await decidePar2Completeness(db, ids)
looked += ids.length
after = ids.at(-1)!
if (looked % 50_000 < BATCH) this.logger.info(`${looked} releases with a par2 set looked at, ${decided} decided (up to id ${after}).`)
}
this.logger.info(`Done: ${looked} releases with a par2 set looked at, ${decided} decided.`)
}
}
commands/releases_readable.ts
-90
View File
@@ -1,90 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { isUnreadableName } from '#lib/release/index'
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
const CURSOR = 'readable'
/**
* The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
* readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
* false, so the readable filter hides them (an inner name from the prober still shows them), and their probe
* candidates marked unreadable, the prober's first tiers. By the release name,
* which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
* Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
*/
export default class ReleasesReadable extends BaseCommand {
static commandName = 'releases:readable'
static description =
'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'Release ids per batch' })
declare batch: number
@flags.number({ description: 'Stop after this id (a sample)' })
declare toId: number
@flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
declare apply: boolean
@flags.boolean({ description: 'Start from the first id, not the cursor' })
declare restart: boolean
async run() {
const batch = this.batch ?? 20_000
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
CURSOR,
])
let after =
this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
let looked = 0
let changed = 0
const sample: string[] = []
const started = Date.now()
while (after < hi) {
const to = Math.min(after + batch, hi)
const { rows } = await db.rawQuery(
`SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
[after, to]
)
looked += rows.length
const ids: number[] = []
for (const r of rows) {
if (!isUnreadableName(r.name)) continue
ids.push(Number(r.id))
if (sample.length < 40) sample.push(r.name)
}
changed += ids.length
if (this.apply) {
if (ids.length > 0) {
await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
ids,
])
// Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)).
await db.rawQuery(
`UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`,
[ids]
)
}
await db.rawQuery(
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
[CURSOR, String(to)]
)
}
after = to
if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
this.logger.info(
`Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
)
}
}
this.logger.info(
`${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
)
if (!this.apply) for (const n of sample) this.logger.info(` ${n}`)
}
}
commands/releases_recategorize.ts
-92
View File
@@ -1,92 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { setTimeout as sleep } from 'node:timers/promises'
import { DiskSpace } from '#services/disk_space'
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
const CURSOR = 'recategorize'
/**
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
* the dead versions reusable) and goes on from its cursor when run again.
*/
export default class ReleasesRecategorize extends BaseCommand {
static commandName = 'releases:recategorize'
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'Release ids per batch' })
declare batch: number
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
declare toId: number
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
declare floorGb: number
@flags.boolean({ description: 'Count what would change, write nothing' })
declare dryRun: boolean
@flags.boolean({ description: 'Start from the first id, not the cursor' })
declare restart: boolean
async run() {
const batch = this.batch ?? 20_000
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
const changes = new Map<string, number>()
let changed = 0
let looked = 0
const started = Date.now()
while (after < hi) {
while (!this.dryRun && (await space.short())) await sleep(60_000)
const to = Math.min(after + batch, hi)
const fresh = `release_category_for(name, inner_name, groups, NULL)`
if (this.dryRun) {
const { rows } = await db.rawQuery(
`SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
[after, to]
)
for (const r of rows) {
looked += r.n
if (r.was === r.now) continue
changed += r.n
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
}
} else {
const result = await db.rawQuery(
`UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
[after, to]
)
changed += result.rowCount ?? 0
await db.rawQuery(
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
[CURSOR, String(to)]
)
}
after = to
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
}
}
if (this.dryRun) {
this.logger.info(`Would change ${changed} of ${looked}:`)
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
return
}
// The spots' categories for the linked releases whose names say nothing.
const spots = await db.rawQuery(
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
)
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
}
}
commands/releases_regroup.ts
+2 -3
View File
@@ -3,7 +3,6 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { decodeChunk } from '#lib/chunks/index'
import { releaseIdentity } from '#lib/release/index'
import { sourceCode } from '#lib/release/source'
/**
* What lib/release's identity makes of releases already in the catalog: a
@@ -28,7 +27,7 @@ export default class ReleasesRegroup extends BaseCommand {
async run() {
for (const name of this.group ?? []) {
const { rows } = await db.rawQuery(
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
[name]
)
const ids = rows.map((r: { release_id: string }) => r.release_id)
@@ -55,7 +54,7 @@ export default class ReleasesRegroup extends BaseCommand {
}
if (this.show?.length) return
for (const source of ['subject', 'post', 'stem']) {
const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
let split = 0
let pieces = 0
const examples: string[] = []
commands/replay_corpus.ts
-396
View File
@@ -1,396 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { spawn } from 'node:child_process'
import { access, readdir, readFile, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import db from '@adonisjs/lucid/services/db'
import coordinatorConfig from '#config/coordinator'
import BatchStore from '#services/batch_store'
import Leases from '#services/leases'
import RuleSets from '#services/rule_sets'
import { fromWire, type WireBatch } from '#lib/edge/index'
/**
* Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
* Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)). With
* REPLAY_INTO_CATALOG=1 any database but the test one: voyager 3 deployed, history replayed from the corpus into
* the catalog itself (deploy/replay-window.sh, LOG.md (116)).
*/
const SCRATCH_DB = /_(replay|scratch)$/
const TEST_DB = /_test$/
const NODE_NAME = 'replay'
interface StoredRange {
id: number
start: number
end: number
}
/**
* The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
* window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
* scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
*
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
*
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
*/
export default class ReplayCorpus extends BaseCommand {
static commandName = 'replay:corpus'
static description =
"Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
static options: CommandOptions = { startApp: true }
@flags.string({ description: 'The group (its directory under the corpus)', required: true })
declare group: string
@flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
declare dir?: string
@flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
declare limit?: number
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
declare from?: number
@flags.number({
description: "Skip ranges starting after this article (one window of the group's history)",
})
declare to?: number
@flags.number({
description: 'Chunks of 100k articles per batch, as the worker posts them',
default: 5,
})
declare batchChunks: number
@flags.string({
description:
'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
})
declare bench?: string
@flags.boolean({
description:
'Leave the window open at the end (the last ranges provisional), to continue later',
})
declare noFinish: boolean
async run() {
const database = db.connection().getReadClient().client.config.connection.database as string
const intoCatalog = process.env.REPLAY_INTO_CATALOG === '1' && !TEST_DB.test(database)
if (!SCRATCH_DB.test(database) && !intoCatalog) {
this.logger.error(
`The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}"; REPLAY_INTO_CATALOG=1 allows it)`
)
this.exitCode = 1
return
}
const dir = this.dir ?? coordinatorConfig.corpus.dir
if (!dir) {
this.logger.error('No corpus directory: --dir or CORPUS_DIR')
this.exitCode = 1
return
}
const bench = await this.benchBinary()
if (!bench) {
this.logger.error(
'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
)
this.exitCode = 1
return
}
const groupDir = join(dir, this.group)
const stored = await this.storedRanges(groupDir)
if (stored.length === 0) {
this.logger.error(`${groupDir}: no complete range`)
this.exitCode = 1
return
}
const nodeId = await this.nodeId()
const groupId = await this.groupId()
const ranges = await this.leaseRanges(groupId, nodeId, stored)
const rules = await RuleSets.current()
const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
await writeFile(rulesPath, JSON.stringify(rules))
this.logger.info(
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
)
// edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
const args = [
'--group',
groupDir,
'--rules',
rulesPath,
'--batch-chunks',
String(this.batchChunks),
'--wire',
'-',
]
if (this.from) args.push('--from', String(this.from))
if (this.to) args.push('--to', String(this.to))
if (this.limit) args.push('--limit', String(this.limit))
if (this.noFinish) args.push('--no-finish')
const started = Date.now()
const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
const totals = {
batches: 0,
headers: 0,
releases: 0,
newReleases: 0,
finalized: 0,
candidates: 0,
storeMs: 0,
}
let current: StoredRange | null = null
const storeLine = async (line: string) => {
const batch = fromWire(JSON.parse(line) as WireBatch)
if (!current || batch.from < current.start || batch.from > current.end) {
current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
}
const at = Date.now()
const result = await BatchStore.store(current.id, nodeId, batch)
totals.storeMs += Date.now() - at
totals.batches++
totals.headers += batch.headers
totals.releases += result.releases
totals.newReleases += result.newReleases
totals.finalized += result.finalized
totals.candidates += result.candidates
if (batch.next > current.end) {
await Leases.finish(current.id, nodeId)
const seconds = (Date.now() - started) / 1000
this.logger.info(
` range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
`(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
)
}
}
// Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
let rest = ''
let failure: Error | null = null
try {
for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
rest += chunk.toString('utf8')
let nl: number
while ((nl = rest.indexOf('\n')) >= 0) {
const line = rest.slice(0, nl)
rest = rest.slice(nl + 1)
if (line.length > 0) await storeLine(line)
}
}
if (rest.trim().length > 0) await storeLine(rest)
} catch (error: any) {
failure = error
child.kill('SIGTERM')
}
const code = await new Promise<number | null>((done) =>
child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
)
if (failure) {
this.logger.error(`Replay failed: ${failure.message}`)
this.exitCode = 1
return
}
if (code !== 0) {
this.logger.error(`edge-bench exited with ${code}`)
this.exitCode = 1
return
}
await this.report(groupId, ranges, Date.now() - started)
}
private async benchBinary(): Promise<string | null> {
const candidates = [
this.bench,
process.env.EDGE_BENCH,
resolve('edge/target/release/edge-bench'),
'/usr/local/bin/edge-bench',
].filter((p): p is string => !!p)
for (const path of candidates) {
if (
await access(path).then(
() => true,
() => false
)
)
return path
}
return null
}
/** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
const names = await readdir(groupDir).catch(() => [] as string[])
const ranges: { start: number; end: number }[] = []
for (const name of names) {
const m = /^(\d+)-(\d+)$/.exec(name)
if (!m) continue
const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
if (!json) continue
ranges.push({ start: Number(m[1]), end: Number(m[2]) })
}
ranges.sort((a, b) => a.start - b.start)
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
if (this.limit) chosen = chosen.slice(0, this.limit)
return chosen
}
private async nodeId(): Promise<number> {
const row = await db.from('nodes').where('name', NODE_NAME).first()
if (row) return Number(row.id)
const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
return Number(inserted.id)
}
/** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
private async groupId(): Promise<number> {
const row = await db.from('groups').where('name', this.group).first()
if (row) return Number(row.id)
const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
const [inserted] = await db
.table('groups')
.insert({ name: this.group, priority: Number(rows[0].next), active: true })
.returning('id')
this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
return Number(inserted.id)
}
/** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
private async leaseRanges(
groupId: number,
nodeId: number,
stored: { start: number; end: number }[]
): Promise<StoredRange[]> {
const out: StoredRange[] = []
const now = new Date()
for (const r of stored) {
const lease = {
status: 'leased',
node_id: nodeId,
next_id: r.start,
headers: 0,
posted_low: null,
posted_high: null,
leased_at: now,
heartbeat_at: now,
done_at: null,
error: null,
capture: true,
}
const existing = await db
.from('ranges')
.where('group_id', groupId)
.where('start_id', r.start)
.first()
if (existing) {
await db
.from('ranges')
.where('id', existing.id)
.update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
out.push({ id: Number(existing.id), start: r.start, end: r.end })
continue
}
const [inserted] = await db
.table('ranges')
.insert({
...lease,
group_id: groupId,
start_id: r.start,
end_id: r.end,
priority: 0,
attempts: 1,
})
.returning('id')
out.push({ id: Number(inserted.id), start: r.start, end: r.end })
}
return out
}
/** What the replay left in the catalog (this group's releases; the sizes are the whole catalog's), per article. */
private async report(groupId: number, ranges: StoredRange[], elapsedMs: number) {
const ids = ranges.map((r) => r.id)
const { rows: counts } = await db.rawQuery(
`SELECT count(*)::bigint AS releases,
count(settled_at)::bigint AS finished,
count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
count(*) FILTER (WHERE edge)::bigint AS edges,
count(*) FILTER (WHERE complete)::bigint AS complete,
count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
count(*) FILTER (WHERE readable)::bigint AS readable,
count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
coalesce(sum(files), 0)::bigint AS files,
coalesce(sum(segments), 0)::bigint AS segments,
coalesce(sum(bytes), 0)::bigint AS bytes
FROM releases WHERE group_id = ?`,
[groupId]
)
const { rows: sources } = await db.rawQuery(
`SELECT source, count(*)::bigint AS n FROM releases WHERE group_id = ? GROUP BY source ORDER BY source`,
[groupId]
)
const { rows: pending } = await db.rawQuery(
`SELECT count(*)::bigint AS n, coalesce(sum(segments), 0)::bigint AS segments, count(*) FILTER (WHERE jsonb_array_length(candidates) = 0)::bigint AS unprobable
FROM pending_releases WHERE group_id = ?`,
[groupId]
)
const { rows: headerRows } = await db.rawQuery(
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
[ids]
)
const headers = Number(headerRows[0].headers)
const { rows: sizes } = await db.rawQuery(
`SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
sum(pg_total_relation_size(c.oid))::bigint AS bytes
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relkind = 'r'
AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks', 'pending_releases')
OR c.relname LIKE 'release_chunks%')
GROUP BY 1 ORDER BY 2 DESC`
)
// On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
const hdd = new Set([
'release_chunks',
'probe_candidates',
'release_inspections',
'pending_releases',
])
let nvme = 0
let other = 0
const c = counts[0]
const fmt = (n: unknown) => Number(n).toLocaleString('en')
this.logger.info(
`Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
)
this.logger.info(
`Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
`complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
`${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
)
this.logger.info(
`By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
)
this.logger.info(
`Pending (unnamed, outside the catalog): ${fmt(pending[0].n)} releases, ${fmt(pending[0].segments)} segments, ${fmt(pending[0].unprobable)} without a probe target.`
)
for (const s of sizes) {
const bytes = Number(s.bytes)
if (hdd.has(s.name)) other += bytes
else nvme += bytes
this.logger.info(
` ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
)
}
this.logger.info(
`NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header (the whole catalog's sizes over this group's headers: only the first group's are its own).`
)
}
}
commands/rules_classes.ts
-54
View File
@@ -1,54 +0,0 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import RuleSets from '#services/rule_sets'
import { classKey, describeClass } from '#lib/release/index'
/**
* Publishes a rule set whose dropped classes are the pending ladder's (pending_classes.dropped; LOG.md (130)): the
* edge keeps unnamed releases of those classes out of the pending store from then on, their explored sample
* excepted. Run again as the record grows; unchanged classes publish nothing.
*/
export default class RulesClasses extends BaseCommand {
static commandName = 'rules:classes'
static description = "Publishes a rule set with the pending ladder's dropped classes"
static options: CommandOptions = { startApp: true }
@flags.boolean({ description: 'Show what would be published, write nothing' })
declare dryRun: boolean
async run() {
const { rows } = await db.rawQuery(
`SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named FROM pending_classes c JOIN groups g ON g.id = c.group_id WHERE c.dropped ORDER BY 1, 2, 3`
)
const next = new Set<string>(rows.map((r: any) => classKey(r.tool, r.grp, Number(r.klass))))
const { version, priorities: _p, ...current } = await RuleSets.current()
const before = new Set(current.droppedClasses ?? [])
const added = [...next].filter((c) => !before.has(c))
const removed = [...before].filter((c) => !next.has(c))
if (added.length === 0 && removed.length === 0) {
this.logger.info(
`Unchanged: rule set ${version} has the ${before.size} dropped classes already.`
)
return
}
for (const r of rows.slice(0, 40)) {
this.logger.info(
` ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} ${r.named} of ${r.probed} named`
)
}
if (this.dryRun) {
this.logger.info(
`Would publish ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
)
return
}
const published = await RuleSets.publish(
{ ...current, droppedClasses: [...next].sort() },
`${next.size} dropped classes from the pending ladder`
)
this.logger.info(
`Rule set ${published}: ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
)
}
}
config/access.ts
+31
View File
@@ -0,0 +1,31 @@
import env from '#start/env'
/**
* Who may use the catalog and the Newznab API.
*
* `open` (the default): as before, one key for Newznab (NEWZNAB_API_KEY, or
* none) and an open web catalog, no limits. `accounts`: every request names a
* user (a Newznab `apikey`, or the web interface's token), whose tier sets what
* they may do per UTC day and how far back they see.
*/
export const ACCESS = env.get('ACCESS', 'open')
/** Who may sign up: anyone, with an unused invite code, or no one. With accounts, invites unless set. */
export const SIGNUP = env.get('SIGNUP', ACCESS === 'accounts' ? 'invite' : 'open')
export interface TierLimits {
/** Searches and release details per day (Newznab t=search & co., the catalog); null: no limit. */
apiHits: number | null
/** NZBs per day; null: no limit. */
grabs: number | null
/** How many days back releases are found and fetched; null: everything. */
retentionDays: number | null
}
/** A tier that runs out (users.tier_until) falls back to `free`. */
export const TIERS: Record<string, TierLimits> = {
free: { apiHits: 100, grabs: 10, retentionDays: 365 },
vip: { apiHits: 10_000, grabs: 1_000, retentionDays: null },
// Staff and friends.
unlimited: { apiHits: null, grabs: null, retentionDays: null },
}
config/coordinator.ts
+2 -61
View File
@@ -31,37 +31,8 @@ export default {
},
/** A leased range without a heartbeat for this long goes back to the queue. */
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
/**
* The ranges group leases hand out (REBUILD.md design 1), by priority: the tail is negative, the capture windows
* 0–2 (2016, 2021, 2025), the history by its position in the group (0 oldest … 1000 newest: its first buckets
* share 0–2 with the windows) or 4000+. Bounds so a deployment can run one window first, with
* LEASE_CAPTURE_ONLY so the history's 0 isn't taken along (homelab 2026-10-26: 0..0 and capture only, the 2016
* window; LOG.md (118)); unbounded and any range by default.
*/
leasePriorityMin: env.get('LEASE_MIN_PRIORITY', -1_000_000),
leasePriorityMax: env.get('LEASE_MAX_PRIORITY', 1_000_000),
leaseCaptureOnly: env.get('LEASE_CAPTURE_ONLY', false),
/**
* How long a range's stream waits for a chunk the prefetcher hasn't spooled yet (app/controllers/node_controller.ts).
* Long: a chunk that keeps coming in isn't given up (only a stalled one is), and some groups' chunks take 20+
* minutes (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
*/
streamChunkWaitMs: env.get('STREAM_CHUNK_WAIT_MINUTES', 60) * 60 * 1000,
/** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
/** NNTP connections the coordinator fetches headers with. */
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
/**
* The account's connections, shared by the coordinator and the prober by priority (app/services/nntp_shares.ts):
* all of them less the reserved ones (the spotter's and the spot inspector's 1 each, the coordinator's plain one,
* and 2 for a hedged request's or a reconnect's new connection while the old one closes).
*/
nntpShares: {
total: env.get('NNTP_ACCOUNT_CONNECTIONS', 50) - env.get('NNTP_RESERVED_CONNECTIONS', 5),
everyMs: 15_000,
/** Headers before probes: history takes what it can use, the prober the rest. */
coordinatorPriority: 2,
proberPriority: 1,
},
/**
* A reply with no data for this long fails, and its connection is replaced.
* Eweka sometimes stops answering on long-lived connections (all 40 quiet,
@@ -97,36 +68,17 @@ export default {
* under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
*/
nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
/**
* A connection with nothing to do for this long is closed, and opened again
* when there's work. 0: kept. Eweka drops connections that stay quiet, and
* the next command on one failed ("Spot inspection failed: NNTP connection
* closed"); idle ones also take the account's connections from downloads.
*/
nntpIdleCloseMs: env.get('NNTP_IDLE_CLOSE_SECONDS', 60) * 1000,
probe: {
/**
* At most (its share, nntpShares): NNTP connections the prober reads articles with, besides the
* NNTP connections the prober reads articles with, besides the
* coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
* couple stay free for downloads. ~7 MB/s each (~10 articles/s).
*/
connections: env.get('PROBE_CONNECTIONS', 6),
/** Of them, always (its share's minimum; nntpShares). */
minConnections: env.get('PROBE_MIN_CONNECTIONS', 2),
/** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */
minFreeGb: env.get('PROBE_MIN_FREE_GB', 0),
/** Candidates claimed at a time. */
batch: 400,
/** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
pipeline: env.get('PROBE_PIPELINE', 8),
/**
* The pending ladder (REBUILD.md decision f; app/services/pending_ladder.ts): the share of each claim taken
* from pending_releases at least (the catalog's own candidates come first, the rest of the claim is theirs).
*/
pendingShare: env.get('PENDING_SHARE', 0.25),
/** A signature (tool, group, poster) with this many climbs and at most pendingMaxNamed of them named is dropped. */
pendingMinProbes: env.get('PENDING_MIN_PROBES', 100),
pendingMaxNamed: env.get('PENDING_MAX_NAMED', 0.005),
},
/**
* The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
@@ -155,18 +107,7 @@ export default {
* ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
*/
ahead: env.get('CORPUS_AHEAD', 16),
/**
* Stored without the lines the edge worker drops on their own (crossposts another indexed group owns, obfuscated
* subjects; ~48% of the indexed ranges' headers), what was left out in each chunk's `.drops`: twice the history
* in the same space (LOG.md 2026-10-23 (101)). Off: as the news server sent them.
*/
filter: env.get('CORPUS_FILTER', true),
},
/**
* The group windows' snapshots (REBUILD.md, "Voyager 3: window indexing", 5; app/services/group_leases.ts): a
* file a group on bulk, next to the spool by default.
*/
snapshotDir: env.get('SNAPSHOT_DIR', `${env.get('SPOOL_DIR', app.tmpPath('spool'))}/snapshots`),
spool: {
dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
/** Ranges kept fetched ahead, per active lease. */
database/migrations/1770000003300_access.ts
+49
View File
@@ -0,0 +1,49 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Accounts with limits (config/access.ts): a Newznab key and a tier per user,
* invites, and what each user used per day.
*/
export default class extends BaseSchema {
async up() {
this.schema.alterTable('users', (table) => {
// Existing users get a key of their own (a volatile default is evaluated per row).
table.string('api_key', 32).notNullable().unique().defaultTo(this.raw(`md5(random()::text || clock_timestamp()::text)`))
table.string('tier', 16).notNullable().defaultTo('free')
// Null: the tier doesn't run out.
table.timestamp('tier_until', { useTz: true }).nullable()
table.boolean('suspended').notNullable().defaultTo(false)
})
this.schema.createTable('invites', (table) => {
table.string('code', 32).primary()
// What the invite gives (e.g. a sold VIP invite); null: the free tier.
table.string('tier', 16).nullable()
table.integer('tier_days').nullable()
table.string('note').nullable()
table.integer('used_by').nullable().references('id').inTable('users').onDelete('SET NULL')
table.timestamp('used_at', { useTz: true }).nullable()
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
})
this.schema.createTable('usage', (table) => {
table.integer('user_id').notNullable().references('id').inTable('users').onDelete('CASCADE')
// A UTC day.
table.date('day').notNullable()
table.integer('api_hits').notNullable().defaultTo(0)
table.integer('grabs').notNullable().defaultTo(0)
table.primary(['user_id', 'day'])
})
}
async down() {
this.schema.dropTable('usage')
this.schema.dropTable('invites')
this.schema.alterTable('users', (table) => {
table.dropColumn('api_key')
table.dropColumn('tier')
table.dropColumn('tier_until')
table.dropColumn('suspended')
})
}
}
database/migrations/1770000004200_releases_sidecar_only.ts
-16
View File
@@ -1,16 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* releases.sidecar_only: set by the batch store when it inserts a small release (at most 3 files, under 1 MB) whose
* files are all sidecars (.nfo, .sfv, .nzb, …). The settler queues only those for the sidecar fold: it queued every
* small release, 13.5M (single images mostly), and the fold's queue scans held the merger back (LOG.md 2026-10-17 (72)).
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS sidecar_only boolean`)
}
async down() {
this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS sidecar_only`)
}
}
database/migrations/1770000004300_worker_cursors_key.ts
-16
View File
@@ -1,16 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* worker_cursors.key: a tie-breaker after `at`. The par2 pruner's cursor was a timestamp alone, read back to the
* millisecond: 20,000 sets of one prober flush share created_at 2026-10-15 14:38:39.883984, and every round took the
* same batch again (LOG.md 2026-10-17 (73)). Now (at to the microsecond, set_id).
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`ALTER TABLE worker_cursors ADD COLUMN IF NOT EXISTS key text`)
}
async down() {
this.schema.raw(`ALTER TABLE worker_cursors DROP COLUMN IF EXISTS key`)
}
}
database/migrations/1770000004400_nntp_shares.ts
-26
View File
@@ -1,26 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* nntp_shares: the account's connections shared by the processes that read articles (app/services/nntp_shares.ts).
* Each process's row says what it could use and has open, and what it may open; a row without a heartbeat for a
* minute is a process gone (its connections closed with it). The split was fixed per process: with history paused
* the coordinator's 38 stood idle while the prober's 8 were the limit (LOG.md 2026-10-18 (75)).
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE IF NOT EXISTS nntp_shares (
name text PRIMARY KEY,
priority integer NOT NULL,
min integer NOT NULL,
want integer NOT NULL,
open integer NOT NULL,
granted integer NOT NULL,
heartbeat_at timestamptz NOT NULL DEFAULT now()
)`)
}
async down() {
this.schema.raw(`DROP TABLE IF EXISTS nntp_shares`)
}
}
database/migrations/1770000004500_probe_cache_pkey_on_nvme.ts
-26
View File
@@ -1,26 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* probe_cache's primary key in the catalog's tablespace (default_tablespace: nvme on homelab); the table stays on the
* HDD. Each claim looks its candidates up by message_id (`message_id = ANY(…)`): with the index on the HDD that took
* 1.27 s a claim, 48 s of each minute, the prober's lanes waiting on claims (LOG.md 2026-10-18 (76)). ~97% aren't in it,
* so only the index is read. Homelab moved it by hand (prober stopped); a no-op where it's there already, or without a
* default tablespace (tests).
*/
export default class extends BaseSchema {
async up() {
this.defer(async (db) => {
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
const target = setting[0].target
if (!target) return
const { rows } = await db.rawQuery(
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
WHERE c.relname = 'probe_cache_pkey' AND coalesce(t.spcname, 'pg_default') <> ?`,
[target]
)
if (rows.length > 0) await db.rawQuery(`ALTER INDEX probe_cache_pkey SET TABLESPACE ${target}`)
})
}
async down() {}
}
database/migrations/1770000004600_release_inspections_on_hdd.ts
-12
View File
@@ -1,12 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Was: release_inspections' rows on the HDD (pg_default), for the NVMe's room during the probe backlog (LOG.md
* 2026-10-18 (79)). The judge's fake rule reads all of it every round: on the HDD that took 17–18 min, was ~3
* (LOG.md 2026-10-19 (81)). Migration 4800 moves it back; nothing here now.
*/
export default class extends BaseSchema {
async up() {}
async down() {}
}
database/migrations/1770000004700_judge_indexes.ts
-22
View File
@@ -1,22 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* The junk judge's locked rule by an index of the passworded releases (1.2M of 74M), not a scan of all of releases
* every round (30 s; LOG.md 2026-10-19 (80)). And the posters of the spot inspector's own releases (source `spot`),
* which it didn't add to posters: poster search, and the judge's scraped rule by posters, missed them.
*/
export default class extends BaseSchema {
static disableTransactions = true
async up() {
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_passworded_idx ON releases (id) WHERE passworded`)
this.schema.raw(
`INSERT INTO posters (poster) SELECT DISTINCT left(s.poster, 200) FROM spots s
JOIN releases r ON r.id = s.release_id AND r.source = 'spot' WHERE s.poster IS NOT NULL ON CONFLICT DO NOTHING`
)
}
async down() {
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_passworded_idx`)
}
}
database/migrations/1770000004800_release_inspections_on_nvme.ts
-25
View File
@@ -1,25 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* release_inspections back in the catalog's tablespace (default_tablespace: nvme on homelab), where migration 4600 had
* moved it to the HDD: the judge's fake rule reads all of it every round, 17–18 min on the HDD, ~3 on the NVMe (LOG.md
* 2026-10-19 (81)). Homelab moved it by hand (prober, merger and judge stopped); a no-op where it's there already, or
* without a default tablespace (tests).
*/
export default class extends BaseSchema {
async up() {
this.defer(async (db) => {
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
const target = setting[0].target
if (!target) return
const { rows } = await db.rawQuery(
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
WHERE c.relname = 'release_inspections' AND c.relkind = 'r' AND coalesce(t.spcname, 'pg_default') <> ?`,
[target]
)
if (rows.length > 0) await db.rawQuery(`ALTER TABLE release_inspections SET TABLESPACE ${target}`)
})
}
async down() {}
}
database/migrations/1770000004900_judge_dirty.ts
-119
View File
@@ -1,119 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* The junk judge by what changed (LOG.md 2026-10-19 (83)): a full round over every release took 35–38 min of each
* hour, longer with every inspection the prober added. Triggers note each release whose facts a rule reads changed
* (judge_dirty: its probes' statuses, inspections, file hashes, passwords, spots, par2 sets, merges, its own judged
* columns); the judge judges those every few minutes, and every release in a daily full round (judge_state: when the
* last one was, by which rule set), its matches gathered in judge_full. judge_dirty is unlogged: what a crash loses
* the next full round judges.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_dirty (release_id bigint NOT NULL)`)
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_full (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL)`)
this.schema.raw(
`CREATE TABLE IF NOT EXISTS judge_state (id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1), full_at timestamptz NOT NULL, version integer NOT NULL)`
)
// The verdicts are an hourly full round's: the next full one a day from now.
this.schema.raw(`INSERT INTO judge_state (full_at, version) VALUES (now(), 1) ON CONFLICT DO NOTHING`)
// Rows of a table with release_id: the releases of the rows inserted, deleted, or updated (both, when moved).
this.schema.raw(`
CREATE OR REPLACE FUNCTION judge_dirty_by_release_id() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF TG_OP IN ('INSERT', 'UPDATE') THEN
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
END IF;
IF TG_OP IN ('UPDATE', 'DELETE') THEN
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM old_rows WHERE release_id IS NOT NULL;
END IF;
RETURN NULL;
END $$`)
// probe_candidates: only a status that counts for the dead rule, or a candidate moved by a merge (not the claims'
// 'taken', the settler's ready).
this.schema.raw(`
CREATE OR REPLACE FUNCTION judge_dirty_probes() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
INSERT INTO judge_dirty (release_id)
SELECT DISTINCT unnest(ARRAY[n.release_id, o.release_id]) FROM new_rows n JOIN old_rows o ON o.id = n.id
WHERE n.release_id IS DISTINCT FROM o.release_id
OR (n.status IS DISTINCT FROM o.status AND (n.status IN ('done', 'missing', 'error') OR o.status IN ('done', 'missing', 'error')));
RETURN NULL;
END $$`)
this.schema.raw(`
CREATE OR REPLACE FUNCTION judge_dirty_merges() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
INSERT INTO judge_dirty (release_id) SELECT DISTINCT into_id FROM new_rows;
RETURN NULL;
END $$`)
this.schema.raw(`
CREATE OR REPLACE FUNCTION judge_dirty_releases() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF TG_OP = 'INSERT' THEN
INSERT INTO judge_dirty (release_id) SELECT id FROM new_rows;
ELSIF TG_OP = 'DELETE' THEN
INSERT INTO judge_dirty (release_id) SELECT id FROM old_rows;
ELSE
INSERT INTO judge_dirty (release_id) VALUES (NEW.id);
END IF;
RETURN NULL;
END $$`)
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets']) {
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON ${table}`)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON ${table}`)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON ${table}`)
this.schema.raw(
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON ${table} REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON ${table} REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON ${table} REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
}
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON probe_candidates`)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON probe_candidates`)
this.schema.raw(
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON probe_candidates REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_probes()`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON probe_candidates REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON release_merges`)
this.schema.raw(
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_merges REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_merges()`
)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON releases`)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON releases`)
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON releases`)
this.schema.raw(
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
)
// Per row, and only when a column a rule reads changed: the tail and the settler update releases all the time.
this.schema.raw(
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
WHEN (OLD.name IS DISTINCT FROM NEW.name OR OLD.poster IS DISTINCT FROM NEW.poster OR OLD.source IS DISTINCT FROM NEW.source
OR OLD.file_total IS DISTINCT FROM NEW.file_total OR OLD.passworded IS DISTINCT FROM NEW.passworded
OR OLD.settled_at IS DISTINCT FROM NEW.settled_at OR OLD.readable IS DISTINCT FROM NEW.readable
OR OLD.inner_name IS DISTINCT FROM NEW.inner_name OR OLD.group_id IS DISTINCT FROM NEW.group_id
OR OLD.first_article IS DISTINCT FROM NEW.first_article OR OLD.last_article IS DISTINCT FROM NEW.last_article
OR OLD.posted_at IS DISTINCT FROM NEW.posted_at OR OLD.last_posted_at IS DISTINCT FROM NEW.last_posted_at
OR OLD.complete IS DISTINCT FROM NEW.complete)
EXECUTE FUNCTION judge_dirty_releases()`
)
}
async down() {
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets', 'probe_candidates', 'release_merges', 'releases']) {
for (const event of ['insert', 'update', 'delete']) this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_${event} ON ${table}`)
}
this.schema.raw(`DROP FUNCTION IF EXISTS judge_dirty_by_release_id(), judge_dirty_probes(), judge_dirty_merges(), judge_dirty_releases()`)
this.schema.raw(`DROP TABLE IF EXISTS judge_dirty, judge_full, judge_state`)
}
}
database/migrations/1770000004900_range_dates.ts
-27
View File
@@ -1,27 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* When a group's articles were posted, sampled from the news server (ranges:dates; app/services/group_dates.ts), and
* from those each range's estimated posting date, so ranges can be chosen by era: ranges:capture --from --to marks a
* posting-time window of every group for the corpus (voyager 3's test data; REBUILD.md, corpus). `stored_at`: the
* range is complete in the corpus (the prefetcher saw its range.json), so the corpus fill skips it without looking.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE IF NOT EXISTS group_dates (
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
article bigint NOT NULL,
posted_at timestamptz NOT NULL,
PRIMARY KEY (group_id, article)
)`)
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS est_date timestamptz`)
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS stored_at timestamptz`)
}
async down() {
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS stored_at`)
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS est_date`)
this.schema.raw(`DROP TABLE IF EXISTS group_dates`)
}
}
database/migrations/1770000005000_file_hashes_hash_size_release.ts
-21
View File
@@ -1,21 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* file_hashes by (hash16k, size, release_id), in place of (hash16k): how many releases have a file, from the index
* alone. The judge counts that for every file of each changed release (repeated); with the row fetched for size and
* release_id per entry, a file posted in 100k releases cost 100k heap reads every round (24–32 s a chunk; LOG.md
* 2026-10-19 (84)). The merger and pruner join on (hash16k, size) too.
*/
export default class extends BaseSchema {
static disableTransactions = true
async up() {
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_size_release_idx ON file_hashes (hash16k, size, release_id)`)
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_idx`)
}
async down() {
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_idx ON file_hashes (hash16k)`)
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_size_release_idx`)
}
}
database/migrations/1770000005100_late_candidates_ready.ts
-17
View File
@@ -1,17 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Probe candidates added to a release after it settled (a later range of its upload, a merge into it) stayed not
* ready: the settler marks a release's candidates only when it settles it (LOG.md 2026-10-20 (87)). 93,642 waited
* since 2026-10-13 (17k lone, 31k par2, 45k archive). The batch store and the merge mark them now; these are made
* claimable once.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
WHERE pc.status = 'waiting' AND NOT pc.ready AND r.id = pc.release_id AND r.settled_at IS NOT NULL`)
}
async down() {}
}
database/migrations/1770000005200_release_category_trigger.ts
-128
View File
@@ -1,128 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* A release's category from more than its name (REBUILD.md, part 9; LOG.md 2026-10-23 (95)). Migration 1200 made it
* a generated column of release_category(name), and knex took the `?` in its regexes for bindings: the function held
* `blu-$1ray`, `web-$2dl`, `h\.$326[45]`, so BluRay, WEB-DL and H.264 names without a resolution were Misc. Now:
* - release_category(name), fixed (no `?` in any regex here: `{0,1}`) and with XXX, audiobooks, console and PC games,
* and TV codes the old one missed (`S01e07[`, `S04E01E02`);
* - the inner name the prober found (an obfuscated release's real name) when the name says nothing;
* - else a category given before (a spot's, below), else the groups' (erotica, sounds, ebooks, …);
* - a spot's category (spot_category: its head category and subcategories), set by the spot inspector.
* `category` becomes a plain column (DROP EXPRESSION keeps the values, no rewrite) set by a trigger; existing rows are
* recategorized by `releases:recategorize` in batches, not here (a rewrite of the 70 GB table).
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(String.raw`
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
SELECT CASE
-- XXX first: its names carry years, resolutions and episode-like codes too.
WHEN name ~* '(^|[^a-z0-9])xxx([^a-z0-9]|$)' THEN
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 6045
WHEN name ~* '720p|1080p|x26[45]|h\.{0,1}26[45]|hevc' THEN 6040
WHEN name ~* 'imageset|picset|(^|[._ -])pics([._ -]|$)' THEN 6060 ELSE 6070 END
-- TV: S01E02 (also S01e02[, S04E01E02), 1x02, or a season pack (S01).
WHEN name ~* '(^|[^a-z0-9])(s[0-9]{1,2}[ ._-]{0,1}e[0-9]{1,3}|[0-9]{1,2}x[0-9]{2})([^0-9]|$)' OR name ~* '(^|[._ -])s[0-9]{1,2}([._ -]|$)' THEN
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
-- Movies: a year plus a source or quality marker.
WHEN name ~* '(^|[._ (-])(19|20)[0-9]{2}([._ )-]|$)'
AND name ~* '(480p|576p|720p|1080p|2160p|blu-{0,1}ray|web-{0,1}dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.{0,1}26[45]|hevc|remux)' THEN
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
WHEN name ~* '720p|1080p|blu-{0,1}ray' THEN 2040 ELSE 2030 END
-- Audio: audiobooks, lossless, mp3, or scene music names (-WEB-2026-GROUP).
WHEN name ~* '(audiobook|audioboek|luisterboek|h(oe|ö)rbuch)' THEN 3030
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)[0-9]{2}-[A-Za-z0-9]+$' THEN 3010
WHEN name ~* '(^|[._ -])(epub|ebook|e-book|mobi|azw3)([._ -]|$)' THEN 7020
-- Games: console tags, PC scene groups and repackers.
WHEN name ~* '(^|[._ -])(nsw|ps4|ps5|xboxone|xbox360|wiiu|3ds|nds)([._ -]|$)' THEN 1000
WHEN name ~* '-(codex|plaza|skidrow|reloaded|flt|doge|tenoke|rune|empress|razor1911|tinyiso|darksiders|simplex)$' OR name ~* '(fitgirl|dodi|elamigos)[._ -]{0,1}repack' THEN 4050
ELSE 8010
END
$$`)
this.schema.raw(String.raw`
CREATE OR REPLACE FUNCTION release_group_category(groups jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
SELECT CASE
WHEN g ~* '(erotica|xxx|porn|erotic|nude|naked)' THEN CASE WHEN g ~* '(pictures|pics|images|photos)' THEN 6060 ELSE 6070 END
WHEN g ~* 'anime' THEN 5070
WHEN g ~* 'audiobook' THEN 3030
WHEN g ~* '(sounds|mp3|music)' THEN CASE WHEN g ~* '(lossless|flac)' THEN 3040 ELSE 3010 END
WHEN g ~* 'comics' THEN 7030
WHEN g ~* 'e-{0,1}books{0,1}' THEN 7020
WHEN g ~* '(games|nintendo|xbox|playstation|wii)' THEN 1000
WHEN g ~* '(^|[. ])(tv|hdtv|teevee|tvseries)([. ]|$)' THEN 5000
WHEN g ~* '(^|[. ])(movies|dvd|bluray|dvdr)([. ]|$)' THEN 2000
ELSE 8010
END
FROM (SELECT coalesce(string_agg(value, ' '), '') AS g FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(groups) = 'array' THEN groups ELSE '[]' END)) x
$$`)
// The name's, else the inner name's, else one given before (a spot's), else the groups'.
this.schema.raw(`
CREATE OR REPLACE FUNCTION release_category_for(name text, inner_name text, groups jsonb, given integer) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
SELECT CASE WHEN n <> 8010 THEN n WHEN i <> 8010 THEN i WHEN coalesce(given, 8010) <> 8010 THEN given ELSE release_group_category(groups) END
FROM (SELECT release_category(name) AS n, coalesce(release_category(inner_name), 8010) AS i) x
$$`)
// Whether a spot's subcategories of one type letter ({"a": [9], "z": [0]}) hold one of `vals`.
this.schema.raw(`
CREATE OR REPLACE FUNCTION spot_has(subcats jsonb, letter text, vals integer[]) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
SELECT coalesce((SELECT bool_or((e #>> '{}')::integer = ANY(vals)) FROM jsonb_array_elements(
CASE WHEN jsonb_typeof(subcats->letter) = 'array' THEN subcats->letter ELSE '[]' END) e), false)
$$`)
// A spot's category (lib/spotnet/categories.ts) in Newznab's; null for pictures and the unknown.
this.schema.raw(`
CREATE OR REPLACE FUNCTION spot_category(category integer, subcats jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
SELECT CASE category
WHEN 0 THEN CASE
WHEN spot_has(subcats, 'z', '{3}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 6045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 6040
WHEN spot_has(subcats, 'a', '{12,13}') THEN 6060 ELSE 6070 END
WHEN spot_has(subcats, 'z', '{1}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 5045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 5040 ELSE 5030 END
WHEN spot_has(subcats, 'z', '{2}') OR spot_has(subcats, 'a', '{5,11}') THEN 7020
WHEN spot_has(subcats, 'z', '{4}') THEN NULL
ELSE CASE WHEN spot_has(subcats, 'a', '{15}') THEN 2045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 2040 ELSE 2030 END
END
WHEN 1 THEN CASE WHEN spot_has(subcats, 'z', '{3}') THEN 3030 WHEN spot_has(subcats, 'a', '{2,4,7,8}') OR spot_has(subcats, 'c', '{8}') THEN 3040 ELSE 3010 END
WHEN 2 THEN CASE WHEN spot_has(subcats, 'a', '{0,1,2}') THEN 4050 ELSE 1000 END
WHEN 3 THEN 4000
END
$$`)
this.schema.raw(`
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
NEW.category := release_category_for(NEW.name, NEW.inner_name, NEW.groups, CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
RETURN NEW;
END
$$`)
// ACCESS EXCLUSIVE for a moment, while the workers write: a few tries instead of a queue behind them.
this.schema.raw(`
DO $$
BEGIN
FOR attempt IN 1..30 LOOP
BEGIN
PERFORM set_config('lock_timeout', '5s', true);
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'releases' AND column_name = 'category' AND is_generated = 'ALWAYS') THEN
ALTER TABLE releases ALTER COLUMN category DROP EXPRESSION;
END IF;
DROP TRIGGER IF EXISTS releases_category ON releases;
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, groups ON releases
FOR EACH ROW EXECUTE FUNCTION releases_set_category();
RETURN;
EXCEPTION WHEN lock_not_available THEN
PERFORM pg_sleep(2);
END;
END LOOP;
RAISE EXCEPTION 'releases: no lock for the category trigger in 30 tries';
END
$$`)
}
async down() {
this.schema.raw(`DROP TRIGGER IF EXISTS releases_category ON releases`)
this.schema.raw(`DROP FUNCTION IF EXISTS releases_set_category()`)
this.schema.raw(`DROP FUNCTION IF EXISTS spot_category(integer, jsonb)`)
this.schema.raw(`DROP FUNCTION IF EXISTS spot_has(jsonb, text, integer[])`)
this.schema.raw(`DROP FUNCTION IF EXISTS release_category_for(text, text, jsonb, integer)`)
this.schema.raw(`DROP FUNCTION IF EXISTS release_group_category(jsonb)`)
}
}
database/migrations/1770000006000_voyager3_catalog.ts
-177
View File
@@ -1,177 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
import env from '#start/env'
/**
* Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
* place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
* inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
* groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
* with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
* - posters get an id; group_sets holds each distinct group combination once (~140k); releases is created anew
* without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
* columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
* (null: a tail release still provisional). The category and judge triggers follow the new columns;
* releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts
* (migration 6200), which holds the name and every searchable string learned later.
* - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
* release_final() goes: a release is final once finished.
* - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
* - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
* release, not by search.
*/
export default class extends BaseSchema {
async up() {
this.defer(async (db) => {
const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
throw new Error(
`The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
)
}
})
this.schema.raw(`
TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
// Spots matched to releases that are gone: the spot inspector matches them again.
this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)
this.schema.raw(`DROP TABLE IF EXISTS fragments`)
this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
this.schema.raw(
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
)
this.schema.raw(
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
)
// Posters by id (a release's poster_id); searched by trigram as before (poster_search).
this.schema.raw(`DROP TABLE IF EXISTS posters`)
this.schema.raw(`
CREATE TABLE posters (
id serial PRIMARY KEY,
poster text NOT NULL UNIQUE
)`)
this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
// Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
this.schema.raw(`
CREATE TABLE group_sets (
id serial PRIMARY KEY,
groups jsonb NOT NULL UNIQUE
)`)
this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
this.schema.raw(`
CREATE TABLE releases (
id bigserial PRIMARY KEY,
-- The upload's posting window (first and last article's Date).
posted_at timestamptz,
last_posted_at timestamptz,
first_article bigint,
last_article bigint,
segments bigint NOT NULL DEFAULT 0,
bytes bigint NOT NULL DEFAULT 0,
-- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
settled_at timestamptz,
inspected_at timestamptz,
first_seen_at timestamptz NOT NULL DEFAULT now(),
-- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
group_id integer,
group_set_id integer NOT NULL,
poster_id integer,
file_total integer,
files integer NOT NULL DEFAULT 0,
chunks integer NOT NULL DEFAULT 0,
category integer,
-- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
source smallint NOT NULL,
readable boolean NOT NULL DEFAULT false,
has_par2 boolean NOT NULL DEFAULT false,
complete boolean,
passworded boolean,
sidecar_only boolean,
-- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
edge boolean NOT NULL DEFAULT false,
-- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
release_key bytea NOT NULL,
identity_key bytea NOT NULL,
name text NOT NULL,
inner_name varchar(255)
)`)
this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
// No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more.
this.schema.raw(
`CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
)
this.schema.raw(
`CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
)
this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
this.schema.raw(
`CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
)
this.schema.raw(
`CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
)
// The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
this.schema.raw(`
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
RETURN NEW;
END $$`)
this.schema.raw(`
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
// The judge's dirty marks (migration 4900), on the columns its rules read.
this.schema.raw(`
CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
this.schema.raw(`
CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
this.schema.raw(`
CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
OR old.complete IS DISTINCT FROM new.complete)
EXECUTE FUNCTION judge_dirty_releases()`)
// Chunks, rule counts and samples by the block that wrote them.
this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
this.schema.raw(
`ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
)
this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
this.schema.raw(
`ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
)
// Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
const hdd = env.get('CHUNK_TABLESPACE')
if (hdd) {
for (const table of ['release_inspections', 'probe_candidates']) {
this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
this.schema.raw(`
DO $$ DECLARE i record; BEGIN
FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
END LOOP;
END $$`)
}
}
}
async down() {
throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
}
}
database/migrations/1770000006100_blocks.ts
-61
View File
@@ -1,61 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive
* ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high),
* read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the
* prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and
* counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE blocks (
id bigserial PRIMARY KEY,
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
-- The first range's start and the last range's end.
start_id bigint NOT NULL,
end_id bigint NOT NULL,
-- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were).
next_id bigint NOT NULL,
-- The posting time the block covers: estimated when planned, the worker's measured span once done.
posted_low timestamptz,
posted_high timestamptz,
-- planned, leased, done, failed.
status varchar(8) NOT NULL DEFAULT 'planned',
-- As ranges': the tail before everything, then the history by era.
priority integer NOT NULL DEFAULT 0,
node_id integer,
attempts integer NOT NULL DEFAULT 0,
rule_version integer,
headers bigint NOT NULL DEFAULT 0,
releases integer NOT NULL DEFAULT 0,
leased_at timestamptz,
heartbeat_at timestamptz,
done_at timestamptz,
error text,
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (group_id, start_id)
)`)
this.schema.raw(
`CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')`
)
this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`)
this.schema.raw(
`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL`
)
this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`)
// The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture,
// est_date, stored_at, plain_chunks and the measured posted_low/high stay.
this.schema.raw(`
UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0,
spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL,
rule_version = NULL
WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`)
}
async down() {
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`)
this.schema.raw(`DROP TABLE IF EXISTS blocks`)
}
}
database/migrations/1770000006200_release_texts.ts
-35
View File
@@ -1,35 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* release_texts (REBUILD.md, "Voyager 3 … Schema"): every string search matches against, one table, one trigram
* index. The release name goes in (kind 1) with whatever anyone learns later: the prober's file names inside an
* archive, NFO titles, par2 set names, media titles (lib/release/text_kind.ts). Appending is an insert, never an
* update of the wide releases row; a repeat insert is a no-op by the primary key; the judge's dirty mark follows
* (judge_dirty_by_release_id, migration 4900), so a rule can react to a name arriving later. On the NVMe: search
* reads it. releases keeps name and inner_name for display, without trigram indexes (migration 6000).
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE release_texts (
release_id bigint NOT NULL,
-- lib/release/text_kind.ts: 1 name, 2 file, 3 nfo, 4 par2, 5 media.
kind smallint NOT NULL,
text varchar(255) NOT NULL,
PRIMARY KEY (release_id, kind, text)
)`)
this.schema.raw(
`CREATE INDEX release_texts_trgm ON release_texts USING gin (text gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_texts REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
this.schema.raw(
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON release_texts REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
)
}
async down() {
this.schema.raw(`DROP TABLE IF EXISTS release_texts`)
}
}
database/migrations/1770000006300_pending_releases.ts
-60
View File
@@ -1,60 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
import env from '#start/env'
/**
* pending_releases (REBUILD.md decision f, 2026-10-26): nothing enters the catalog without a usable name. A release
* the window closed on whose files all have random names is written here instead of releases: its keys (as the
* catalog would store them), its counts and article span, the upload tool from its message-id (with the group its
* signature), and its probe targets by effort (the edge's candidates: a par2, an archive start, a lone file, a 7z
* tail), no chunk. The pending ladder probes them cheapest first and moves a named one into the catalog, reading its
* segments again from the corpus or the server; a signature that names nothing is discarded. On CHUNK_TABLESPACE
* (the HDD): written once, read by the ladder, never by search.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE pending_releases (
-- lib/release/keys.ts: the identity, the group and the first article; the identity alone.
release_key bytea PRIMARY KEY,
identity_key bytea NOT NULL,
group_id integer NOT NULL,
-- The stem or post the edge keyed it by (a random name: no release name, but the inner-stem merge's handle).
name varchar(255) NOT NULL,
poster text,
posted_at timestamptz,
last_posted_at timestamptz,
first_article bigint NOT NULL,
last_article bigint NOT NULL,
files integer NOT NULL DEFAULT 0,
segments bigint NOT NULL DEFAULT 0,
bytes bigint NOT NULL DEFAULT 0,
-- A file misses parts.
partial boolean NOT NULL DEFAULT false,
-- False: a tail release still open; a later chunk of its key adds its counts up.
finished boolean NOT NULL DEFAULT true,
-- lib/rules uploadTool: ngPost, nyuu, hex32, random, other, …
tool varchar(16) NOT NULL,
-- [{kind, messageId, file}] by effort (lib/edge ProbeCandidate); empty: nothing to read, only a signature.
candidates jsonb NOT NULL DEFAULT '[]',
-- 0 waiting, 1 named (in the catalog), 2 discarded (its signature's verdict, or every probe gave nothing).
status smallint NOT NULL DEFAULT 0,
tries smallint NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL DEFAULT now(),
probed_at timestamptz
)`)
// The ladder's order: a group's and tool's waiting releases, oldest first.
this.schema.raw(
`CREATE INDEX pending_releases_ladder_idx ON pending_releases (group_id, tool, status, created_at)`
)
const hdd = env.get('CHUNK_TABLESPACE')
if (hdd) {
this.schema.raw(`ALTER TABLE pending_releases SET TABLESPACE ${hdd}`)
this.schema.raw(`ALTER INDEX pending_releases_pkey SET TABLESPACE ${hdd}`)
this.schema.raw(`ALTER INDEX pending_releases_ladder_idx SET TABLESPACE ${hdd}`)
}
}
async down() {
this.schema.raw(`DROP TABLE IF EXISTS pending_releases`)
}
}
database/migrations/1770000006400_pending_ladder.ts
-49
View File
@@ -1,49 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* The pending ladder (REBUILD.md decision f, 2026-10-26): what the prober needs to work pending_releases cheapest
* first and to learn which signatures never name anything.
* - pending_releases: `file_total` (the counter's, for the catalog row once named), `inspections` (what each probe
* showed, by kind), `named` (the usable name a probe found; status 4 until the namer moved it into the catalog).
* Status: 0 waiting, 1 named and in the catalog, 2 discarded (its signature's verdict), 3 exhausted (every
* target probed, no name), 4 named, awaiting the move, 9 taken by a running prober.
* - pending_signatures: per (tool, group, poster) how many pending releases were probed to the end and how many
* got a name; `dropped` once probed ≥ PENDING_MIN_PROBES and named ≤ PENDING_MAX_NAMED of them (voyager 2's
* garbage learner, LOG.md 2026-10-08 (5)). The poster '' row is the tool's and group's together.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS file_total integer`)
this.schema.raw(
`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS inspections jsonb NOT NULL DEFAULT '[]'`
)
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS named varchar(255)`)
// The ladder's claim: the oldest waiting ones; the namer's: the named ones.
this.schema.raw(
`CREATE INDEX IF NOT EXISTS pending_releases_waiting_idx ON pending_releases (created_at) WHERE status = 0`
)
this.schema.raw(
`CREATE INDEX IF NOT EXISTS pending_releases_naming_idx ON pending_releases (probed_at) WHERE status = 4`
)
this.schema.raw(`
CREATE TABLE pending_signatures (
tool varchar(16) NOT NULL,
group_id integer NOT NULL,
poster text NOT NULL DEFAULT '',
probed integer NOT NULL DEFAULT 0,
named integer NOT NULL DEFAULT 0,
dropped boolean NOT NULL DEFAULT false,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (tool, group_id, poster)
)`)
}
async down() {
this.schema.raw(`DROP TABLE IF EXISTS pending_signatures`)
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_naming_idx`)
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_waiting_idx`)
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS named`)
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS inspections`)
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS file_total`)
}
}
database/migrations/1770000006500_group_leases.ts
-33
View File
@@ -1,33 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5; 2026-10-26): a worker thread holds a group, not
* a range, and indexes its ranges in article order through one carried window. One row a group: the holder and
* its heartbeat, the range it works, and the snapshot of its window (the article it stands at: the next one to
* index) that the coordinator keeps on bulk. Ranges stay what is planned, fetched and finished.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(`
CREATE TABLE group_leases (
group_id integer PRIMARY KEY REFERENCES groups (id) ON DELETE CASCADE,
-- The holder; null between holders. A holder quiet for LEASE_TTL_SECONDS is taken over.
node_id integer,
leased_at timestamptz,
heartbeat_at timestamptz,
-- The range the holder works (null between ranges).
range_id bigint,
-- The window's snapshot: the article it stands at (a resume point), and when it was taken; null: none.
snapshot_next bigint,
snapshot_at timestamptz,
ranges_done integer NOT NULL DEFAULT 0,
headers bigint NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL DEFAULT now()
)`)
this.schema.raw(`CREATE INDEX group_leases_held_idx ON group_leases (heartbeat_at) WHERE node_id IS NOT NULL`)
}
async down() {
this.schema.raw(`DROP TABLE IF EXISTS group_leases`)
}
}
database/migrations/1770000006600_group_lease_lanes.ts
-27
View File
@@ -1,27 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Lease lanes (REBUILD.md, "Voyager 3: window indexing", 1, amended 2026-10-27): a group has two cursors, each with
* its own holder, window and snapshot. The `history` lane works the ranges the lease bounds hand out (the windows,
* the history), the `tail` lane the tail planner's (priority at or below the tail's): a thread on the tail never
* waits behind a group's history, and a held history group's tail isn't stuck behind its days of ranges.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(
`ALTER TABLE group_leases ADD COLUMN lane varchar(8) NOT NULL DEFAULT 'history'`
)
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id, lane)`)
// Counts the leases handed out: odd and even ones start from the other half of the order's mix.
this.schema.raw(`CREATE SEQUENCE group_lease_turns`)
}
async down() {
this.schema.raw(`DROP SEQUENCE IF EXISTS group_lease_turns`)
this.schema.raw(`DELETE FROM group_leases WHERE lane <> 'history'`)
this.schema.raw(`ALTER TABLE group_leases DROP CONSTRAINT group_leases_pkey`)
this.schema.raw(`ALTER TABLE group_leases DROP COLUMN lane`)
this.schema.raw(`ALTER TABLE group_leases ADD PRIMARY KEY (group_id)`)
}
}
database/migrations/1770000006700_ancient_ranges.ts
-20
View File
@@ -1,20 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* One "ancient" range a group, articles 1 … (its first range − 1): what had expired on the server before the
* group was planned (the planner cuts from the low watermark). Status 'ancient': nothing to fetch, never leased,
* so the tiling is complete from article 1 and the span below the low watermark is on record (user, 2026-10-28).
* Planner.historyRows adds it for groups planned from now on.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(
`INSERT INTO ranges (group_id, start_id, end_id, next_id, priority, status)
SELECT group_id, 1, min(start_id) - 1, 1, 0, 'ancient' FROM ranges GROUP BY group_id HAVING min(start_id) > 1`
)
}
async down() {
this.schema.raw(`DELETE FROM ranges WHERE status = 'ancient'`)
}
}
database/migrations/1770000006800_pending_classes.ts
-57
View File
@@ -1,57 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* The pending ladder by class (REBUILD.md decision f; LOG.md (130)): a pending release's class (lib/release/classes.ts:
* size bucket × name shape × partial, 1..70; 0 for rows from before) and the bytes of its first target, so the
* claim takes the classes with the best record first and the cheapest article within one; pending_classes per
* (tool, group, class) how many climbed to the end and how many got a name, `waiting` how many wait (kept by the
* batch store, the claim and the discard; recounted by pending:classes), `dropped` as pending_signatures.
* The SQL pending_class() is the TypeScript's pendingClass, for the batch store's insert and the backfill.
*/
export default class extends BaseSchema {
static disableTransactions = true
async up() {
this.schema.raw(`
CREATE OR REPLACE FUNCTION pending_class(name text, bytes bigint, partial boolean) RETURNS smallint LANGUAGE sql IMMUTABLE AS $$
SELECT ((CASE WHEN bytes < 1000000 THEN 0 WHEN bytes < 10000000 THEN 1 WHEN bytes < 100000000 THEN 2 WHEN bytes < 1000000000 THEN 3 ELSE 4 END) * 14
+ (CASE WHEN s ~ '^[0-9a-fA-F]{16,}$' THEN 1
WHEN s ~ '^[0-9]{6,}$' THEN 2
WHEN s ~ '^[a-z2-7]{20,}$' AND s ~ '[2-7]' THEN 3
WHEN s ~ '^[A-Za-z]+$' THEN 4
WHEN s ~ '^[A-Za-z0-9]+$' THEN 5
WHEN s ~ '^[A-Za-z0-9._-]+$' THEN 6
ELSE 7 END - 1) * 2
+ (CASE WHEN partial THEN 1 ELSE 0 END) + 1)::smallint
FROM (SELECT regexp_replace(name, '\\.[A-Za-z0-9]{1,4}$', '') AS s) x
$$`)
this.schema.raw(
`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS klass smallint NOT NULL DEFAULT 0`
)
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS target_bytes bigint`)
this.schema.raw(`
CREATE TABLE IF NOT EXISTS pending_classes (
tool varchar(16) NOT NULL,
group_id integer NOT NULL,
klass smallint NOT NULL,
probed integer NOT NULL DEFAULT 0,
named integer NOT NULL DEFAULT 0,
waiting integer NOT NULL DEFAULT 0,
dropped boolean NOT NULL DEFAULT false,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (tool, group_id, klass)
)`)
// The claim by class: the cheapest target first within one, the oldest among equals.
this.schema.raw(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_class_idx ON pending_releases (tool, group_id, klass, target_bytes, created_at) WHERE status = 0`
)
}
async down() {
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_class_idx`)
this.schema.raw(`DROP TABLE IF EXISTS pending_classes`)
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS target_bytes`)
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS klass`)
this.schema.raw(`DROP FUNCTION IF EXISTS pending_class(text, bigint, boolean)`)
}
}
database/schema.ts
+41 -1
View File
@@ -87,6 +87,25 @@ export class GroupSchema extends BaseModel {
declare tail: boolean
}
export class InviteSchema extends BaseModel {
static $columns = ['code', 'createdAt', 'note', 'tier', 'tierDays', 'usedAt', 'usedBy'] as const
$columns = InviteSchema.$columns
@column({ isPrimary: true })
declare code: string
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column()
declare note: string | null
@column()
declare tier: string | null
@column()
declare tierDays: number | null
@column.dateTime()
declare usedAt: DateTime | null
@column()
declare usedBy: number | null
}
export class NodeAccessTokenSchema extends BaseModel {
static $columns = ['abilities', 'createdAt', 'expiresAt', 'hash', 'id', 'lastUsedAt', 'name', 'tokenableId', 'type', 'updatedAt'] as const
$columns = NodeAccessTokenSchema.$columns
@@ -399,9 +418,24 @@ export class RuleSetSchema extends BaseModel {
declare version: number
}
export class UsageSchema extends BaseModel {
static $columns = ['apiHits', 'day', 'grabs', 'userId'] as const
$columns = UsageSchema.$columns
@column()
declare apiHits: number
@column.date()
declare day: DateTime
@column()
declare grabs: number
@column({ isPrimary: true })
declare userId: number
}
export class UserSchema extends BaseModel {
static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
static $columns = ['apiKey', 'createdAt', 'email', 'fullName', 'id', 'password', 'suspended', 'tier', 'tierUntil', 'updatedAt'] as const
$columns = UserSchema.$columns
@column()
declare apiKey: string
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column()
@@ -412,6 +446,12 @@ export class UserSchema extends BaseModel {
declare id: number
@column({ serializeAs: null })
declare password: string
@column()
declare suspended: boolean
@column()
declare tier: string
@column.dateTime()
declare tierUntil: DateTime | null
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime | null
}
deploy/compose.yaml
+8 -36
View File
@@ -115,26 +115,18 @@ services:
<<: *app-env
NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38}
# Phase 2's prober: reads the probe queue's articles over connections of its
# own. The account has 50 (NNTP_ACCOUNT_CONNECTIONS): 5 reserved (the
# coordinator's plain one, the spotter's and the spot inspector's 1 each, 2
# to spare: a hedged request closes its connection and opens another; with
# 10 the spot inspector got 502 Too many connections), the other 45 shared
# by the coordinator (NNTP_MIN_CONNECTIONS–NNTP_CONNECTIONS) and the prober
# (PROBE_MIN_CONNECTIONS–PROBE_CONNECTIONS), headers first
# (app/services/nntp_shares.ts).
# Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
# connections of its own. The account has 50: the coordinator's 38 (+ its
# plain one), the prober's 8, the spotter's and the spot inspector's 1 each,
# 2 to spare (a hedged request closes its connection and opens another)
# (with 10 the spot inspector got 502 Too many connections).
prober:
<<: *app
command: ['node', 'ace', 'probe:run']
# The pending namer reads a named release's articles again from the corpus (no spool).
volumes:
- ./corpus:/corpus:ro
volumes: []
environment:
<<: *app-env
PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8}
# No claims under this much free on the NVMe (its overlay root): 2 GB above the 4 GB floor (2026-10-24), for
# what the tail, the merger and the judge write meanwhile.
PROBE_MIN_FREE_GB: ${PROBE_MIN_FREE_GB:-6}
# The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP.
merger:
@@ -142,8 +134,8 @@ services:
command: ['node', 'ace', 'releases:merge']
volumes: []
# The junk judge: releases judged by what their probes showed (dead, fake, repeated, locked, unusable), the changed
# ones every 2 minutes, all daily; verdicts hide them from search. No NNTP.
# The junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable);
# verdicts hide them from search. No NNTP.
judge:
<<: *app
command: ['node', 'ace', 'releases:judge']
@@ -175,11 +167,6 @@ services:
environment:
COORDINATOR_URL: http://node-api:3335
NODE_TOKEN: ${NODE_TOKEN}
# The window worker (REBUILD.md design 1 and 5): a group per thread, the window's snapshot to the coordinator
# this often; the coordinator keeps them under SNAPSHOT_DIR (the spool's snapshots/ on bulk by default).
EDGE_SNAPSHOT_MINUTES: ${EDGE_SNAPSHOT_MINUTES:-30}
# Of the threads, this many on the groups' tail lane (the live tail; the rest work the history lane).
EDGE_TAIL_THREADS: ${EDGE_TAIL_THREADS:-1}
stop_grace_period: 60s
depends_on:
node-api:
@@ -197,21 +184,6 @@ services:
node-api:
condition: service_started
# The replay harness into the scratch catalog usenet_replay on bulk (REBUILD.md, "The replay harness"):
# docker compose run --rm --no-deps replay --group alt.binaries.ath
replay:
<<: *app
profiles: [replay]
restart: 'no'
entrypoint: ['node', 'ace', 'replay:corpus']
command: ['--help']
environment:
<<: *app-env
DB_DATABASE: usenet_replay
volumes:
- ./corpus:/corpus:ro
depends_on: []
# One-off ace commands: docker compose run --rm ace releases:duplicates
ace:
<<: *app
deploy/deploy.sh
+1 -8
View File
@@ -51,14 +51,7 @@ if [ -e "$CACHE/paused" ]; then
docker compose up -d --no-deps node-api web
else
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
if [ -e "$CACHE/no-edge" ]; then
# Voyager 3 (2026-10-26, LOG.md (116)): the worker indexes range by range, without the carried window, and
# would split releases at every range edge with nothing to join them; history comes from the corpus through
# deploy/replay-window.sh until the window worker is built. Remove the marker to start it again.
echo "== no-edge: the edge worker left stopped"
else
docker compose up -d --no-deps edge
fi
docker compose up -d --no-deps edge
fi
echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
deploy/replay-window.sh
-74
View File
@@ -1,74 +0,0 @@
#!/bin/bash
# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
# from the deployment root:
# setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
# DB=usenet_rebuild replays into the catalog itself (voyager 3 deployed, 2026-10-26: history comes from the corpus
# this way until the window worker is built); the harness then needs REPLAY_INTO_CATALOG=1, passed here. OUT keeps
# a run's logs and markers apart (.cache/replay-catalog for the catalog's).
set -u
ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
PRIORITY=${PRIORITY:-0}
PARALLEL=${PARALLEL:-4}
MEMORY=${MEMORY:-12g}
IMAGE=${IMAGE:-usenet-voyager3:latest}
DB=${DB:-usenet_replay}
OUT=${OUT:-$ROOT/.cache/replay}
mkdir -p "$OUT"
cd "$ROOT" || exit 1
say() { echo "$(date -Is) $*"; }
catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }
# The window's groups: name, first article, last article, whether every range is stored; smallest first.
window() {
catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
FROM ranges r JOIN groups g ON g.id = r.group_id
WHERE r.capture AND r.priority = $PRIORITY
GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
}
replay() {
local group=$1 from=$2 to=$3
say "$group: replaying $from–$to"
if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
-e NODE_ENV=production -e HOST=0.0.0.0 -e PORT=3334 -e APP_URL=http://web:3334 -e SPOOL_DIR=/spool -e CORPUS_DIR=/corpus \
-e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE="$DB" -e CHUNK_TABLESPACE=pg_default -e REPLAY_INTO_CATALOG=1 \
-v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
touch "$OUT/$group.done"
say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
else
touch "$OUT/$group.failed"
say "$group: FAILED (see $OUT/$group.log)"
fi
rm -f "$OUT/$group.running"
}
while true; do
todo=0
while read -r group from to complete; do
[ -n "$group" ] || continue
[ -e "$OUT/$group.done" ] && continue
[ -e "$OUT/$group.failed" ] && continue
todo=$((todo + 1))
[ "$complete" = 1 ] || continue
[ -e "$OUT/$group.running" ] && continue
[ -e "$ROOT/.cache/paused" ] && continue
while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
touch "$OUT/$group.running"
replay "$group" "$from" "$to" &
sleep 5
done <<< "$(window)"
if [ "$todo" = 0 ]; then
wait
say "every group of the window at priority $PRIORITY is replayed"
exit 0
fi
sleep 300
done
deploy/usenet-rebuild-resume.sh
+1 -6
View File
@@ -7,10 +7,5 @@ flock 9
rm -f .cache/paused
date "+%F %T resuming"
docker compose up -d --no-deps coordinator prober merger judge spotter spot-inspector
if [ -e .cache/no-edge ]; then
# Voyager 3 without its window worker yet (deploy.sh, LOG.md (116)): the edge stays stopped.
echo "no-edge: the edge worker left stopped"
else
docker compose up -d --no-deps edge
fi
docker compose up -d --no-deps edge
docker compose ps --format "{{.Service}} {{.Status}}"
edge/Cargo.lock
Generated
-10
View File
@@ -44,15 +44,6 @@ version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@@ -154,7 +145,6 @@ name = "edge"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"bincode",
"chrono",
"flate2",
"libdeflater",
edge/Cargo.toml
+1 -2
View File
@@ -2,7 +2,7 @@
name = "edge"
version = "0.1.0"
edition = "2021"
description = "The edge worker (edge-node) and its benchmark (edge-bench): voyager 3's window indexing in Rust"
description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)"
[[bin]]
name = "edge-bench"
@@ -14,7 +14,6 @@ path = "src/bin/edge-node.rs"
[dependencies]
base64 = "0.22"
bincode = "1.3"
chrono = "0.4"
flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
libdeflater = "1.26.1"
edge/README.md
+18 -74
View File
@@ -6,21 +6,18 @@ from the node API, reads their prefetched XOVER chunks, finds releases and
posts batches. `edge-bench` (`src/main.rs`) replays a captured range through
the same indexing (`src/lib.rs`) for timing and for comparing.
Since voyager 3 (2026-10-25) the Rust indexer is the one implementation of
the window (REBUILD.md, "Voyager 3: window indexing"; `src/indexer.rs`):
files and releases are finished when the frontier (the median Date of the
last 1,024 plausibly dated articles) is 24 h past their last part, or at
the block's end (provisionally, as edges), and written once. The
TypeScript `RangeIndexer` (`lib/edge`) keeps the range path for the corpus
scanner and the tests; the field-by-field comparison (`check.ts`) went
with the parity. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs
`cargo test` and `bench-fixtures.sh`, which replays the committed captures
in `fixtures/` (25k overview lines each of a.b.ath, a.b.boneless, a.b.fta
and a.b.models, in 5 chunks).
The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the
reference: `check.ts` runs both on a capture and compares every output field
by field. An identity or rule change goes into both and has to come out
identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test`
and `check-fixtures.sh`, which compares both on the committed captures in
`fixtures/` (25k overview lines each of a.b.models, a.b.ath and
a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they
match.
```sh
cargo test --release --manifest-path edge/Cargo.toml
edge/bench-fixtures.sh # from the repository root
edge/check-fixtures.sh # from the repository root
```
In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores
@@ -113,30 +110,11 @@ files, segments, payload size, candidates, fragments and samples. The
summary can hide compensating differences, so compare the outputs
themselves.
## Replaying a group through one window
A corpus group directory (`<corpus>/<group>/<start>-<end>/`, each range with
its `range.json`) is replayed in article order with the window carried from
range to range, the end finished after the last range (REBUILD.md, "The
replay harness"):
```sh
./target/release/edge-bench --group /corpus/alt.binaries.ath --rules rules.json --batch-chunks 5 --wire - > batches.jsonl
# --from <article> skips ranges ending before it, --to <article> ranges starting after it (one window of the
# group's history), --limit <n> stops after n ranges, --no-finish leaves the window open
```
`--rules` takes the lease's rules JSON (`src/lease.rs`: the catalog's rule
set and group priorities, as `replay:corpus` writes it); without it the
defaults with only the group indexed apply. `--wire -` writes each batch as
the worker posts it to stdout, one JSON per line, and the report to stderr;
`node ace replay:corpus` reads them into the batch store. Each range end
prints the frontier, the releases out and the releases and files still open.
## The dump
## Comparing the output
```sh
./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl
cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl
```
`--dump` writes every output of every batch as JSON lines:
@@ -234,45 +212,17 @@ them:
```sh
cargo build --release
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--tail-threads T] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
```
- **Two lanes** (2026-10-27): `--tail-threads T` (EDGE_TAIL_THREADS; one of
two or more threads by default) puts that many threads on the groups'
*tail* lane (`?lane=tail` on the group routes): the tail planner's
ranges, through a window and snapshot of their own
(`<group>-tail.gz`). The other threads work the history lane: the
windows and the history the coordinator's bounds hand out. A tail thread
never wades through a group's history first, and a held history group's
tail isn't stuck behind its days of ranges; the coordinator orders each
lane's leases by alternating the group with the most planned articles
and the longest unheld one.
- **One group per thread** (default one per core; REBUILD.md, "Voyager 3:
window indexing", 1 and 5): a thread leases a group (`POST
groups/lease`), takes its ranges in article order (`POST
groups/:id/next`), streams each range's chunks (`GET ranges/:id/stream`:
the corpus or the spool, the worker can't tell) through one carried
window, posts a batch at least at every range end and finishes the
range; one thread reads the stream and inflates a frame ahead, another
posts the batches in order, the indexer goes on meanwhile. The window's
snapshot (bincode, gzip) goes to the coordinator every
`--snapshot-minutes` (EDGE_SNAPSHOT_MINUTES, 30) right after a batch,
and when the group is given back: caught up (`next` answers 204: the
tail), a stop, a lost lease or a failure; a stop or a failed stream
posts the open batch first, so the snapshot stands at its resume point
(a batch the coordinator refused can't, and that window is lost); the
next holder restores it (`GET groups/:id/snapshot`) and continues at
its article. `--groups N`
stops after N groups; `--ranges N` is the old per-range lease
(`POST lease`), one range per thread without a carried window.
- **One range per thread** (default one per core): one thread reads the
coordinator's stream and inflates a frame ahead, another posts the
batches in order (one may wait), the indexer goes on meanwhile.
- **As `commands/edge_node.ts`:** retries for network trouble and 5xx;
409 means the lease is lost (the range stops, isn't given back); a
batch once 500k articles or 100k files wait (a picture group's batches
went over the server's 256 MB limit); a chunk that doesn't inflate is
reported (the coordinator fetches it plain next time) after the batch
before it went out, and a held group asks for the range again from
that chunk, its window kept; SIGTERM/SIGINT
reported (the coordinator fetches it plain next time); SIGTERM/SIGINT
give each range back after its current chunk; `--max-mbit` is the
node's download cap, shared by its threads. Plus a heartbeat when no
batch went out for a minute.
@@ -284,11 +234,5 @@ COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-nod
same (releases with every column, chunks, candidates, rule counts,
fragments with their JSON).
- **On homelab:** compose service `edge` (deploy/compose.yaml), the
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8),
EDGE_TAIL_THREADS (1); it talks to `node-api`.
- **End to end:** `tests/e2e/edge_node.spec.ts` runs this binary
(`edge/target/release/edge-node`, or EDGE_NODE_BIN) against the test
server, database, spool and corpus: a chunk the corpus fill stores while
the stream waits, a failed stream and the restart that restores the
snapshot, a restart whose snapshot file is gone, a takeover. `node ace
test e2e` after `cargo build --release`; CI builds the binary first.
`edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8);
it talks to `node-api`.
edge/bench-fixtures.sh
-11
View File
@@ -1,11 +0,0 @@
#!/bin/sh
# The Rust indexer over the committed captures (edge/fixtures: 25k overview lines each of a.b.ath, a.b.boneless,
# a.b.fta and a.b.models, in 5 chunks): a smoke test of the window on real headers, exiting non-zero when the bench
# fails on one. CI runs it after cargo test; run it from the repository root after a change to the indexer.
set -eu
cd "$(dirname "$0")/.."
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
for capture in edge/fixtures/*/; do
echo "== $(basename "$capture")"
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 | tail -1
done
edge/check-fixtures.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k
# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared
# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from
# the repository root after any change to either side.
set -eu
cd "$(dirname "$0")/.."
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
out=$(mktemp -d)
trap 'rm -rf "$out"' EXIT
status=0
for capture in edge/fixtures/*/; do
name=$(basename "$capture")
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1
echo "== $name"
node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1
done
exit $status
edge/check.ts
+139
View File
@@ -0,0 +1,139 @@
/**
* Compares edge-bench's output (--dump) with the TypeScript RangeIndexer
* on the same captured range, item by item: releases (every field, and the
* payload decoded with lib/chunks), samples, probe candidates, rule counts,
* fragments and batch boundaries.
*
* node --import=@poppinss/ts-exec edge/check.ts <dir> <dump.jsonl> [--wire batches.jsonl] [--batch-chunks 10] [--idle 5]
*
* With --wire (edge-bench --wire): also each batch as the Rust worker
* posts it, against toWire of the TypeScript batch, field by field.
*/
import { readFile, readdir } from 'node:fs/promises'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import { isDeepStrictEqual } from 'node:util'
import { RangeIndexer, toWire, type EdgeBatch } from '#lib/edge/index'
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
import { decodeChunk } from '#lib/chunks/index'
const [dir, dumpPath] = process.argv.slice(2)
const flag = (name: string, fallback: number) => {
const i = process.argv.indexOf(name)
return i === -1 ? fallback : Number(process.argv[i + 1])
}
const batchChunks = flag('--batch-chunks', 10)
const idle = flag('--idle', 5)
const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
const batches: EdgeBatch[] = []
for (const [i, file] of files.entries()) {
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
const last = i === files.length - 1
if (last) indexer.finish()
if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
}
/** Canonical lines per kind, in a stable order, from both sides. */
type Item = Record<string, unknown>
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
const payloadBytes = { ts: 0, rs: 0 }
let side: 'ts' | 'rs' = 'ts'
const release = (r: Item, b: number) => {
const { payload, ...rest } = r as Item & { payload: Buffer }
// The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
// differently); what the payload decodes to is.
payloadBytes[side] += payload.length
return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
}
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
batches.forEach((batch, b) => {
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers, postedLow: batch.postedLow, postedHigh: batch.postedHigh })
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
})
const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
side = 'rs'
let b = -1
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
if (!line) continue
const { t, ...item } = JSON.parse(line)
if (t === 'b') b++
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
else rs[t].push({ b, ...item })
}
const keyOf: Record<string, (i: Item) => string> = {
b: (i) => String(i.b),
r: (i) => `${i.b} ${i.key} ${i.upload}`,
s: (i) => `${i.b} ${i.key} ${i.upload}`,
c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
n: (i) => `${i.b} ${i.rule} ${i.key}`,
f: (i) => `${i.b} ${i.fileKey}`,
}
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
let failed = false
for (const kind of Object.keys(keyOf)) {
const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
const missing = [...a.keys()].filter((k) => !z.has(k))
const extra = [...z.keys()].filter((k) => !a.has(k))
const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
failed ||= !ok
console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`)
for (const k of differ.slice(0, 2)) {
const x = a.get(k)!
const y = z.get(k)!
const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
}
}
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
const wireIndex = process.argv.indexOf('--wire')
if (wireIndex !== -1) {
// Order within a batch doesn't count; payloads by what they decode to.
const by = (f: (x: any) => string) => (a: any, b: any) => (f(a) < f(b) ? -1 : f(a) > f(b) ? 1 : 0)
const normal = (w: any) => ({
...w,
releases: w.releases.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
samples: w.samples.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
candidates: [...w.candidates].sort(by((c) => JSON.stringify(c))),
counts: [...w.counts].sort(by((c) => `${c.rule} ${c.key}`)),
fragments: [...w.fragments].sort(by((f) => `${f.fileKey} ${f.segments[0]?.[1]}`)),
})
const rust = (await readFile(process.argv[wireIndex + 1], 'utf8')).split('\n').filter(Boolean).map((l) => normal(JSON.parse(l)))
const typescript = batches.map((b) => normal(JSON.parse(JSON.stringify(toWire(b)))))
let wireSame = rust.length === typescript.length
const firstDiff = (x: any, y: any, path: string): string | null => {
if (isDeepStrictEqual(x, y)) return null
if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null) return `${path}: ${JSON.stringify(x)?.slice(0, 120)} vs ${JSON.stringify(y)?.slice(0, 120)}`
for (const k of new Set([...Object.keys(x), ...Object.keys(y)])) {
const d = firstDiff(x[k], y[k], `${path}.${k}`)
if (d) return d
}
return path
}
for (let i = 0; i < Math.max(rust.length, typescript.length); i++) {
const d = firstDiff(typescript[i], rust[i], `batch ${i}`)
if (d) {
wireSame = false
console.log(`DIFF wire: ${d}`)
break
}
}
if (wireSame) console.log(`same wire batches: ${typescript.length} TypeScript, ${rust.length} Rust`)
failed ||= !wireSame
}
process.exit(failed ? 1 : 0)
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff. Show more