Compare commits
63
Commits
No files matched your search
@@ -83,8 +83,8 @@ jobs:
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
- name: cargo test
|
||||
run: cargo test --release --manifest-path edge/Cargo.toml
|
||||
- name: Rust and TypeScript identical on the fixtures
|
||||
run: edge/check-fixtures.sh
|
||||
- name: The Rust indexer over the fixtures
|
||||
run: edge/bench-fixtures.sh
|
||||
|
||||
deploy:
|
||||
needs: [test, edge]
|
||||
|
||||
+11
-9
@@ -8,6 +8,15 @@
|
||||
# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
|
||||
ARG NODE_VERSION=24.16
|
||||
|
||||
# The Rust worker (before the app: its image carries edge-bench). Registry and target dir are build caches, so a rebuild only compiles what changed.
|
||||
FROM rust:1-bookworm AS edge-build
|
||||
WORKDIR /src
|
||||
COPY edge ./
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release --locked --bin edge-node --bin edge-bench \
|
||||
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
|
||||
|
||||
# The app, compiled, with production dependencies only.
|
||||
FROM node:${NODE_VERSION}-bookworm AS app-build
|
||||
WORKDIR /src
|
||||
@@ -22,19 +31,12 @@ FROM node:${NODE_VERSION}-bookworm-slim AS app
|
||||
ENV NODE_ENV=production
|
||||
WORKDIR /app
|
||||
COPY --from=app-build /src/build ./
|
||||
# The Rust indexer's bench: `node ace replay:corpus` runs a group's corpus through it (the replay harness).
|
||||
COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
|
||||
USER node
|
||||
# Every service picks its own command, e.g. `node ace coordinator:run`.
|
||||
CMD ["node", "ace", "list"]
|
||||
|
||||
# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed.
|
||||
FROM rust:1-bookworm AS edge-build
|
||||
WORKDIR /src
|
||||
COPY edge ./
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release --locked --bin edge-node --bin edge-bench \
|
||||
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
|
||||
|
||||
FROM debian:bookworm-slim AS edge
|
||||
COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
|
||||
# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
|
||||
|
||||
+466
-16
@@ -5,7 +5,8 @@ rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
|
||||
decisions D1–D42 are superseded; its domain model still holds, and its
|
||||
features not yet here are the phase 2 backlog below.
|
||||
|
||||
Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
|
||||
Branch `rebuild`; its successor is designed under *Voyager 3* below (branch
|
||||
`voyager-3`). Phase 1 finds releases from headers and makes their NZBs.
|
||||
Everything learned from articles (names from yEnc headers, par2, archive
|
||||
listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
|
||||
never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
|
||||
@@ -189,13 +190,22 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
number; merged when the neighbouring range is done, then written as a
|
||||
chunk. Never staged per segment.
|
||||
|
||||
8. **Settling**. A release is settled when the ranges covering its posting
|
||||
span are done and no chunk arrived for a day: completeness is computed
|
||||
then (expected vs found files and segments). Phase 2 picks its work from
|
||||
settled releases.
|
||||
8. **Settling**. A release is settled when nothing was added to it for 6
|
||||
hours (30 minutes once its last post is that old: history), and its
|
||||
completeness decided: complete when every file its [i/N] counter
|
||||
announced was found. Without a counter, by its par2 sets once their index
|
||||
is probed (`par2_completeness`): every file a set describes is there by
|
||||
name or by its first 16 KiB, or, renamed, as many files and bytes; decided
|
||||
again after a par2 merge and before the pruner drops a set's file list.
|
||||
Segments aren't compared. Phase 2 picks its work from settled releases.
|
||||
|
||||
9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
|
||||
search (names) and get. Web UI later.
|
||||
9. **NZB and API**. NZB built from a release's chunks, with a known
|
||||
password in its meta; Newznab search (names, inner names) and get; a web
|
||||
interface. Categories (Newznab's) by a trigger on releases (migration
|
||||
5200): from the name, else the inner name the prober found, else a spot's
|
||||
category (spot_category, set by the spot inspector), else the groups
|
||||
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
|
||||
in batches.
|
||||
|
||||
## Group list
|
||||
|
||||
@@ -221,6 +231,39 @@ range with the deployed build's edge-bench (`replay`, ~seconds a range) and
|
||||
reports suspects and the change since the previous build (`report`). A rule
|
||||
change is judged on the whole corpus before a reset.
|
||||
|
||||
Since 2026-10-23 also history fetched ahead of the indexing: while the
|
||||
catalog is over its NVMe cap, ranges marked `capture` (priority >= 0, not
|
||||
leased) are fetched into the corpus by idle loops, at most perGroup /
|
||||
perRange of a group at a time, to be indexed from disk once there's room
|
||||
(LOG.md 2026-10-23 (100)). Stored filtered (`CORPUS_FILTER`, default on):
|
||||
without the lines the edge worker only counts, decided per line as it does
|
||||
(a crosspost another indexed group owns, a subject that names no file part;
|
||||
~48% of headers), the chunk compressed again as zlib; beside it a `.drops`
|
||||
(gzip JSON: rule version, the owners' priorities, per rule the article
|
||||
numbers as differences, crossposts by owner) and `filtered` in range.json.
|
||||
The worker makes the same releases of a filtered chunk; rule_counts lack
|
||||
those two rules' counts. `corpus:filter` filters chunks stored raw. A
|
||||
corpus filtered this way no longer serves a sweep of those two rules.
|
||||
|
||||
Windows by posting time (2026-10-25; voyager 3's test data, LOG.md (104)):
|
||||
`ranges:dates` samples when each active group's articles were posted (one
|
||||
XOVER of 32 articles every 500M articles of the group, 16 … 1000 samples,
|
||||
the median Date of each slice, kept monotone; `group_dates`; over plain
|
||||
connections: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
output, which some small replies begin with, `TruncatedReply`, LOG.md
|
||||
2026-10-25 (105)) and estimates
|
||||
every range's posting date from the curve (`ranges.est_date`; the edges of
|
||||
a window are off by up to one sample interval, three weeks in boneless).
|
||||
`ranges:capture --from 2025-01-01 --to 2026-01-01` marks every active
|
||||
group's ranges posted in the window `capture`: planned ones not started at
|
||||
`--priority` (0), done ones as they are, both fetched into the corpus by the
|
||||
fill, which takes capture ranges in priority order, done ones too (they're
|
||||
fetched again, corpus only), and skips those complete (`ranges.stored_at`,
|
||||
set when it sees the range.json). A year of every group is the data to
|
||||
replay voyager 3's 30-day blocks on: marked 2026-10-25, 2016 (priority 0,
|
||||
30B articles, ~0.56 TB) and 2021 (priority 1, 87B, ~1.6 TB); a recent year
|
||||
is ~200B articles, ~3.6 TB.
|
||||
|
||||
## Phase 2
|
||||
|
||||
Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
|
||||
@@ -238,6 +281,21 @@ files the set describes, into the one with the most files (chunks,
|
||||
candidates, inspections, counts; the par2's shared name when none was
|
||||
readable); `release_merges` keeps what went where. Obfuscated uploads whose
|
||||
files were each a release of their own (`lone` candidates) become one.
|
||||
Readable files have no hash: a name pass (2026-10-23) joins the releases
|
||||
of the set's poster in its group posted within 12 hours whose files the
|
||||
set all names (no case or punctuation; sidecars, par2 and images aside),
|
||||
as `via = 'name'`: an album's tracks and its index, an index beside its
|
||||
differently spelled video. `releases:merge-names` for sets from before.
|
||||
Also merged: lone files whose inner names share a random stem (`inner`),
|
||||
and small sidecar-only releases into their upload (`sidecar`).
|
||||
It reads what's new an hour (`created_at`) at a time, its place kept after
|
||||
each, and merges only the sets held or matched by two releases or more; a
|
||||
file whose first 16 KiB and size more than 20 files share is a common one and
|
||||
matches nothing (LOG.md 2026-10-20 (85)).
|
||||
A set's file list is dropped once it can't merge anything more: complete (all
|
||||
its files in its holder), judged (every holder junk), or final (every holder
|
||||
inside a run of done ranges, those ranges' candidates probed, its hashes seen
|
||||
by the merger); `par2_sets` stays (LOG.md 2026-10-20 (87)).
|
||||
|
||||
**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
|
||||
voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
|
||||
@@ -252,7 +310,7 @@ with 68,032 releases with a password):
|
||||
| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
|
||||
| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
|
||||
| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
|
||||
| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed |
|
||||
| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
|
||||
| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
|
||||
| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
|
||||
|
||||
@@ -260,9 +318,11 @@ Live indexing (the tail) runs since 2026-10-09.
|
||||
|
||||
**Junk** (voyager 1's second filter, D42). After the header pass's rules
|
||||
(garbage signatures, executables, per-article and obfuscated subjects), the
|
||||
judge (`releases:judge`, service `judge`) judges releases hourly by what the
|
||||
prober, the merges and the spots found, in this order (the first rule
|
||||
wins): *dead* (every probed first article gone), *fake* (a film, episode
|
||||
judge (`releases:judge`, service `judge`) judges releases by what the
|
||||
prober, the merges and the spots found: every 2 minutes the releases whose
|
||||
facts changed (noted by triggers in `judge_dirty`), daily every release (for
|
||||
what time changes: a day after settling, ranges read beside it). In this
|
||||
order (the first rule wins): *dead* (every probed first article gone), *fake* (a film, episode
|
||||
or music name whose content is a program: archive entries all
|
||||
executables, or the file itself one by name or by its first bytes, sniffed
|
||||
`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
|
||||
@@ -312,6 +372,340 @@ processes, one NNTP connection each:
|
||||
`spot-text`, and the spot), not on the catalog row; the catalog only says
|
||||
`passworded`.
|
||||
|
||||
## Voyager 3: window indexing (design, 2026-10-25)
|
||||
|
||||
Branch `voyager-3` (a push to it only runs the tests; `rebuild` keeps
|
||||
running on homelab until the reset). Goal: the whole history (~1,220 B
|
||||
articles, 3% indexed on 2026-10-25) indexed and searchable on a 4 TB NVMe
|
||||
and the 15 TB HDD. The catalog costs 3.9 B of NVMe per article today
|
||||
(144 GB for 36.5 B), which puts the full history at ~4.7 TB; the target is
|
||||
≤ 2.5 B per article (~3 TB) with room for the WAL and bloat. Sources:
|
||||
LOG.md 2026-10-24 (102).
|
||||
|
||||
What is wrong today: a release is written as soon as one range sees it,
|
||||
updated by every later range that finds more parts, merged afterwards
|
||||
(fragments, par2, names) and settled by wall-clock quiet time, which means
|
||||
little in a backfill. That churn is most of the WAL (0.6 TB a day) and the
|
||||
20 k write IOPS bursts, and leaves bloat. Half the rows are single-article
|
||||
releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
||||
(102)).
|
||||
|
||||
1. **One cursor a group (amended 2026-10-26; was: blocks).** A group's
|
||||
history is still fetched as ranges of 2 M articles in chunks of 100 k
|
||||
(the spool, the corpus, the prefetcher stay) and leased as ranges, but
|
||||
a group's ranges are indexed in article order by one worker at a time,
|
||||
and the window below is *carried* from range to range: the next range
|
||||
finishes what the previous one left open. The first design leased
|
||||
blocks of 30 days of posting time, ran a group's blocks in parallel and
|
||||
joined their edges afterwards (a block merger). The user asked why
|
||||
(LOG.md 2026-10-25 (110)): downloading stays parallel through the
|
||||
prefetcher and a Rust thread parses faster than a connection delivers,
|
||||
so one worker a group loses nothing but the end of a reset, when the
|
||||
last big group runs alone. The merger, the block cutting and the
|
||||
`blocks` lease go. Costs: one seam a group where the history meets the
|
||||
tail (the releases in flight then end as two incomplete halves; history
|
||||
runs forward in one pass a group, priority only between groups), and
|
||||
the carried state must survive a restart (5 below). Measured on six
|
||||
consecutive ranges of a.b.ath (2016): one range is 15 hours of posting,
|
||||
so without the carry no release would finish inside a range (LOG.md
|
||||
2026-10-26 (111)).
|
||||
|
||||
2. **The window.** The worker reads a block in article order and keeps its
|
||||
open files and releases in memory (counters, names, sizes; the segment
|
||||
lists spooled to a file per block on the worker's disk). The *frontier*
|
||||
is the median Date of the last 1,024 articles with a plausible date
|
||||
(within 7 days of the frontier; 1,024 such dates in a row that agree
|
||||
among themselves move it, the group having paused); Date headers alone
|
||||
aren't trusted (clock skew, bogus years). 1,024, not more: the median
|
||||
trails the newest article by half the ring, a day in a group of 500
|
||||
articles a day. A release is **finished** when the frontier is 24 h past
|
||||
its last part (`WINDOW_MS`): written once, with its completeness decided
|
||||
then (every announced file there and no file short of parts; par2
|
||||
decides later for those without a counter), never updated by the header
|
||||
pass again. Incomplete releases are kept (user, 2026-10-24). A fill
|
||||
posted after the window is a release of its own (the judge's `repeated`
|
||||
rule already knows reposts). At a lease's end (the tail's poll, a worker
|
||||
stopping; amended 2026-10-26, was: the block's end) the releases the
|
||||
window hasn't closed on are written **provisionally** (`settled_at`
|
||||
null, `finished: false`, flagged `edge`): a release being uploaded
|
||||
shows in the catalog, and the carried window finishes the same row
|
||||
later, since the release key is the identity, the group and the first
|
||||
article; the store's finish path adds the files not sent before. No
|
||||
merger: the carry does the joining. The settler goes: `settled_at` is
|
||||
the finish. The **tail** needs no path of its own: its lease ends leave
|
||||
such provisional releases, the next lease's continuation finishes them.
|
||||
|
||||
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
||||
only by a number sequence, or share a directory-like prefix, posted
|
||||
within the window, become one release (`source = 'album'`): its name the
|
||||
shared template (or the prefix), its files kept by name so each stays
|
||||
findable; at least two files, else the single stays a release. This is
|
||||
for subjects that name no release (today's `post`/`stem` singles); a
|
||||
poster's ten different cracks in a day stay ten releases. The stake,
|
||||
from voyager 2's final analysis (LOG.md (112)): 77% of its 83.7 M
|
||||
releases had one file, 50% one segment (0.3% of the segments);
|
||||
a.b.pictures.erotica.porn alone was 27.7 M single-file releases of one
|
||||
poster. A counter total shared by many stems of one poster within the
|
||||
window is a batch counter, not a file count (Yenc-PP-A&A's [n/68067]):
|
||||
`file_total` null for them, not 68,067 incomplete singles.
|
||||
**Built 2026-10-26** (edge/src/indexer.rs, LOG.md (113)): a release the
|
||||
window closed on with one file named by its stem or post, the name with
|
||||
a number and no archive, volume, sidecar or random name, waits in the
|
||||
album of its poster and template (the name's digit runs as `#`, the
|
||||
extension apart: `#_#.jpg.data`) posted within WINDOW_MS of the album's
|
||||
first file, at most ALBUM_MAX_FILES (1,000) a album; the album closes a
|
||||
window after that span and is written as one release (`source: album`,
|
||||
no file total) named by the members' shared numbers (`1481439079025_#
|
||||
.jpg.data`), a lone member as the release it was. The batch store writes
|
||||
the members' names as `release_texts` kind 6 (`member`, the first 20).
|
||||
The directory-like prefix variant is not built (no case seen in the
|
||||
corpus samples). The batch counter: a total ≥ 100 under which one poster
|
||||
started 20+ releases within 100 batches gives none of them a file
|
||||
total. On six a.b.ath ranges of 2016: 46 albums of 790 files among
|
||||
2,099 chunks, most of them a subject parser miss (`[n/60] Shogun I
|
||||
(1980) Shogun I (1980).avi.012`: the counter text taken into the name,
|
||||
a volume each) that the album rule glues back together.
|
||||
|
||||
4. **A slimmer catalog.** `releases` loses its text copies: `poster_id` →
|
||||
`posters(id)` (the table gets an id; 33 B → 4), `group_set_id` →
|
||||
`group_sets` (the distinct group combinations: 139,927 in voyager 2's
|
||||
83.7 M releases, LOG.md (112); 85 B jsonb → 4),
|
||||
`release_key` and `identity_key` as 20-byte `bytea` (41 B each → 20),
|
||||
`source` a smallint. ~150 of ~430 B a row; the poster and key indexes
|
||||
shrink with them (~12 GB of 50 today); `releases_category_newest_idx`
|
||||
(5.3 GB, used once) goes, category browsing filters the newest index.
|
||||
Today's 85 GB would be ~55. Cold tables move to the HDD tablespace:
|
||||
`release_inspections` and `probe_candidates` (28 GB, written once, not
|
||||
read by search); `probe_cache` is there already.
|
||||
|
||||
5. **Memory and disk at the edge.** The busiest group posts ~50 M articles
|
||||
a day; at a file per ~50 articles that is ~1 M open files of ~100 B in
|
||||
memory per block, and their segment lists (~25 B each) in the block's
|
||||
spool file, read back when a release is finished. Per worker thread a
|
||||
few hundred MB at worst. The carried window (amended 2026-10-26) has to
|
||||
survive a restart (deploys restart the workers daily): its counters and
|
||||
names are small (kilobytes for most groups, ~100 MB for the busiest),
|
||||
the open files' segment lists are the bulk (~1.25 GB for a day of the
|
||||
busiest group). A snapshot at every 2 M range would be ~600 B/article
|
||||
of writes, so the state is snapshotted at a lease's end and on a timer,
|
||||
to a file on bulk, and a crash redoes the ranges since the last snapshot
|
||||
from the corpus or the prefetch cache: the deterministic keys make the
|
||||
redo idempotent. Not built yet; the replay harness (8) keeps a group's
|
||||
window in one process.
|
||||
|
||||
6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
|
||||
(30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
|
||||
those 73% are random 32-hex tokens, 27% `[PRiVATE]` hashes, ~0.3% posts
|
||||
with a readable name and no counter. The filter (crosspost, counterless)
|
||||
loses nothing voyager 3 would index; albums work from filtered chunks
|
||||
(the image dump's subjects carry "(1/1)"). Indexing without a counter at
|
||||
all (obfuscated uploads, ~31% of all lines) would need bodies, and the
|
||||
raw corpus (2,328 ranges) for a sweep.
|
||||
|
||||
7. **Where it lives.** The window, blocks and albums are in the Rust edge
|
||||
(`edge/`), the batch store, the planner and leases; the TypeScript
|
||||
RangeIndexer keeps the scanner (the corpus filter, tests) and drops its
|
||||
parity with the Rust worker for the window (the `check.ts` comparison
|
||||
covered the range path, which goes). New migrations rebuild the schema
|
||||
from a reset, no in-place conversion of the 81 M rows.
|
||||
|
||||
8. **Test before the reset.** Replay the corpus (the raw ranges and the
|
||||
captures, several B articles) with the new edge into a scratch database
|
||||
on bulk: NVMe bytes per article (the one number that decides whether
|
||||
4 TB is enough), the share of albums and incomplete releases, and a
|
||||
sample of releases against today's. Speed is measured after the reset,
|
||||
on the NVMe.
|
||||
|
||||
9. **The reset.** Tail first, then history by priority, read from the
|
||||
corpus where stored. Survives: `probe_cache` (HDD, by message-id),
|
||||
`spots`, `groups` and their priorities, `nntp_shares`, the corpus.
|
||||
Recomputed: releases, chunks, inspections, candidates, par2 data,
|
||||
verdicts, file hashes. Resets are paused (2026-10-12); this one waits
|
||||
for the user's go-ahead, and for the new NVMe.
|
||||
|
||||
### Schema (built 2026-10-25; migrations 6000 and 6100)
|
||||
|
||||
Migration 6000 *is* the reset: it truncates every table the indexing
|
||||
recomputes and refuses to run on a catalog with releases unless
|
||||
`VOYAGER3_RESET=1`, so merging `voyager-3` into `rebuild` is the go-ahead.
|
||||
Survives: `groups`, `ranges` (planned again from their start, with their
|
||||
priorities, `capture`, `est_date`, `stored_at`, `plain_chunks` and measured
|
||||
`posted_low/high`), the corpus, `probe_cache`, `spots` (unlinked from
|
||||
releases; the inspector matches them again), `nntp_shares`, `nodes`,
|
||||
`users`, the rule sets.
|
||||
|
||||
- `releases`: created anew, 8-byte columns first: `id`, `posted_at`,
|
||||
`last_posted_at`, `first_article`, `last_article`, `segments`, `bytes`,
|
||||
`settled_at` (the finish; null while a tail release is provisional),
|
||||
`inspected_at`, `first_seen_at`; `group_id`, `group_set_id`, `poster_id`,
|
||||
`file_total`, `files`, `chunks`, `category`; `source smallint`
|
||||
(`lib/release/source.ts`: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6
|
||||
album); `readable`, `has_par2`, `complete`, `passworded`, `sidecar_only`,
|
||||
`edge` (open at its block's end); `release_key` and `identity_key` as
|
||||
20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`,
|
||||
`last_seen_at`. Indexes: pkey, release_key unique, (identity_key,
|
||||
group_id), newest, (poster_id, newest), passworded, `(group_id,
|
||||
identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS
|
||||
NULL` for the provisional ones. No trigram indexes: search matches
|
||||
`release_texts`. The category trigger reads the groups from
|
||||
`group_sets`; the judge's dirty trigger watches `poster_id` instead of
|
||||
`poster`.
|
||||
- `pending_releases` (migrations 6300 and 6400, the HDD): the unnamed
|
||||
releases outside the catalog (decision f): `release_key` (primary),
|
||||
`identity_key`, `group_id`, `name` (the random stem), `poster` text, the
|
||||
posting span, `first_article`, `last_article`, counts, `file_total`,
|
||||
`partial`, `finished`, `tool`, `candidates` jsonb (the probe targets),
|
||||
`inspections` jsonb (what each probe showed), `named`, `status` (0
|
||||
waiting, 1 named and in the catalog, 2 discarded, 3 exhausted, 4 named
|
||||
awaiting the move, 9 taken), `tries`, `created_at`, `probed_at`; partial
|
||||
indexes on the waiting (by `created_at`) and the named (by `probed_at`).
|
||||
- `pending_signatures` (6400): per (tool, group, poster; poster '' for the
|
||||
tool and group together) `probed`, `named`, `dropped`, `updated_at`.
|
||||
- `release_texts(release_id, kind smallint, text varchar(255))` (6200),
|
||||
primary key on all three, one trigram index (fastupdate): every string
|
||||
search matches against, appended by whoever learns one and never an
|
||||
update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name
|
||||
(the release name, stored once more so one index serves everything),
|
||||
2 file (the prober's names inside an archive), 3 nfo (an NFO's title),
|
||||
4 par2 (a set's name), 5 media (a container's title); at most 20 per
|
||||
release and kind (`textsToStore`). A repeat insert is a no-op; the
|
||||
judge's dirty mark follows an insert or delete, so a rule can react to a
|
||||
name arriving later. Search: the trigram index yields release ids, joined
|
||||
to `releases` by primary key (never an OR across the two tables, which
|
||||
the planner cannot serve from indexes). `inner_name` stays on `releases`
|
||||
as the one display string. Facets to filter on (resolution, codec,
|
||||
language) are not texts: columns, or a `meta jsonb` with a
|
||||
`jsonb_path_ops` index once one is proven useful.
|
||||
- `posters(id serial, poster unique)` with its trigram index;
|
||||
`group_sets(id serial, groups jsonb unique)`, a sorted array of names.
|
||||
- `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume
|
||||
point), `posted_low/high`, `status`, `priority`, `node_id`, `attempts`,
|
||||
`rule_version`, `headers`, `releases`, `leased_at`, `heartbeat_at`,
|
||||
`done_at`, `error`; indexes todo (priority, id) and leased.
|
||||
`ranges.block_id` points at the block; the ranges' own lease columns stay
|
||||
until the leases move (then a migration drops them).
|
||||
- `release_chunks`, `rule_counts`, `rule_samples`: `range_id` → `block_id`.
|
||||
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
|
||||
`settled_at IS NOT NULL`), the settler, the fragment merger,
|
||||
`releases_category_newest_idx`.
|
||||
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
|
||||
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
|
||||
rule read all of `release_inspections` every round, 17 min on the HDD;
|
||||
the judge must read it by release or keep what it needs on the NVMe.
|
||||
|
||||
### Batch store (built 2026-10-25)
|
||||
|
||||
`app/services/batch_store.ts`: every release chunk of a batch is one
|
||||
release, inserted once. Its keys (`lib/release/keys.ts`): `identity_key`
|
||||
the identity's SHA-1 as 20 bytes; `release_key` the SHA-1 of the identity,
|
||||
the owner group and the release's first article, so it is deterministic
|
||||
(no numbering under a group lock, no catalog lookups before writing) and a
|
||||
repost, starting at another article, is a release of its own. Its poster
|
||||
and group set by id (`app/services/catalog_refs.ts`: insert what's new,
|
||||
read the ids, cache them per process; the tests forget the cache per
|
||||
test), its source as a code, its name into `release_texts` (kind 1). The
|
||||
chunk's own counts go in as the release's; `complete` is decided then from
|
||||
the counter. A finished release's probe candidates are claimable at once
|
||||
(`ready`, `unreadable`); a provisional tail release (`finished: false` on
|
||||
the chunk, `settled_at` null) is finished by a later chunk of the same
|
||||
release key, which adds the files not sent before (sums, one more chunk),
|
||||
sets `settled_at` and makes its candidates claimable; a finished release
|
||||
sent again is left as it is (`ON CONFLICT … WHERE settled_at IS NULL`).
|
||||
Gone: the join of a chunk into an earlier release of its key (the window
|
||||
keeps an upload together; the block merger joins what a block's end cut),
|
||||
the join across groups (FTD's spread uploads: a gap, to be handled by the
|
||||
block merger or the par2 merger across groups), the numbering of release
|
||||
keys, the group lock, `fragments` (ignored when sent; the Rust edge still
|
||||
sends them). The chunk type carries `finished` and `edge` (both optional;
|
||||
the Rust edge doesn't send them yet, so its releases are finished, not
|
||||
edges). Interim until blocks are leased: chunks, rule counts and samples
|
||||
are keyed by `ranges.block_id`, and a range without a block gets a block
|
||||
of its own in the store (`blockOf`).
|
||||
|
||||
Readers moved with it: search joins `posters` and `group_sets`, matches
|
||||
words in `release_texts` (`r.id IN (SELECT release_id … ILIKE)`), searches
|
||||
posters by id, filters `source` by code and returns the source name; the
|
||||
par2 merger, the sidecar folder and the judge's scraped rule use
|
||||
`poster_id` (`release_merges.from_key` holds the key as hex); the spot
|
||||
inspector matches by bytea identity keys and inserts its spot releases by
|
||||
id references with their title in `release_texts`; the spotter and the NZB
|
||||
service compare the source code. Tests insert fixtures through
|
||||
`tests/functional/helpers/releases.ts` (`insertRelease`: text keys, poster
|
||||
and groups by name, source by name). The unit and functional suites pass.
|
||||
|
||||
### The window in the Rust edge (built 2026-10-25)
|
||||
|
||||
`edge/src/indexer.rs`: the scanner and the file aggregator as before; then
|
||||
the window instead of the range path. `Frontier` keeps a ring of the last
|
||||
`FRONTIER_ARTICLES` (1,024) plausible dates (within `PLAUSIBLE_MS`, 7
|
||||
days, of the frontier; 1,024 implausible dates in a row that agree move
|
||||
it), recomputed once per chunk; established after 64 dated articles
|
||||
(`settle` at the block's end for smaller blocks); its first value is the
|
||||
block's start in posting time. Per chunk: completed files, then the files
|
||||
whose last part is before frontier − `WINDOW_MS` (24 h; undated ones idle
|
||||
for `idle` chunks), go into the open release of their key whose posting
|
||||
span is within `UPLOAD_GAP_MS` (3 h) of theirs, else start one (the
|
||||
upload number per key, distinct within a batch); releases whose last part
|
||||
is before frontier − `WINDOW_MS` are judged and written `finished`, as an
|
||||
`edge` when posted within `WINDOW_MS` of the block's start. At the end
|
||||
(`finish`) everything left is written: finished when the window closed on
|
||||
it, else `finished: false, edge: true`. A file short of parts stays in
|
||||
its release and sets `partial` (the store: `complete = false`). A lone
|
||||
article of a multi-part random file is still counted `per-article`, not
|
||||
written. Gone: fragments and the cut-off, `uploads_of` (online now),
|
||||
`check.ts` and the fixture comparison (the parity with the TypeScript
|
||||
indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
|
||||
(incomplete releases are kept). The wire carries `finished`, `edge` and
|
||||
`partial`; the store reads them (`partial` into `complete`). Memory: the
|
||||
window holds a day of a group's segments in memory (the busiest group's
|
||||
~50 M); the spool file per block of the design is not built yet.
|
||||
|
||||
### The replay harness (built 2026-10-26)
|
||||
|
||||
`node ace replay:corpus --group <name> [--dir CORPUS_DIR] [--from article]
|
||||
[--to article] [--limit ranges] [--batch-chunks 5] [--no-finish]` (commands/
|
||||
replay_corpus.ts; design 8): a group's complete corpus ranges in article
|
||||
order through one window in the Rust edge (`edge-bench --group <dir>
|
||||
--rules rules.json --wire -`: the window carried from range to range, the
|
||||
end finished after the last range; `edge/src/lease.rs` has the lease JSON
|
||||
the worker and the bench share) and the batch store into a scratch
|
||||
catalog, as the worker would store them: each range leased to a node named
|
||||
`replay`, its batches stored at its resume point (pulled from the bench's
|
||||
stdout as the store keeps up), the range finished when its last batch is
|
||||
in. It refuses a database not named `*_replay` or `*_scratch`.
|
||||
The rules are the scratch catalog's current set and group priorities:
|
||||
copy `groups` and `rule_sets` from the catalog for a faithful replay. The
|
||||
report: releases finished, provisional, edges; complete, incomplete,
|
||||
undecided; readable, with par2; files, segments as a share of the
|
||||
headers; bytes per header of the NVMe tables (releases, release_texts,
|
||||
posters, group_sets, rule counts and samples) and of those on
|
||||
CHUNK_TABLESPACE (release_chunks, probe_candidates, release_inspections).
|
||||
The app image carries `edge-bench` for it (Dockerfile); on homelab the
|
||||
scratch catalog is a second database (`usenet_replay`, default tablespace
|
||||
`nvme` like the catalog, the chunk tables on `pg_default`; migrated with
|
||||
the voyager-3 image built from the deploy cache's repo, `groups` and
|
||||
`rule_sets` copied over), the corpus mounted read-only.
|
||||
`deploy/replay-window.sh` runs a window there: each group whose window
|
||||
ranges are all stored is replayed once with the window's bounds as
|
||||
`--from`/`--to` (the corpus holds other windows of the same group), 4
|
||||
groups at a time, logs and `.done` markers in `.cache/replay/`, looping
|
||||
until the window is done (LOG.md 2026-10-26 (111) has the first local
|
||||
numbers, (112) the homelab run). **Deployed 2026-10-26** (LOG.md (116)):
|
||||
voyager 3 runs on homelab without its edge worker (`.cache/no-edge`:
|
||||
deploy.sh and the nightly resume leave it stopped, since range by range
|
||||
it would split releases at every edge with no merger left), and history
|
||||
comes from the corpus through this driver into the catalog itself
|
||||
(`DB=usenet_rebuild OUT=.cache/replay-catalog`, the harness allowed by
|
||||
`REPLAY_INTO_CATALOG=1`), the prober's ladder, the merger and the judge
|
||||
working on it as it fills; the scratch run's numbers (63 groups) were
|
||||
read before (LOG.md (115)). The tail waits for the window worker.
|
||||
|
||||
Next: the replay on homelab over the 2016 window (the NVMe bytes per
|
||||
header for real), the ordered range leases with the carried window in
|
||||
the edge-node and its snapshot (design 1 and 5), the prober writing
|
||||
`release_texts` kind 2, the spot/NZB match on pending keys (decision f). Gone from the list: block
|
||||
leases, the planner and the block merger (design 1, amended).
|
||||
|
||||
## Decisions
|
||||
|
||||
a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
|
||||
@@ -321,8 +715,64 @@ c. Group list: editable, owner by priority number (above). **Accepted.**
|
||||
d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
|
||||
e. Headers through the coordinator's relay, with prefetch (part 1).
|
||||
**Accepted.**
|
||||
f. Random-named releases (no file with a readable name) keep no chunk in
|
||||
the header pass: their file keys and probe candidates only, until
|
||||
phase 2 names them (bodies) and fetches their segments again. The full
|
||||
history is ~3 TB with them, ~1.2–1.4 TB without (725 M of the 1.25 B
|
||||
kept files; LOG.md 2026-10-09). **Accepted** 2026-10-09, not built yet.
|
||||
f. **Nothing enters the catalog without a usable name** (amended
|
||||
2026-10-26, the user's direction; before: random-named releases keep
|
||||
no chunk but their rows). A release the window closed on whose files
|
||||
all have random names is no release yet: it goes to `pending_releases`
|
||||
(migration 6300, the HDD) with its keys, counts, article span, upload
|
||||
tool and probe targets, no row in `releases`, `release_texts` or
|
||||
`posters`, no chunk. The pending ladder works them **from least to most
|
||||
effort**: (0) no fetch: a signature verdict (tool and group, poster)
|
||||
learned from earlier probes, a spot or NZB that names the message-ids;
|
||||
(1) a par2 main file's first article (it names the files); (2) the first
|
||||
article of the first file (the yEnc name, an archive's listing in its
|
||||
first 16 KiB); (3) a 7z's tail; then nothing more. A probe that yields
|
||||
a usable name moves the release into the catalog, its segments read
|
||||
again from the corpus (the range stored) or the server; one that
|
||||
yields none counts against its signature, and a signature with
|
||||
≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
|
||||
2026-10-08 (5)) is **discarded** from then on, a sample of it still
|
||||
probed to notice a change. The reason, in the 2016 replay: one poster's
|
||||
hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
|
||||
groups) were 76% of the rows and three quarters of the NVMe bytes for
|
||||
11% of the segments and nothing anyone can find; the full history has
|
||||
725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
|
||||
edge sends an unnamed release with an empty payload and its tool, the
|
||||
batch store writes `pending_releases` (LOG.md (114)); the ladder in the
|
||||
prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
|
||||
the oldest waiting releases' next target (par2, archive, lone, 7z-tail
|
||||
in that order), at least PENDING_SHARE (25%) of each claim next to the
|
||||
catalog's own candidates, answered from `probe_cache` when the article
|
||||
was read before; `usableName` (lib/probe/inspect.ts) takes the largest
|
||||
named file of a par2 set or a listing, or a lone file's yEnc name, that
|
||||
is neither packaging nor random; a name sets status 4, no name the next
|
||||
target or status 3 (exhausted); each finished climb counts on
|
||||
`pending_signatures` for (tool, group, poster) and (tool, group), and a
|
||||
signature with PENDING_MIN_PROBES (100) climbs and at most
|
||||
PENDING_MAX_NAMED (0.5%) named is dropped: every five minutes the
|
||||
waiting releases of dropped signatures are discarded (status 2), the
|
||||
rules' `samplePercent` of them left to climb. The namer
|
||||
(`app/services/pending_namer.ts`, in the prober's process): a named
|
||||
release's article span read again from the corpus (the stored range's
|
||||
chunks) or by XOVER on a connection of its own, through the TypeScript
|
||||
indexer, written as a release under the pending key with `source`
|
||||
weak-stem, the name as `inner_name`, its chunk and its candidates (the
|
||||
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
|
||||
the password flag). Not built: the spot/NZB match on pending keys (step
|
||||
0 of the ladder beyond signatures), and a signature by poster alone
|
||||
across groups.
|
||||
g. The account's NNTP connections shared by priority (`nntp_shares`): the
|
||||
coordinator (headers) before the prober, each between its minimum and
|
||||
maximum, 5 kept for the spotter, the spot inspector, the plain
|
||||
connection and reconnects. A process grows only into connections the
|
||||
others closed, so the account never goes over its limit. Replaces the
|
||||
fixed split (38 + 8), under which the prober stayed at 8 while history
|
||||
was paused and the coordinator's connections idle (LOG.md 2026-10-18
|
||||
(75)). **Accepted** 2026-10-18.
|
||||
h. Idle NNTP connections closed: one with nothing to do for
|
||||
`NNTP_IDLE_CLOSE_SECONDS` (60) is closed and opened again with work
|
||||
(the coordinator's pool on its next sync, the plain one too; a dry
|
||||
prober lane; the spotter between rounds; the spot inspector with no
|
||||
spots). Eweka dropped the quiet ones and the next command failed
|
||||
("Spot inspection failed: NNTP connection closed"), and idle ones took
|
||||
the account's connections from downloads. **Accepted** 2026-10-22.
|
||||
+276
-181
@@ -1,24 +1,57 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
|
||||
import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import CatalogRefs, { groupSetKey, posterText } from '#services/catalog_refs'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { SIDECAR } from '#services/sidecar_folder'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
import { uploadTool } from '#lib/rules/index'
|
||||
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
|
||||
|
||||
/**
|
||||
* A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
|
||||
* fold's candidates (releases.sidecar_only). Its file list decoded here, small as it is.
|
||||
*/
|
||||
function sidecarOnly(r: ReleaseChunk): boolean | null {
|
||||
if (r.files > 3 || r.bytes >= 1_000_000) return null
|
||||
const files = decodeChunk(r.payload)
|
||||
return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
|
||||
}
|
||||
|
||||
export class StaleBatch extends Error {}
|
||||
|
||||
export interface StoredBatch {
|
||||
/** Release chunks in the batch. */
|
||||
releases: number
|
||||
/** Releases the batch started. */
|
||||
newReleases: number
|
||||
/** Provisional (tail) releases the batch finished. */
|
||||
finalized: number
|
||||
candidates: number
|
||||
fragments: number
|
||||
/** Unnamed releases written to the pending store instead of the catalog (decision f). */
|
||||
pending: number
|
||||
}
|
||||
|
||||
/** A chunk's slot in its batch: the key and the upload the edge numbered it with (ProbeCandidate points at it). */
|
||||
const slot = (key: string, upload: number) => `${key}\n${upload}`
|
||||
|
||||
/**
|
||||
* Stores one batch of a leased range (REBUILD.md, parts 3–7), in one
|
||||
* transaction: release aggregates in the catalog, one chunk per release,
|
||||
* probe candidates, rule counts and samples, edge fragments, and the
|
||||
* range's resume point. A batch that doesn't start at the resume point was
|
||||
* Stores one batch of a leased range (REBUILD.md, "Voyager 3: window indexing", 2 and 4) in one transaction: each
|
||||
* release chunk is a whole release the edge's window finished, written once (its poster and group set by id, its
|
||||
* keys as bytea, its name into release_texts), with one chunk of its files, its probe candidates, and the batch's
|
||||
* rule counts and samples; then the range's resume point moves. A batch that doesn't start at the resume point was
|
||||
* sent twice or too late, and is refused (StaleBatch), so the sums stay right.
|
||||
*
|
||||
* A tail release comes twice: provisionally (finished: false, settled_at null, its candidates waiting) and then
|
||||
* finished, under the same release key (the identity, the group and its first article: lib/release/keys.ts), with
|
||||
* the files not sent before (a chunk's counts are its own; the finish adds them up); the finish is the one update a
|
||||
* release gets from the header pass. A finished release sent again is left as it is.
|
||||
*
|
||||
* Interim, until blocks are leased (the next step): chunks, counts and samples are keyed by the range's block
|
||||
* (ranges.block_id), and a range without one gets a block of its own here.
|
||||
*/
|
||||
class BatchStore {
|
||||
async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
|
||||
@@ -27,7 +60,9 @@ class BatchStore {
|
||||
// One batch of a range at a time, without holding the range's row for the whole transaction: the
|
||||
// prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
|
||||
// LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [rangeId])
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [
|
||||
rangeId,
|
||||
])
|
||||
const range = await trx.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
|
||||
@@ -35,35 +70,48 @@ class BatchStore {
|
||||
throw new StaleBatch(
|
||||
`Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
|
||||
)
|
||||
const blockId = await this.blockOf(trx, range)
|
||||
const result: StoredBatch = {
|
||||
releases: batch.releases.length,
|
||||
newReleases: 0,
|
||||
finalized: 0,
|
||||
candidates: 0,
|
||||
fragments: batch.fragments.length,
|
||||
pending: 0,
|
||||
}
|
||||
|
||||
// Nothing enters the catalog without a usable name (decision f): an unnamed release (no readable file name)
|
||||
// goes to the pending store with its probe targets, the named ones to the catalog.
|
||||
const named = batch.releases.filter((r) => r.readable)
|
||||
const unnamed = batch.releases.filter((r) => !r.readable)
|
||||
const written = await this.writeReleases(
|
||||
trx,
|
||||
rangeId,
|
||||
{ groupId: Number(range.group_id), blockId },
|
||||
batch.from,
|
||||
batch.releases,
|
||||
named,
|
||||
batch.candidates
|
||||
)
|
||||
result.newReleases = written.newReleases
|
||||
result.finalized = written.finalized
|
||||
result.candidates = written.candidates
|
||||
result.pending = await this.writePending(
|
||||
trx,
|
||||
Number(range.group_id),
|
||||
unnamed,
|
||||
batch.candidates
|
||||
)
|
||||
|
||||
if (batch.counts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (range_id, rule, key, articles, files)
|
||||
`INSERT INTO rule_counts (block_id, rule, key, articles, files)
|
||||
SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
|
||||
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
|
||||
[rangeId, JSON.stringify(batch.counts)]
|
||||
ON CONFLICT (block_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
|
||||
[blockId, JSON.stringify(batch.counts)]
|
||||
)
|
||||
}
|
||||
|
||||
for (const s of batch.samples) {
|
||||
await trx.insertQuery().table('rule_samples').insert({
|
||||
range_id: rangeId,
|
||||
block_id: blockId,
|
||||
rule: s.rule,
|
||||
rule_key: s.ruleKey,
|
||||
release_key: s.key,
|
||||
@@ -74,18 +122,6 @@ class BatchStore {
|
||||
})
|
||||
}
|
||||
|
||||
for (let i = 0; i < batch.fragments.length; i += 2000) {
|
||||
const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
|
||||
range_id: rangeId,
|
||||
file_key: f.fileKey,
|
||||
// Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
|
||||
article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
|
||||
segments: f.segments.length,
|
||||
fragment: JSON.stringify(f),
|
||||
}))
|
||||
await trx.insertQuery().table('fragments').multiInsert(rows)
|
||||
}
|
||||
|
||||
const moved = await trx
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
@@ -95,106 +131,138 @@ class BatchStore {
|
||||
.update({
|
||||
next_id: batch.next,
|
||||
headers: db.raw('headers + ?', [batch.headers]),
|
||||
// The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
|
||||
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
|
||||
// The range's posting time span: the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedLow ?? null,
|
||||
]),
|
||||
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedHigh ?? null,
|
||||
]),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1))
|
||||
throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* An upload spread over groups (FTD: par2 and part1 to a.b.music.misc,
|
||||
* part2 to a.b.sounds.music, each article in one group only; LOG.md
|
||||
* 2026-10-11 (9)): a chunk without a release in its own group joins one of
|
||||
* its key in another group whose window is within UPLOAD_GAP_MS, when none
|
||||
* of its file names are in that release yet. A post of the same files to
|
||||
* another group (other message-ids, the same names) stays a release of its
|
||||
* own, or the NZB would hold every file twice. Reads the candidate's file
|
||||
* names from its chunks; candidates are few.
|
||||
*/
|
||||
private async joinAcrossGroups(
|
||||
trx: TransactionClientContract,
|
||||
part: ReleaseChunk[],
|
||||
windows: { idx: number; key: string; first: number | null | undefined; last: number | null | undefined }[],
|
||||
groupId: number,
|
||||
joined: Map<number, number>
|
||||
) {
|
||||
const open = windows.filter((w) => !joined.has(w.idx) && w.first)
|
||||
if (open.length === 0) return
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT v.idx, r.id
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
|
||||
JOIN releases r ON r.identity_key = v.key AND r.group_id <> ?
|
||||
WHERE r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0)
|
||||
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
|
||||
[JSON.stringify(open), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
|
||||
)
|
||||
if (candidates.length === 0) return
|
||||
const ids: number[] = [...new Set<number>(candidates.map((c: { id: string }) => Number(c.id)))]
|
||||
const names = new Map<number, Set<string>>()
|
||||
const { rows: chunks } = await trx.rawQuery(`SELECT release_id, payload FROM release_chunks WHERE release_id = ANY(?)`, [ids])
|
||||
for (const c of chunks) {
|
||||
const set = names.get(Number(c.release_id)) ?? new Set<string>()
|
||||
for (const f of decodeChunk(c.payload)) set.add(f.name)
|
||||
names.set(Number(c.release_id), set)
|
||||
}
|
||||
for (const c of candidates as { idx: number; id: string }[]) {
|
||||
if (joined.has(c.idx)) continue
|
||||
const have = names.get(Number(c.id)) ?? new Set<string>()
|
||||
const incoming = decodeChunk(part[c.idx].payload).map((f) => f.name)
|
||||
if (incoming.every((name) => !have.has(name))) {
|
||||
joined.set(c.idx, Number(c.id))
|
||||
for (const name of incoming) have.add(name)
|
||||
names.set(Number(c.id), have)
|
||||
}
|
||||
}
|
||||
/** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
|
||||
async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
|
||||
if (range.block_id) return Number(range.block_id)
|
||||
const [block] = await trx
|
||||
.table('blocks')
|
||||
.insert({
|
||||
group_id: range.group_id,
|
||||
start_id: range.start_id,
|
||||
end_id: range.end_id,
|
||||
next_id: range.next_id,
|
||||
status: 'leased',
|
||||
priority: range.priority ?? 0,
|
||||
node_id: range.node_id,
|
||||
leased_at: range.leased_at ?? new Date(),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
.returning('id')
|
||||
await trx.from('ranges').where('id', range.id).update({ block_id: block.id })
|
||||
return Number(block.id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Release aggregates in the catalog, one write-once chunk per release, and
|
||||
* probe candidates (a batch's, or files the fragment merger completed).
|
||||
* A chunk joins the release of its key whose articles the range's group
|
||||
* owns and whose posting window is within UPLOAD_GAP_MS of its own, else
|
||||
* starts one: re-uploads are releases of their own (decision d). The
|
||||
* group is locked, so concurrent writers don't both start one.
|
||||
* The unnamed releases of a batch into pending_releases (migration 6300, the HDD): keys, counts, the upload
|
||||
* tool and the probe targets, no chunk. The pending ladder (REBUILD.md decision f) probes them cheapest first,
|
||||
* and a name moves one into the catalog; a provisional tail release sent again adds its counts up.
|
||||
*/
|
||||
async writePending(
|
||||
trx: TransactionClientContract,
|
||||
groupId: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
): Promise<number> {
|
||||
if (releases.length === 0) return 0
|
||||
const targets = new Map<string, { kind: string; messageId: string; file: string }[]>()
|
||||
for (const c of candidates) {
|
||||
const key = slot(c.releaseKey, c.upload ?? 0)
|
||||
const list = targets.get(key) ?? []
|
||||
list.push({ kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
targets.set(key, list)
|
||||
}
|
||||
let written = 0
|
||||
for (let i = 0; i < releases.length; i += 1000) {
|
||||
const rows = releases.slice(i, i + 1000).map((r) => {
|
||||
const probes = targets.get(slot(r.key, r.upload)) ?? []
|
||||
return {
|
||||
release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name.slice(0, TEXT_MAX),
|
||||
poster: r.poster || null,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
first_article: r.firstArticle,
|
||||
last_article: r.lastArticle,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
bytes: r.bytes,
|
||||
partial: r.partial === true,
|
||||
finished: r.finished !== false,
|
||||
file_total: r.fileTotal,
|
||||
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
|
||||
candidates: probes,
|
||||
}
|
||||
})
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
|
||||
files, segments, bytes, partial, finished, file_total, tool, candidates)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
|
||||
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
|
||||
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
|
||||
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
|
||||
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
|
||||
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END
|
||||
WHERE NOT pending_releases.finished
|
||||
RETURNING release_key`,
|
||||
[groupId, JSON.stringify(rows)]
|
||||
)
|
||||
written += done.length
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases of a batch into the catalog, one chunk each, and their probe candidates. Every chunk is one
|
||||
* release: inserted, or (a provisional tail release sent again) finished.
|
||||
*/
|
||||
async writeReleases(
|
||||
trx: TransactionClientContract,
|
||||
rangeId: number,
|
||||
where: { groupId: number; blockId: number },
|
||||
batchFrom: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
) {
|
||||
const out = { newReleases: 0, candidates: 0 }
|
||||
const out = { newReleases: 0, finalized: 0, candidates: 0 }
|
||||
if (releases.length === 0) return out
|
||||
/** Release ids by slot, for the chunks and candidates; a slot not here was a finished release sent again. */
|
||||
const ids = new Map<string, number>()
|
||||
const slot = (key: string, upload: number) => `${key}\n${upload}`
|
||||
const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
|
||||
// One writer per group at a time: a chunk only matches releases of its group, and a lock per key
|
||||
// ran out of shared memory (a picture group's batch has 50k keys; LOG.md 2026-10-09 (12)).
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('releases:' || ?::text, 0))`, [groupId])
|
||||
const finalized = new Set<number>()
|
||||
|
||||
const posters = await CatalogRefs.posterIds(trx, releases.map((r) => r.poster).filter(Boolean))
|
||||
const groupSets = await CatalogRefs.groupSetIds(
|
||||
trx,
|
||||
releases.map((r) => r.groups)
|
||||
)
|
||||
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
|
||||
for (let i = 0; i < sorted.length; i += 1000) {
|
||||
const part = sorted.slice(i, i + 1000)
|
||||
const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
|
||||
const { rows: matches } = await trx.rawQuery(
|
||||
`SELECT DISTINCT ON (v.idx) v.idx, r.id
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
|
||||
JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
|
||||
WHERE v.first IS NULL OR r.posted_at IS NULL
|
||||
OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
|
||||
ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
|
||||
[JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
|
||||
)
|
||||
const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
|
||||
await this.joinAcrossGroups(trx, part, windows, groupId, joined)
|
||||
const row = (r: ReleaseChunk) => ({
|
||||
const rows = part.map((r, idx) => ({
|
||||
idx,
|
||||
release_key: releaseKey(r.key, where.groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceCode(r.source),
|
||||
poster_id: r.poster ? (posters.get(posterText(r.poster)) ?? null) : null,
|
||||
group_set_id: groupSets.get(groupSetKey(r.groups))!,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
file_total: r.fileTotal,
|
||||
@@ -205,98 +273,114 @@ class BatchStore {
|
||||
last_article: r.lastArticle,
|
||||
readable: r.readable,
|
||||
has_par2: r.hasPar2,
|
||||
})
|
||||
const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
|
||||
if (updates.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
|
||||
a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
|
||||
max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
|
||||
FROM v GROUP BY id)
|
||||
UPDATE releases r SET
|
||||
files = r.files + a.files,
|
||||
segments = r.segments + a.segments,
|
||||
bytes = r.bytes + a.bytes,
|
||||
chunks = r.chunks + a.chunks,
|
||||
posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
|
||||
last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
|
||||
file_total = coalesce(r.file_total, a.file_total),
|
||||
first_article = least(r.first_article, a.first_article),
|
||||
last_article = greatest(r.last_article, a.last_article),
|
||||
readable = r.readable OR a.readable,
|
||||
has_par2 = r.has_par2 OR a.has_par2,
|
||||
-- As it stands (the counter's files all found); settling makes it final.
|
||||
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
|
||||
last_seen_at = now()
|
||||
FROM a WHERE r.id = a.id`,
|
||||
[JSON.stringify(updates)]
|
||||
finished: r.finished !== false,
|
||||
edge: r.edge === true,
|
||||
partial: r.partial === true,
|
||||
sidecar_only: sidecarOnly(r),
|
||||
}))
|
||||
// Inserted, or a provisional release of the key finished (the chunk's files added up); a finished release
|
||||
// sent again (no row returned) is left as it is.
|
||||
const { rows: written } = await trx.rawQuery(
|
||||
`INSERT INTO releases (release_key, identity_key, group_id, group_set_id, poster_id, name, source, posted_at, last_posted_at, file_total, files, segments, bytes, chunks,
|
||||
first_article, last_article, readable, has_par2, complete, edge, sidecar_only, settled_at)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.group_set_id, v.poster_id, v.name, v.source,
|
||||
to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
|
||||
v.first_article, v.last_article, v.readable, v.has_par2,
|
||||
CASE WHEN v.partial THEN false WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END,
|
||||
v.edge, v.sidecar_only, CASE WHEN v.finished THEN now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, release_key text, identity_key text, name text, source int, poster_id int, group_set_id int, first bigint, last bigint,
|
||||
file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean,
|
||||
finished boolean, edge boolean, partial boolean, sidecar_only boolean)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = releases.files + excluded.files, segments = releases.segments + excluded.segments, bytes = releases.bytes + excluded.bytes,
|
||||
chunks = releases.chunks + CASE WHEN excluded.files > 0 THEN 1 ELSE 0 END,
|
||||
posted_at = least(releases.posted_at, excluded.posted_at), last_posted_at = greatest(releases.last_posted_at, excluded.last_posted_at),
|
||||
file_total = coalesce(excluded.file_total, releases.file_total),
|
||||
first_article = least(releases.first_article, excluded.first_article), last_article = greatest(releases.last_article, excluded.last_article),
|
||||
readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2,
|
||||
complete = CASE WHEN releases.complete IS FALSE AND releases.file_total IS NULL THEN false
|
||||
WHEN coalesce(excluded.file_total, releases.file_total) IS NULL THEN releases.complete
|
||||
ELSE releases.files + excluded.files >= coalesce(excluded.file_total, releases.file_total) END,
|
||||
edge = excluded.edge, sidecar_only = coalesce(releases.sidecar_only, excluded.sidecar_only), settled_at = excluded.settled_at
|
||||
WHERE releases.settled_at IS NULL
|
||||
RETURNING id, encode(release_key, 'hex') AS release_key, (xmax = 0) AS inserted, settled_at`,
|
||||
[where.groupId, JSON.stringify(rows)]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; inserted: boolean; settled: boolean }>(
|
||||
written.map(
|
||||
(w: { id: string; release_key: string; inserted: boolean; settled_at: Date | null }) => [
|
||||
w.release_key,
|
||||
{ id: Number(w.id), inserted: w.inserted, settled: w.settled_at !== null },
|
||||
]
|
||||
)
|
||||
part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
|
||||
}
|
||||
const fresh = part.filter((_, idx) => !joined.has(idx))
|
||||
if (fresh.length > 0) {
|
||||
// The first upload of a key keeps the key; later ones are key:2, key:3, …
|
||||
const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
|
||||
const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
|
||||
const numbered = (r: ReleaseChunk) => {
|
||||
const n = (taken.get(r.key) ?? 0) + 1
|
||||
taken.set(r.key, n)
|
||||
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
|
||||
)
|
||||
const texts: { release_id: number; kind: number; text: string }[] = []
|
||||
for (const row of rows) {
|
||||
const w = byKey.get(row.release_key)
|
||||
if (!w) continue
|
||||
const r = part[row.idx]
|
||||
ids.set(slot(r.key, r.upload), w.id)
|
||||
if (w.inserted) out.newReleases++
|
||||
else if (w.settled) {
|
||||
out.finalized++
|
||||
finalized.add(w.id)
|
||||
}
|
||||
// Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
|
||||
const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
|
||||
if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
|
||||
let inserts = fresh.map(numbered)
|
||||
// Writers of other groups number the same key at the same time (a spam post crossposted to 67
|
||||
// groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
|
||||
for (let attempt = 0; inserts.length > 0; attempt++) {
|
||||
if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
|
||||
const { rows } = await trx.rawQuery(
|
||||
`INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
|
||||
SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
|
||||
v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
|
||||
ON CONFLICT (release_key) DO NOTHING
|
||||
RETURNING id, release_key`,
|
||||
[groupId, JSON.stringify(inserts)]
|
||||
)
|
||||
const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
|
||||
const missed: typeof inserts = []
|
||||
for (const r of inserts) {
|
||||
const id = byReleaseKey.get(r.release_key)
|
||||
if (id === undefined) missed.push(r)
|
||||
else ids.set(slot(r.key, r.upload), id)
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.name, text: row.name.slice(0, TEXT_MAX) })
|
||||
// An album's members stay findable by their own names (the first TEXTS_PER_KIND of them).
|
||||
if (r.source === 'album' && r.files > 0) {
|
||||
for (const text of textsToStore(decodeChunk(r.payload).map((f) => f.name))) {
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.member, text })
|
||||
}
|
||||
out.newReleases += rows.length
|
||||
const chunks = new Map(fresh.map((r) => [slot(r.key, r.upload), r]))
|
||||
inserts = missed.map((r) => numbered(chunks.get(slot(r.key, r.upload))!))
|
||||
}
|
||||
}
|
||||
// The name (and an album's members), searchable (release_texts; a text sent again is a no-op).
|
||||
if (texts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_texts (release_id, kind, text)
|
||||
SELECT v.release_id, v.kind, v.text FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind int, text text) WHERE v.text <> ''
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[JSON.stringify(texts)]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// One write-once chunk per release.
|
||||
// One write-once chunk per release (a finish adds the files not sent before).
|
||||
for (let i = 0; i < sorted.length; i += 500) {
|
||||
const rows = sorted.slice(i, i + 500).map((r) => ({
|
||||
release_id: ids.get(slot(r.key, r.upload))!,
|
||||
range_id: rangeId,
|
||||
batch_from: batchFrom,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
payload: r.payload,
|
||||
}))
|
||||
const rows = sorted
|
||||
.slice(i, i + 500)
|
||||
.filter((r) => ids.has(slot(r.key, r.upload)) && r.files > 0)
|
||||
.map((r) => ({
|
||||
release_id: ids.get(slot(r.key, r.upload))!,
|
||||
block_id: where.blockId,
|
||||
batch_from: batchFrom,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
payload: r.payload,
|
||||
}))
|
||||
if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
|
||||
}
|
||||
|
||||
// A finished release's candidates are claimable at once; a provisional one's wait for its finish.
|
||||
const settled = new Set<number>()
|
||||
for (const r of sorted) {
|
||||
const id = ids.get(slot(r.key, r.upload))
|
||||
if (id !== undefined && r.finished !== false) settled.add(id)
|
||||
}
|
||||
const chunks = new Map(sorted.map((r) => [slot(r.key, r.upload), r]))
|
||||
const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
|
||||
for (let i = 0; i < kept.length; i += 2000) {
|
||||
const rows = kept
|
||||
.slice(i, i + 2000)
|
||||
.map((c) => ({
|
||||
release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
|
||||
const rows = kept.slice(i, i + 2000).map((c) => {
|
||||
const releaseId = ids.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
const chunk = chunks.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
return {
|
||||
release_id: releaseId,
|
||||
kind: c.kind,
|
||||
message_id: c.messageId,
|
||||
file: c.file,
|
||||
}))
|
||||
ready: settled.has(releaseId),
|
||||
unreadable: !chunk.readable,
|
||||
}
|
||||
})
|
||||
const added = await trx
|
||||
.knexQuery()
|
||||
.table('probe_candidates')
|
||||
@@ -305,6 +389,17 @@ class BatchStore {
|
||||
.ignore()
|
||||
.returning('id')
|
||||
out.candidates += added.length
|
||||
// A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
|
||||
const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
|
||||
if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
|
||||
}
|
||||
// The finish of a provisional release: its waiting candidates become claimable.
|
||||
if (finalized.size > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[[...finalized]]
|
||||
)
|
||||
}
|
||||
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import type db from '@adonisjs/lucid/services/db'
|
||||
|
||||
type Client = Pick<typeof db, 'rawQuery'>
|
||||
|
||||
/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
|
||||
export const POSTER_MAX = 200
|
||||
/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
|
||||
const CACHE_MAX = 200_000
|
||||
|
||||
const posterCache = new Map<string, number>()
|
||||
const groupSetCache = new Map<string, number>()
|
||||
|
||||
/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
|
||||
export function groupSetKey(groups: string[]): string {
|
||||
return JSON.stringify([...new Set(groups)].sort())
|
||||
}
|
||||
|
||||
/** A poster as posters stores it. */
|
||||
export function posterText(poster: string): string {
|
||||
return poster.slice(0, POSTER_MAX)
|
||||
}
|
||||
|
||||
async function ids(
|
||||
client: Client,
|
||||
cache: Map<string, number>,
|
||||
table: 'posters' | 'group_sets',
|
||||
column: 'poster' | 'groups',
|
||||
cast: 'text' | 'jsonb',
|
||||
values: string[]
|
||||
): Promise<Map<string, number>> {
|
||||
const out = new Map<string, number>()
|
||||
const missing: string[] = []
|
||||
for (const v of values) {
|
||||
const id = cache.get(v)
|
||||
if (id === undefined) missing.push(v)
|
||||
else out.set(v, id)
|
||||
}
|
||||
if (missing.length === 0) return out
|
||||
// Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
|
||||
await client.rawQuery(
|
||||
`INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
|
||||
[missing]
|
||||
)
|
||||
const { rows } = await client.rawQuery(
|
||||
`SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
|
||||
[missing]
|
||||
)
|
||||
if (cache.size + rows.length > CACHE_MAX) cache.clear()
|
||||
for (const r of rows) {
|
||||
// jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
|
||||
const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
|
||||
cache.set(key, Number(r.id))
|
||||
out.set(key, Number(r.id))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
|
||||
* values are inserted; known ones come from a cache per process (the batch store sees the same posters and
|
||||
* group sets batch after batch).
|
||||
*/
|
||||
export default {
|
||||
/** Ids by poster (posterText applied), for a batch of posters. */
|
||||
async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
|
||||
return ids(client, posterCache, 'posters', 'poster', 'text', [
|
||||
...new Set(posters.map(posterText)),
|
||||
])
|
||||
},
|
||||
|
||||
async posterId(client: Client, poster: string): Promise<number> {
|
||||
const found = await this.posterIds(client, [poster])
|
||||
return found.get(posterText(poster))!
|
||||
},
|
||||
|
||||
/** Ids by groupSetKey, for a batch of group lists. */
|
||||
async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
|
||||
return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
|
||||
...new Set(sets.map(groupSetKey)),
|
||||
])
|
||||
},
|
||||
|
||||
async groupSetId(client: Client, groups: string[]): Promise<number> {
|
||||
const found = await this.groupSetIds(client, [groups])
|
||||
return found.get(groupSetKey(groups))!
|
||||
},
|
||||
|
||||
/** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
|
||||
forget() {
|
||||
posterCache.clear()
|
||||
groupSetCache.clear()
|
||||
},
|
||||
}
|
||||
+45
-15
@@ -1,11 +1,24 @@
|
||||
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzip } from 'node:zlib'
|
||||
import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
|
||||
import { promisify } from 'node:util'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { filterOverviews } from '#lib/edge/index'
|
||||
import type { ChunkSpan } from '#services/spool'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
const deflate = promisify(deflateCb)
|
||||
const gzip = promisify(gzipCb)
|
||||
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
|
||||
const MAX_FILTER_BYTES = 256 * 1024 * 1024
|
||||
|
||||
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
|
||||
export interface CorpusFilter {
|
||||
ruleVersion: number
|
||||
priorities: Record<string, number>
|
||||
}
|
||||
/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
|
||||
export const FILTERED = ['crosspost', 'obfuscated']
|
||||
|
||||
export interface CorpusRange {
|
||||
group: string
|
||||
@@ -19,20 +32,23 @@ export interface CorpusRange {
|
||||
* every chunk is there, the layout edge-bench reads (edge/fixtures). The
|
||||
* grouping sweeps replay them with each build's rules instead of waiting for
|
||||
* a re-index; the prefetcher reads a stored chunk instead of fetching it
|
||||
* again. Off without CORPUS_DIR.
|
||||
* again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
|
||||
* (101)): the lines the edge worker only counts are left out, ~half of them,
|
||||
* and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
|
||||
* article numbers by rule; its range.json says `filtered`.
|
||||
*/
|
||||
class Corpus {
|
||||
get dir(): string | undefined {
|
||||
return coordinatorConfig.corpus.dir
|
||||
}
|
||||
|
||||
private rangeDir(range: CorpusRange) {
|
||||
return join(this.dir!, range.group, `${range.start}-${range.end}`)
|
||||
private rangeDir(range: CorpusRange, dir = this.dir!) {
|
||||
return join(dir, range.group, `${range.start}-${range.end}`)
|
||||
}
|
||||
|
||||
private path(range: CorpusRange, span: ChunkSpan) {
|
||||
private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
|
||||
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
|
||||
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
/** Whether every chunk of the range is stored (its range.json). */
|
||||
@@ -53,36 +69,50 @@ class Corpus {
|
||||
)
|
||||
}
|
||||
|
||||
/** A stored chunk, or null. */
|
||||
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
|
||||
if (!this.dir) return null
|
||||
return readFile(this.path(range, span)).catch(() => null)
|
||||
/** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
|
||||
async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
|
||||
if (!dir) return null
|
||||
return readFile(this.path(range, span, dir)).catch(() => null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores a chunk that inflates (a reply cut short isn't kept), and the
|
||||
* range.json once the range is complete (`total` chunks).
|
||||
* range.json once the range is complete (`total` chunks). With `filter`,
|
||||
* without the lines the edge worker drops on their own, compressed again
|
||||
* as the news server's (zlib), with its `.drops` written first.
|
||||
*/
|
||||
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
|
||||
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
|
||||
if (!this.dir) return
|
||||
let raw: Buffer
|
||||
try {
|
||||
await inflate(data)
|
||||
raw = await inflate(data)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const dir = this.rangeDir(range)
|
||||
const file = this.path(range, span)
|
||||
await mkdir(dir, { recursive: true })
|
||||
await writeFile(`${file}.part`, data)
|
||||
let stored = data
|
||||
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
|
||||
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
|
||||
if (filter && raw.length < MAX_FILTER_BYTES) {
|
||||
// Latin-1 both ways: every byte back as it was.
|
||||
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
|
||||
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
|
||||
await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
|
||||
await rename(`${file}.drops.part`, `${file}.drops`)
|
||||
}
|
||||
await writeFile(`${file}.part`, stored)
|
||||
await rename(`${file}.part`, file)
|
||||
const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
|
||||
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
|
||||
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
|
||||
}
|
||||
|
||||
/** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
|
||||
async removeChunk(range: CorpusRange, span: ChunkSpan) {
|
||||
if (!this.dir) return
|
||||
await rm(this.path(range, span), { force: true })
|
||||
await rm(`${this.path(range, span)}.drops`, { force: true })
|
||||
await rm(join(this.rangeDir(range), 'range.json'), { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { statfs } from 'node:fs/promises'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** How often free space is measured. */
|
||||
const CHECK_MS = 60_000
|
||||
/** Resumed only this much above the floor: not paused and resumed at every flush. */
|
||||
const MARGIN_GB = 1
|
||||
|
||||
/**
|
||||
* Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a
|
||||
* container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on.
|
||||
* The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md
|
||||
* 2026-10-18 (77)). Off when the floor is 0.
|
||||
*/
|
||||
export class DiskSpace {
|
||||
private checkedAt = 0
|
||||
private low = false
|
||||
|
||||
constructor(
|
||||
private what: string,
|
||||
private path: string,
|
||||
private floorGb: number,
|
||||
private free: (path: string) => Promise<number> = async (p) => {
|
||||
const s = await statfs(p)
|
||||
return s.bavail * s.bsize
|
||||
},
|
||||
private checkMs = CHECK_MS
|
||||
) {}
|
||||
|
||||
async short(): Promise<boolean> {
|
||||
if (!this.floorGb) return false
|
||||
if (Date.now() - this.checkedAt < this.checkMs) return this.low
|
||||
this.checkedAt = Date.now()
|
||||
const gb = (await this.free(this.path)) / 1e9
|
||||
const was = this.low
|
||||
this.low = gb < this.floorGb + (was ? MARGIN_GB : 0)
|
||||
if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`)
|
||||
return this.low
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { RuleSet } from '#lib/rules/index'
|
||||
import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
|
||||
import type { AggregatedFile } from '#lib/files/index'
|
||||
import type { EdgeSegment } from '#lib/edge/index'
|
||||
|
||||
export interface MergeResult {
|
||||
files: number
|
||||
completed: number
|
||||
incomplete: number
|
||||
waiting: number
|
||||
/** This round reached the end of the file keys (the next one starts over). */
|
||||
wrapped: boolean
|
||||
}
|
||||
|
||||
/** File keys per round. */
|
||||
const BATCH = 20_000
|
||||
|
||||
/**
|
||||
* Files cut by a range's edge (REBUILD.md, fragments), put together once
|
||||
* the ranges around them are done: a complete file is judged by the rules
|
||||
* like any other and written as a release chunk (with probe candidates); a
|
||||
* file still missing parts while every neighbouring range is done is
|
||||
* incomplete, and dropped (counted on its ranges). Others wait.
|
||||
*/
|
||||
class FragmentMerger {
|
||||
/** Where the next round goes on: waiting files must not take every round's places. */
|
||||
private cursor = ''
|
||||
|
||||
async run(): Promise<MergeResult> {
|
||||
const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
|
||||
const { rows: keys } = await db.rawQuery(
|
||||
`SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
|
||||
WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
|
||||
[this.cursor, BATCH]
|
||||
)
|
||||
// At the end of the keys: the next round starts over.
|
||||
result.wrapped = keys.length < BATCH
|
||||
this.cursor = result.wrapped ? '' : keys.at(-1).file_key
|
||||
if (keys.length === 0) return result
|
||||
const rules = new RuleSet(await RuleSets.current())
|
||||
const rows = await db
|
||||
.from('fragments as f')
|
||||
.join('ranges as r', 'r.id', 'f.range_id')
|
||||
.whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
|
||||
.select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
|
||||
const byKey = new Map<string, typeof rows>()
|
||||
for (const row of rows) {
|
||||
const list = byKey.get(row.file_key)
|
||||
if (list) list.push(row)
|
||||
else byKey.set(row.file_key, [row])
|
||||
}
|
||||
|
||||
const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
|
||||
const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
|
||||
const neighbours = await this.neighbourStatus([...byKey.values()])
|
||||
for (const list of byKey.values()) {
|
||||
result.files++
|
||||
const file = mergeFragments(list.map((r) => r.fragment as Fragment))
|
||||
const ids = list.map((r) => Number(r.id))
|
||||
const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
|
||||
if (file) {
|
||||
completed.push({ file, rangeId, ids })
|
||||
continue
|
||||
}
|
||||
if (list.every((r) => r.status === 'done') && neighbours(list)) {
|
||||
incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
|
||||
} else {
|
||||
result.waiting++
|
||||
}
|
||||
}
|
||||
|
||||
// Completed files, grouped into their releases, judged and written like a batch's.
|
||||
const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
|
||||
for (const { file, rangeId } of completed) {
|
||||
const key = releaseKeyOf(file.segments[0])
|
||||
const entry = byRelease.get(key)
|
||||
if (entry) entry.files.push(file)
|
||||
else byRelease.set(key, { files: [file], rangeId })
|
||||
}
|
||||
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
||||
for (const [key, { files, rangeId }] of byRelease) {
|
||||
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
|
||||
if (verdict.rule !== null) continue
|
||||
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
||||
}
|
||||
await ChunkPartitions.ensure()
|
||||
await db.transaction(async (trx) => {
|
||||
// In key order (batches lock their group; the merger only ever waits on one batch at a time).
|
||||
releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
|
||||
for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
|
||||
const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
|
||||
for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
|
||||
for (const { rangeId, segments } of incomplete) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
|
||||
ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
|
||||
[rangeId, segments]
|
||||
)
|
||||
}
|
||||
})
|
||||
result.completed = completed.length
|
||||
result.incomplete = incomplete.length
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* For each file's fragments: whether the ranges right before and after
|
||||
* their ranges (where a missing part could be) are done, read in one
|
||||
* query for the whole round (per file it was three, ~2,000 files a
|
||||
* minute). A neighbour not planned yet counts as not done (newer articles
|
||||
* come later), except before the group's oldest article.
|
||||
*/
|
||||
private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
|
||||
const span = (list: (typeof lists)[number]) => ({
|
||||
group: Number(list[0].group_id),
|
||||
first: Math.min(...list.map((r) => Number(r.start_id))),
|
||||
last: Math.max(...list.map((r) => Number(r.end_id))),
|
||||
})
|
||||
const spans = lists.map(span)
|
||||
const groups = [...new Set(spans.map((s) => s.group))]
|
||||
// The range holding the article before and the one after, each by the (group_id, start_id) index:
|
||||
// "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
|
||||
FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
|
||||
ORDER BY r.start_id DESC LIMIT 1) b ON true
|
||||
LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
|
||||
ORDER BY r.start_id DESC LIMIT 1) a ON true`,
|
||||
[spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
|
||||
)
|
||||
const byEnd = new Map<string, string>()
|
||||
const byStart = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
|
||||
if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
|
||||
}
|
||||
const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
|
||||
return (list: (typeof lists)[number]) => {
|
||||
const { group, first, last } = span(list)
|
||||
const before = byEnd.get(`${group}:${first - 1}`)
|
||||
const low = lows.get(group)
|
||||
const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
|
||||
return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new FragmentMerger()
|
||||
@@ -0,0 +1,138 @@
|
||||
import type { Overview, XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
|
||||
export interface DateSample {
|
||||
article: number
|
||||
postedAt: Date
|
||||
}
|
||||
|
||||
/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
|
||||
const OLDEST = Date.UTC(1995, 0, 1)
|
||||
const DAY = 86_400_000
|
||||
|
||||
/**
|
||||
* The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
|
||||
* `min` of them; null otherwise (an expired or empty slice).
|
||||
*/
|
||||
export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
|
||||
const latest = now.getTime() + DAY
|
||||
const times = overviews
|
||||
.map(([, fields]) => Date.parse(fields.date ?? ''))
|
||||
.filter((t) => t >= OLDEST && t <= latest)
|
||||
.sort((a, b) => a - b)
|
||||
if (times.length < min) return null
|
||||
return new Date(times[Math.floor(times.length / 2)])
|
||||
}
|
||||
|
||||
/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
|
||||
export function samplePoints(low: number, high: number, count: number, size: number): number[] {
|
||||
const last = Math.max(low, high - size + 1)
|
||||
if (count <= 1 || last <= low) return [low]
|
||||
const points = new Set<number>()
|
||||
for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
|
||||
return [...points].sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
/**
|
||||
* The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
|
||||
* (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
|
||||
* in number order, bar stragglers and old crossposts).
|
||||
*/
|
||||
export function monotone(samples: DateSample[]): DateSample[] {
|
||||
const sorted = [...samples].sort((a, b) => a.article - b.article)
|
||||
const kept = sorted.filter((s, i) => {
|
||||
const next = sorted.slice(i + 1, i + 3)
|
||||
return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
|
||||
})
|
||||
let floor = Number.NEGATIVE_INFINITY
|
||||
return kept.map((s) => {
|
||||
floor = Math.max(floor, s.postedAt.getTime())
|
||||
return { article: s.article, postedAt: new Date(floor) }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
|
||||
* it, the first or last sample's date beyond them; null without samples.
|
||||
*/
|
||||
export function estimateDate(samples: DateSample[], article: number): Date | null {
|
||||
if (samples.length === 0) return null
|
||||
if (article <= samples[0].article) return samples[0].postedAt
|
||||
const last = samples[samples.length - 1]
|
||||
if (article >= last.article) return last.postedAt
|
||||
let lo = 0
|
||||
let hi = samples.length - 1
|
||||
while (hi - lo > 1) {
|
||||
const mid = (lo + hi) >> 1
|
||||
if (samples[mid].article <= article) lo = mid
|
||||
else hi = mid
|
||||
}
|
||||
const a = samples[lo]
|
||||
const b = samples[hi]
|
||||
const f = (article - a.article) / (b.article - a.article)
|
||||
return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
|
||||
}
|
||||
|
||||
/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
|
||||
export function sampleCount(
|
||||
low: number,
|
||||
high: number,
|
||||
every: number,
|
||||
min = 16,
|
||||
max = 1000
|
||||
): number {
|
||||
return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
|
||||
}
|
||||
|
||||
/**
|
||||
* Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
|
||||
* with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
|
||||
* server's span.
|
||||
*/
|
||||
export async function sampleGroup(
|
||||
use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
|
||||
name: string,
|
||||
options: { every: number; size: number; maxFailures?: number },
|
||||
now = new Date()
|
||||
): Promise<{
|
||||
samples: DateSample[]
|
||||
low: number
|
||||
high: number
|
||||
commands: number
|
||||
failed: string[]
|
||||
}> {
|
||||
const info = await use((client) => client.group(name))
|
||||
const samples: DateSample[] = []
|
||||
const failed: string[] = []
|
||||
let commands = 1
|
||||
// Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
|
||||
// minutes; 16 groups never got through on one connection) costs that point, not the group.
|
||||
for (const point of samplePoints(
|
||||
info.low,
|
||||
info.high,
|
||||
sampleCount(info.low, info.high, options.every),
|
||||
options.size
|
||||
)) {
|
||||
if (failed.length >= (options.maxFailures ?? 3)) break
|
||||
try {
|
||||
const date = await use(async (client) => {
|
||||
await client.group(name)
|
||||
commands += 2
|
||||
const slice = medianDate(
|
||||
await client.xover(point, Math.min(point + options.size - 1, info.high)),
|
||||
now
|
||||
)
|
||||
if (slice) return slice
|
||||
commands++
|
||||
return medianDate(
|
||||
await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
|
||||
now
|
||||
)
|
||||
})
|
||||
if (date) samples.push({ article: point, postedAt: date })
|
||||
} catch (error: any) {
|
||||
failed.push(`${point}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
|
||||
}
|
||||
+63
-9
@@ -41,7 +41,12 @@ class Leases {
|
||||
AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
|
||||
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
|
||||
[
|
||||
nodeId,
|
||||
rules.version,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
]
|
||||
)
|
||||
let { rows } = await take(true)
|
||||
if (rows.length === 0) ({ rows } = await take(false))
|
||||
@@ -49,12 +54,38 @@ class Leases {
|
||||
if (!row) return null
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
|
||||
return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
group: group.name,
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
rules,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ranges read to their end whose worker went quiet before finishing them: done. The lease query skips a range past
|
||||
* its end, so one whose finish never came stayed leased for good (a gap range since 2026-10-16 14:03, during the lock
|
||||
* pile-up; LOG.md 2026-10-17 (72)). Its batches are all stored; its spooled chunks go.
|
||||
*/
|
||||
async finishAbandoned(): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'done', done_at = now()
|
||||
WHERE status = 'leased' AND next_id > end_id AND heartbeat_at < now() - make_interval(secs => ?)
|
||||
RETURNING id`,
|
||||
[coordinatorConfig.leaseTtlSeconds]
|
||||
)
|
||||
for (const { id } of rows) await Spool.remove(Number(id))
|
||||
return rows.length
|
||||
}
|
||||
|
||||
async leased(rangeId: number, nodeId: number) {
|
||||
const range = await db.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
@@ -67,8 +98,14 @@ class Leases {
|
||||
/** The worker reached the range's end: done, and its spooled chunks go. */
|
||||
async finish(rangeId: number, nodeId: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
if (Number(range.next_id) <= Number(range.end_id))
|
||||
throw new LeaseLost(
|
||||
`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`
|
||||
)
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
await Spool.remove(rangeId)
|
||||
}
|
||||
|
||||
@@ -79,12 +116,19 @@ class Leases {
|
||||
*/
|
||||
async corrupt(rangeId: number, nodeId: number, from: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
|
||||
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find(
|
||||
(s) => s.from === from
|
||||
)
|
||||
if (span) {
|
||||
await Spool.removeChunk(rangeId, span)
|
||||
if (range.capture) {
|
||||
const group = await db.from('groups').where('id', range.group_id).select('name').first()
|
||||
await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
|
||||
await Corpus.removeChunk(
|
||||
{ group: group.name, start: Number(range.start_id), end: Number(range.end_id) },
|
||||
span
|
||||
)
|
||||
// Not complete in the corpus any more: the corpus fill fetches the chunk again.
|
||||
await db.from('ranges').where('id', rangeId).update({ stored_at: null })
|
||||
}
|
||||
}
|
||||
const plain: number[] = (range.plain_chunks ?? []).map(Number)
|
||||
@@ -92,11 +136,21 @@ class Leases {
|
||||
// refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
|
||||
if (plain.filter((p) => p === from).length >= 2) {
|
||||
// Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'failed', node_id: null, error: `chunk ${from} doesn't inflate, also fetched plain` })
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({
|
||||
status: 'failed',
|
||||
node_id: null,
|
||||
error: `chunk ${from} doesn't inflate, also fetched plain`,
|
||||
})
|
||||
return
|
||||
}
|
||||
// The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
|
||||
await db.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`, [from, rangeId])
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`,
|
||||
[from, rangeId]
|
||||
)
|
||||
}
|
||||
|
||||
/** A stopping worker gives its range back; the next one resumes it. */
|
||||
|
||||
+93
-13
@@ -2,12 +2,18 @@ import env from '#start/env'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import type { NntpShare } from '#services/nntp_shares'
|
||||
|
||||
/** Connections wanted beyond the most in use at once since the last sync (the planner's, a hedge's). */
|
||||
const SHARE_MARGIN = 2
|
||||
|
||||
/**
|
||||
* The coordinator's header connections (REBUILD.md, decision e: workers
|
||||
* have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
|
||||
* that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
|
||||
* replaced on the next use.
|
||||
* replaced on the next use. Sharing the account (share()), at most its
|
||||
* share: over it, connections close as they come free. One left free for
|
||||
* NNTP_IDLE_CLOSE_SECONDS is closed on the next sync.
|
||||
*/
|
||||
class NntpPool {
|
||||
private idle: XoverClient[] = []
|
||||
@@ -17,9 +23,43 @@ class NntpPool {
|
||||
private openedAt = new Map<XoverClient, number>()
|
||||
/** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
|
||||
private retireAt = new Map<XoverClient, number>()
|
||||
/** When an idle connection came free. */
|
||||
private idleSince = new Map<XoverClient, number>()
|
||||
private limit = coordinatorConfig.nntpConnections
|
||||
/** Since the last sync: the most connections in use at once, and whether work waited for one (starved()). */
|
||||
private peak = 0
|
||||
private short = false
|
||||
|
||||
get size() {
|
||||
return coordinatorConfig.nntpConnections
|
||||
return this.limit
|
||||
}
|
||||
|
||||
/** Work that could use another connection didn't get one (the prefetcher with all its loops running). */
|
||||
starved() {
|
||||
this.short = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the share what the pool could use and has open, and takes the share's grant as its size. After work waited:
|
||||
* twice its size (history going on gets its maximum in a few syncs; a tail round, done in seconds, doesn't take
|
||||
* the prober's connections for one); else the most in use at once since the last sync and a margin.
|
||||
*/
|
||||
async share(share: NntpShare) {
|
||||
const want = this.short || this.waiting.length > 0 ? Math.max(this.limit * 2, this.limit + SHARE_MARGIN) : this.peak + SHARE_MARGIN
|
||||
this.short = false
|
||||
this.peak = this.busy.size
|
||||
const granted = await share.sync(want, this.open)
|
||||
const was = this.limit
|
||||
this.limit = granted
|
||||
// Over the share: the idle ones close now, the busy ones once free (release()).
|
||||
while (this.open > this.limit && this.idle.length > 0) this.retire(this.idle.pop()!)
|
||||
this.closeIdle()
|
||||
// Under it with work waiting: connections for it.
|
||||
while (this.waiting.length > 0 && this.open < this.limit) {
|
||||
const next = this.waiting.shift()!
|
||||
this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
}
|
||||
return { was, granted }
|
||||
}
|
||||
|
||||
async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
@@ -44,6 +84,7 @@ class NntpPool {
|
||||
|
||||
private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.busy.add(client)
|
||||
this.peak = Math.max(this.peak, this.busy.size)
|
||||
let broken = false
|
||||
try {
|
||||
return await work(client)
|
||||
@@ -56,9 +97,28 @@ class NntpPool {
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the connections free for nntpIdleCloseMs, the plain one too. */
|
||||
private closeIdle() {
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs <= 0) return
|
||||
const before = Date.now() - idleMs
|
||||
for (const client of this.idle.filter((c) => (this.idleSince.get(c) ?? Infinity) <= before)) {
|
||||
this.idle.splice(this.idle.indexOf(client), 1)
|
||||
this.retire(client)
|
||||
}
|
||||
if (this.plain && this.plainPending === 0 && this.plainUsedAt <= before) {
|
||||
void this.plain.then((c) => c.close()).catch(() => {})
|
||||
this.plain = null
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<XoverClient> {
|
||||
// The one used last: the others stay free long enough to close.
|
||||
const ready = this.idle.pop()
|
||||
if (ready) return ready
|
||||
if (ready) {
|
||||
this.idleSince.delete(ready)
|
||||
return ready
|
||||
}
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
@@ -84,15 +144,12 @@ class NntpPool {
|
||||
}
|
||||
|
||||
private release(client: XoverClient, broken: boolean) {
|
||||
if (this.open > this.limit) {
|
||||
this.retire(client)
|
||||
return
|
||||
}
|
||||
if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
|
||||
this.open--
|
||||
this.openedAt.delete(client)
|
||||
this.retireAt.delete(client)
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing a failed NNTP connection: ${error.message}`)
|
||||
}
|
||||
this.retire(client)
|
||||
// Someone waiting gets a fresh connection.
|
||||
const next = this.waiting.shift()
|
||||
if (next) this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
@@ -100,12 +157,30 @@ class NntpPool {
|
||||
}
|
||||
const next = this.waiting.shift()
|
||||
if (next) next(client)
|
||||
else this.idle.push(client)
|
||||
else {
|
||||
this.idle.push(client)
|
||||
this.idleSince.set(client, Date.now())
|
||||
}
|
||||
}
|
||||
|
||||
private retire(client: XoverClient) {
|
||||
this.open--
|
||||
this.openedAt.delete(client)
|
||||
this.retireAt.delete(client)
|
||||
this.idleSince.delete(client)
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing an NNTP connection: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes every connection, also those in use: their commands fail. */
|
||||
private plain: Promise<XoverClient> | null = null
|
||||
private plainQueue: Promise<unknown> = Promise.resolve()
|
||||
/** Commands queued or running on the plain connection, and when the last one ended. */
|
||||
private plainPending = 0
|
||||
private plainUsedAt = 0
|
||||
|
||||
/**
|
||||
* One connection without XFEATURE COMPRESS GZIP, opened when first needed,
|
||||
@@ -113,6 +188,7 @@ class NntpPool {
|
||||
* cuts short (Leases.corrupt). Its replies are deflated here.
|
||||
*/
|
||||
async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.plainPending++
|
||||
const run = this.plainQueue.then(async () => {
|
||||
this.plain ??= XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
@@ -132,7 +208,10 @@ class NntpPool {
|
||||
throw error
|
||||
}
|
||||
})
|
||||
this.plainQueue = run.catch(() => {})
|
||||
this.plainQueue = run.catch(() => {}).finally(() => {
|
||||
this.plainPending--
|
||||
this.plainUsedAt = Date.now()
|
||||
})
|
||||
return run
|
||||
}
|
||||
|
||||
@@ -143,6 +222,7 @@ class NntpPool {
|
||||
this.busy.clear()
|
||||
this.openedAt.clear()
|
||||
this.retireAt.clear()
|
||||
this.idleSince.clear()
|
||||
this.open = 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/** A share whose process hasn't synced for this long is gone: its connections closed with it. */
|
||||
const STALE_SECONDS = 60
|
||||
/** Syncs one at a time (pg_advisory_xact_lock). */
|
||||
const LOCK = 2_026_101_875
|
||||
|
||||
export interface Share {
|
||||
name: string
|
||||
priority: number
|
||||
/** Kept whatever the others want (as far as the total goes, by priority). */
|
||||
min: number
|
||||
/** What it could use now. */
|
||||
want: number
|
||||
/** Connections it has open (or may have: a lane without one yet counts). */
|
||||
open: number
|
||||
}
|
||||
|
||||
/** What each share should have: its minimum first, then by priority what it wants, up to the total. */
|
||||
export function targets(total: number, shares: Share[]): Map<string, number> {
|
||||
const ordered = [...shares].sort((a, b) => b.priority - a.priority || a.name.localeCompare(b.name))
|
||||
const target = new Map<string, number>()
|
||||
let left = total
|
||||
for (const s of ordered) {
|
||||
const t = Math.min(s.min, left)
|
||||
target.set(s.name, t)
|
||||
left -= t
|
||||
}
|
||||
for (const s of ordered) {
|
||||
const more = Math.min(Math.max(s.want - target.get(s.name)!, 0), left)
|
||||
target.set(s.name, target.get(s.name)! + more)
|
||||
left -= more
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
/**
|
||||
* The connections `self` may have: its target, but no more than the others leave over. Another share holds what it
|
||||
* has open or was granted, whichever is more (it may open up to its grant until its next sync tells it otherwise),
|
||||
* so the account never goes over the total: a share grows only into what the others closed or gave up.
|
||||
*/
|
||||
export function grant(total: number, self: Share, others: (Share & { granted: number })[]): number {
|
||||
const target = targets(total, [self, ...others]).get(self.name)!
|
||||
const held = others.reduce((sum, o) => sum + Math.max(o.open, o.granted), 0)
|
||||
return Math.max(0, Math.min(target, total - held))
|
||||
}
|
||||
|
||||
/**
|
||||
* A process's share of the NNTP account's connections (LOG.md 2026-10-18 (75)): the coordinator and the prober had a
|
||||
* fixed split, and with history paused the coordinator's 38 stood idle while the prober's 8 were its limit. Each
|
||||
* syncs every nntpShares.everyMs (what it wants and has open) and gets what it may open; one wanting less closes
|
||||
* connections as they come free, and the other grows as those show up closed.
|
||||
*/
|
||||
export class NntpShare {
|
||||
granted = 0
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
private priority: number,
|
||||
readonly min: number,
|
||||
readonly max: number
|
||||
) {}
|
||||
|
||||
async sync(want: number, open: number): Promise<number> {
|
||||
const self: Share = { name: this.name, priority: this.priority, min: this.min, want: Math.min(Math.max(want, this.min), this.max), open }
|
||||
const total = coordinatorConfig.nntpShares.total
|
||||
this.granted = await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(?)`, [LOCK])
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT name, priority, min, want, open, granted FROM nntp_shares WHERE name <> ? AND heartbeat_at > now() - make_interval(secs => ?)`,
|
||||
[this.name, STALE_SECONDS]
|
||||
)
|
||||
const granted = grant(total, self, rows)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO nntp_shares (name, priority, min, want, open, granted, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, now())
|
||||
ON CONFLICT (name) DO UPDATE SET priority = excluded.priority, min = excluded.min, want = excluded.want,
|
||||
open = excluded.open, granted = excluded.granted, heartbeat_at = now()`,
|
||||
[this.name, this.priority, this.min, self.want, open, granted]
|
||||
)
|
||||
return granted
|
||||
})
|
||||
return this.granted
|
||||
}
|
||||
|
||||
/** Stopping: its connections are closed, the others needn't wait for the row to go stale. */
|
||||
async leave() {
|
||||
await db.from('nntp_shares').where('name', this.name).delete()
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,28 @@ import { gunzipSync } from 'node:zlib'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
|
||||
const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
|
||||
|
||||
/**
|
||||
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
|
||||
* chunk store: every chunk decoded, its files in name order. A file stored
|
||||
* twice (the same name and first article, e.g. a range read again) is taken
|
||||
* once.
|
||||
* once. A known password (release_passwords) goes into the NZB's meta, where
|
||||
* SABnzbd and NZBGet take it for unpacking.
|
||||
*/
|
||||
class NzbService {
|
||||
async files(releaseId: number): Promise<ChunkFile[]> {
|
||||
const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
/** The release's known passwords, the most trusted first. */
|
||||
async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
|
||||
const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
|
||||
const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
|
||||
return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
|
||||
}
|
||||
|
||||
async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
|
||||
const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
const files = new Map<string, ChunkFile>()
|
||||
for (const chunk of chunks) {
|
||||
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
|
||||
@@ -22,10 +34,11 @@ class NzbService {
|
||||
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
|
||||
if (!release) return null
|
||||
const password = (await this.passwords(releaseId))[0]?.password
|
||||
// A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
|
||||
if (release.source === 'spot') {
|
||||
if (Number(release.source) === SOURCES.spot) {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
|
||||
if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
|
||||
if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
|
||||
}
|
||||
const xml = buildNzb(
|
||||
(await this.files(releaseId)).map((f) => ({
|
||||
@@ -35,10 +48,18 @@ class NzbService {
|
||||
groups: f.groups,
|
||||
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
})),
|
||||
{ title: release.name }
|
||||
{ title: release.name, password }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
}
|
||||
|
||||
/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
|
||||
export function withPassword(xml: string, password: string | undefined): string {
|
||||
if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
|
||||
const meta = `<meta type="password">${password.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')}</meta>`
|
||||
if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
|
||||
return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
|
||||
}
|
||||
|
||||
export default new NzbService()
|
||||
@@ -0,0 +1,53 @@
|
||||
import NzbService from '#services/nzb_service'
|
||||
|
||||
/** A file a par2 set describes (par2_files). */
|
||||
export interface SetFile {
|
||||
name: string
|
||||
length: number
|
||||
hash16k: string
|
||||
}
|
||||
|
||||
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
||||
|
||||
/**
|
||||
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
||||
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
||||
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
||||
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
||||
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
||||
*/
|
||||
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
||||
const names = new Set(files.map((f) => base(f.name)))
|
||||
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
||||
const byName = set.filter((f) => names.has(base(f.name))).length
|
||||
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
||||
if (byName > 0) return false
|
||||
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
||||
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
||||
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
||||
* a transaction that moved chunks (a merge) reads them as moved.
|
||||
*/
|
||||
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const { rows: releases } = await client.rawQuery(
|
||||
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
let decided = 0
|
||||
for (const { id } of releases) {
|
||||
const { rows: set } = await client.rawQuery(
|
||||
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (set.length === 0) continue
|
||||
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
||||
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
||||
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
||||
decided++
|
||||
}
|
||||
return decided
|
||||
}
|
||||
+243
-60
@@ -2,8 +2,12 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { par2ReleaseName } from '#lib/release/index'
|
||||
import { looksRandom } from '#lib/release/random'
|
||||
import { SidecarFolder } from '#services/sidecar_folder'
|
||||
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
|
||||
import { randomPoster } from '#lib/release/identity'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||
@@ -15,21 +19,45 @@ const MAX_HOLDERS = 4
|
||||
/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
|
||||
const MAX_FILES_PER_SET_FILE = 3
|
||||
const SLACK_FILES = 10
|
||||
/**
|
||||
* A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
|
||||
* it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
|
||||
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
|
||||
*/
|
||||
const COMMON_FILE = 20
|
||||
/**
|
||||
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
|
||||
* NAME_CANDIDATES of them (a poster's uploads of a day).
|
||||
*/
|
||||
const NAME_SPREAD_MS = 12 * 3600 * 1000
|
||||
const NAME_CANDIDATES = 200
|
||||
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
|
||||
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
|
||||
/** Between rounds. */
|
||||
const ROUND_MS = 60_000
|
||||
/** A round looks back this far past the previous one's start (rows committed while it ran). */
|
||||
/** A window looks back this far past the previous one's end (rows committed after it was read). */
|
||||
const OVERLAP_MS = 60_000
|
||||
/**
|
||||
* A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
|
||||
* two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
|
||||
*/
|
||||
const WINDOW_MS = 60 * 60_000
|
||||
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
|
||||
const FOLD_EVERY_MS = 5 * 60_000
|
||||
/** Between par2_files prunes (complete and judged sets' file lists). */
|
||||
const PRUNE_EVERY_MS = 60 * 60_000
|
||||
/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
|
||||
/** A prune's share of the merger's time: the rest goes on in the next one. */
|
||||
const PRUNE_BUDGET_MS = 2 * 60_000
|
||||
/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
|
||||
const CURSOR = 'merger'
|
||||
|
||||
export interface MergeRound {
|
||||
sets: number
|
||||
merges: number
|
||||
releases: number
|
||||
/** The name pass: merges, releases in them. */
|
||||
nameMerges: number
|
||||
nameReleases: number
|
||||
/** The inner-name pass: stems checked, merges, releases in them. */
|
||||
stems: number
|
||||
stemMerges: number
|
||||
@@ -44,13 +72,33 @@ export interface MergeRound {
|
||||
*/
|
||||
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
|
||||
|
||||
/**
|
||||
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
|
||||
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
|
||||
*/
|
||||
export function fileKey(name: string): string {
|
||||
let n = name.replace(/^.*[\\/]/, '')
|
||||
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
|
||||
const read = Buffer.from(n, 'latin1').toString('utf8')
|
||||
if (!read.includes('\ufffd')) n = read
|
||||
}
|
||||
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
|
||||
const HOLDERS = `SELECT c.release_id FROM (
|
||||
SELECT l.release_id, count(*) OVER () AS holders
|
||||
FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
|
||||
) c WHERE c.holders <= ${COMMON_FILE}`
|
||||
|
||||
interface Member {
|
||||
id: number
|
||||
release_key: string
|
||||
group_id: number
|
||||
files: number
|
||||
posted_at: Date | null
|
||||
readable: boolean
|
||||
poster_id?: number | null
|
||||
/** The poster's text (joined from posters) where a query needs it. */
|
||||
poster?: string | null
|
||||
file_total?: number | null
|
||||
}
|
||||
@@ -95,11 +143,12 @@ export class Par2Merger {
|
||||
// First, at start too: a prune needn't wait for a long round.
|
||||
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
|
||||
prunedAt = Date.now()
|
||||
const pruned = await new Par2Pruner().round()
|
||||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
|
||||
const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
|
||||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
|
||||
}
|
||||
const round = await this.round()
|
||||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||||
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
|
||||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||||
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||||
} catch (error: any) {
|
||||
@@ -109,14 +158,18 @@ export class Par2Merger {
|
||||
}
|
||||
}
|
||||
|
||||
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
|
||||
/**
|
||||
* The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
|
||||
* lone inspection), a window at a time.
|
||||
*/
|
||||
async round(): Promise<MergeRound> {
|
||||
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
|
||||
// A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
|
||||
const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
|
||||
const end = new Date(now[0].at)
|
||||
if (!this.since) {
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
this.since = saved ? new Date(saved.at) : new Date(0)
|
||||
}
|
||||
const since = new Date(this.since.getTime() - OVERLAP_MS)
|
||||
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
|
||||
// par2 pass (hours when that starts over).
|
||||
const folded = await this.folder.round()
|
||||
@@ -130,57 +183,170 @@ export class Par2Merger {
|
||||
foldedAt = Date.now()
|
||||
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||||
}
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT set_id FROM par2_sets WHERE created_at >= ?
|
||||
UNION
|
||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
|
||||
[since, since]
|
||||
)
|
||||
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
for (const { set_id: setId } of rows) {
|
||||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
while (this.since < end) {
|
||||
if (this.stopped) return round
|
||||
await foldAgain()
|
||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.merges++
|
||||
round.releases += merged + 1
|
||||
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
|
||||
const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
|
||||
if (from) {
|
||||
const { touched, sets } = await this.setsToMerge(from, until)
|
||||
round.sets += touched
|
||||
for (const setId of sets) {
|
||||
if (this.stopped) return round
|
||||
await foldAgain()
|
||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.merges++
|
||||
round.releases += merged + 1
|
||||
}
|
||||
}
|
||||
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
|
||||
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
|
||||
// 2026-10-23 (97)).
|
||||
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
|
||||
for (const { set_id: setId } of found) {
|
||||
if (this.stopped) return round
|
||||
const merged = await this.mergeByName(setId.trim()).catch((error) => {
|
||||
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.nameMerges++
|
||||
round.nameReleases += merged + 1
|
||||
}
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
|
||||
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
[from, until]
|
||||
)
|
||||
for (const { stem } of stems) {
|
||||
if (this.stopped) return round
|
||||
if (!looksRandom(stem)) continue
|
||||
round.stems++
|
||||
const merged = await this.mergeStem(stem).catch((error) => {
|
||||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.stemMerges++
|
||||
round.stemReleases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only after a whole window: a restart halfway goes on from the window before.
|
||||
this.since = until
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, this.since]
|
||||
)
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i WHERE i.kind = 'lone' AND i.created_at >= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
[since]
|
||||
)
|
||||
for (const { stem } of stems) {
|
||||
if (this.stopped) return round
|
||||
if (!looksRandom(stem)) continue
|
||||
round.stems++
|
||||
const merged = await this.mergeStem(stem).catch((error) => {
|
||||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.stemMerges++
|
||||
round.stemReleases += merged + 1
|
||||
}
|
||||
}
|
||||
// Only after a whole round: a restart halfway goes on from the round before.
|
||||
this.since = now[0].at
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, this.since]
|
||||
)
|
||||
return round
|
||||
}
|
||||
|
||||
/** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
|
||||
private async firstNew(from: Date, end: Date): Promise<Date | null> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT least(
|
||||
(SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
|
||||
[from, from, from]
|
||||
)
|
||||
const at = rows[0].at ? new Date(rows[0].at) : null
|
||||
return at && at <= end ? at : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
|
||||
* change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
|
||||
* hour's 147k): mergeSet needn't look at them.
|
||||
*/
|
||||
private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH touched AS MATERIALIZED (
|
||||
SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
|
||||
UNION
|
||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
|
||||
WHERE h.created_at >= ? AND h.created_at <= ?
|
||||
)
|
||||
SELECT (SELECT count(*) FROM touched) AS touched, array(
|
||||
SELECT t.set_id FROM touched t
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM (
|
||||
SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
|
||||
UNION ALL
|
||||
SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
|
||||
) u HAVING count(DISTINCT u.release_id) >= 2)
|
||||
) AS sets`,
|
||||
[from, until, from, until]
|
||||
)
|
||||
return { touched: Number(rows[0].touched), sets: rows[0].sets }
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
|
||||
* and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
|
||||
* sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
|
||||
* again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
|
||||
*/
|
||||
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
|
||||
const { rows: all } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
|
||||
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
|
||||
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
|
||||
[setId]
|
||||
)
|
||||
const anchor: Member | undefined = all[0]
|
||||
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
|
||||
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
|
||||
const holders = all.filter(
|
||||
(h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
|
||||
)
|
||||
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||||
if (described.length === 0) return null
|
||||
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
|
||||
const at = new Date(anchor.posted_at).getTime()
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
|
||||
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
|
||||
AND r.files <= ?
|
||||
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
|
||||
// More files than the set describes, with its sidecars, can't all be named in it.
|
||||
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
|
||||
)
|
||||
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
|
||||
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
|
||||
for (const c of candidates) {
|
||||
const own = await NzbService.files(Number(c.id), trx)
|
||||
const content = own.filter((f) => !besideSet(f.name))
|
||||
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
|
||||
found.set(String(c.id), c)
|
||||
files += Number(c.files)
|
||||
}
|
||||
if (found.size < 2) return null
|
||||
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
|
||||
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
|
||||
return found
|
||||
}
|
||||
|
||||
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
|
||||
async mergeByName(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const found = await this.nameMatches(trx, setId)
|
||||
return found ? this.merge(trx, found, setId, 'name') : 0
|
||||
})
|
||||
}
|
||||
|
||||
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
|
||||
async mergeStem(stem: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
|
||||
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
|
||||
FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
|
||||
[stem]
|
||||
@@ -192,7 +358,7 @@ export class Par2Merger {
|
||||
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
|
||||
const near = (r: Member) =>
|
||||
r.group_id === anchor.group_id &&
|
||||
(r.poster ?? '') === (anchor.poster ?? '') &&
|
||||
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
|
||||
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
|
||||
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const found = new Map<string, Member>()
|
||||
@@ -210,7 +376,7 @@ export class Par2Merger {
|
||||
async mergeSet(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows: holders } = await trx.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
WHERE s.set_id = ? ORDER BY r.id`,
|
||||
[setId]
|
||||
)
|
||||
@@ -219,8 +385,8 @@ export class Par2Merger {
|
||||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||||
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const { rows: matches } = await trx.rawQuery(
|
||||
`SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
|
||||
FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
|
||||
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
|
||||
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
|
||||
WHERE f.set_id = ?`,
|
||||
[setId]
|
||||
)
|
||||
@@ -250,7 +416,7 @@ export class Par2Merger {
|
||||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||||
*/
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
|
||||
{
|
||||
// Locked in id order (the batch store and other rounds take the same rows).
|
||||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||||
@@ -262,12 +428,28 @@ export class Par2Merger {
|
||||
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
|
||||
SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||||
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||||
[target.id, key, via, others]
|
||||
)
|
||||
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
|
||||
// isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
|
||||
await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
|
||||
AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
|
||||
OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
|
||||
[others, target.id, others]
|
||||
)
|
||||
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
|
||||
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||||
}
|
||||
// An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[target.id]
|
||||
)
|
||||
// The members' 7z tails together: those below the highest volume go.
|
||||
await pruneSevenZipTails(trx, [target.id])
|
||||
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
|
||||
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
|
||||
await trx.rawQuery(
|
||||
@@ -282,25 +464,26 @@ export class Par2Merger {
|
||||
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
|
||||
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
|
||||
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
|
||||
first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
|
||||
first_seen_at = least(t.first_seen_at, s.first_seen_at),
|
||||
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
|
||||
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
|
||||
name = coalesce(nullif(?, ''), t.name)
|
||||
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
|
||||
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
|
||||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
|
||||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
|
||||
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
|
||||
max(file_total) AS file_total, max(inner_name) AS inner_name
|
||||
FROM releases WHERE id = ANY(?)) s
|
||||
WHERE t.id = ?`,
|
||||
[name ?? '', others, target.id]
|
||||
)
|
||||
// Completeness as the settler decides it, once settled.
|
||||
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
[target.id]
|
||||
)
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
||||
await decidePar2Completeness(trx, [target.id])
|
||||
return others.length
|
||||
}
|
||||
}
|
||||
|
||||
+75
-43
@@ -1,75 +1,107 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
||||
const SETTLED_MS = 2 * 86400_000
|
||||
/** Sets per statement. */
|
||||
const BATCH = 20_000
|
||||
/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
|
||||
/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
|
||||
const CURSOR = 'par2-prune'
|
||||
/** The merger's row in worker_cursors: the end of its last whole window. */
|
||||
const MERGER = 'merger'
|
||||
/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
|
||||
const MERGER_OVERLAP_MS = 60_000
|
||||
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
|
||||
const JUNK = ['dead', 'fake', 'repeated']
|
||||
/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
|
||||
const MERGE_KINDS = ['lone', 'par2']
|
||||
|
||||
export interface PruneRound {
|
||||
complete: number
|
||||
judged: number
|
||||
final: number
|
||||
rows: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
|
||||
* 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
|
||||
* done when
|
||||
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
|
||||
* matches file hashes to releases and names an unreadable upload once; it's done when
|
||||
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
|
||||
* set (all of it merged into one);
|
||||
* - judged: every release holding it has a junk verdict (dead, fake, repeated).
|
||||
* - judged: every release holding it has a junk verdict (dead, fake, repeated);
|
||||
* - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
|
||||
* it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
|
||||
* and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
|
||||
* describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
|
||||
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
|
||||
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
||||
* The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
|
||||
* of a run, its ranges' probes not done yet) is looked at again on the next pass.
|
||||
*/
|
||||
export class Par2Pruner {
|
||||
async round(): Promise<PruneRound> {
|
||||
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
let since: Date = saved ? new Date(saved.at) : new Date(0)
|
||||
const until = new Date(Date.now() - SETTLED_MS)
|
||||
/** Sets per batch (BATCH; smaller in the tests). */
|
||||
constructor(private batch = BATCH) {}
|
||||
|
||||
/**
|
||||
* Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
|
||||
* it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
|
||||
* 2026-10-17 (72)).
|
||||
*/
|
||||
async round(budgetMs = Infinity): Promise<PruneRound> {
|
||||
const deadline = Date.now() + budgetMs
|
||||
const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after: string = cursors[0]?.key ?? ''
|
||||
// Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
|
||||
// still to come. Hashes since the merger's last window: it hasn't matched them yet.
|
||||
const { rows: marks } = await db.rawQuery(
|
||||
`SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
|
||||
coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
|
||||
now() - make_interval(secs => ?) AS settled`,
|
||||
[MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
|
||||
)
|
||||
const { probed, merged, settled } = marks[0]
|
||||
for (;;) {
|
||||
const { rows: sets } = await db.rawQuery(
|
||||
`SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
|
||||
HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
|
||||
[since, until, BATCH]
|
||||
`WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
|
||||
recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
|
||||
unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
|
||||
SELECT b.set_id, CASE
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
|
||||
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||
WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
|
||||
WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
|
||||
r.settled_at IS NULL
|
||||
OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
|
||||
AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
|
||||
THEN 'final' END AS reason
|
||||
FROM batch b ORDER BY b.set_id`,
|
||||
[after, this.batch, probed, merged, settled, JUNK, settled]
|
||||
)
|
||||
if (sets.length === 0) break
|
||||
const ids = sets.map((s: { set_id: string }) => s.set_id)
|
||||
const { rows: done } = await db.rawQuery(
|
||||
`SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
|
||||
WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
|
||||
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||
WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
|
||||
[ids]
|
||||
)
|
||||
round.complete += done.length
|
||||
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
|
||||
since = sets.at(-1).at
|
||||
const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
|
||||
for (const reason of ['complete', 'judged', 'final'] as const) {
|
||||
const ids = done(reason)
|
||||
round[reason] += ids.length
|
||||
// The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
|
||||
if (reason !== 'judged' && ids.length > 0) {
|
||||
const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
|
||||
await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
|
||||
}
|
||||
round.rows += await this.drop(ids)
|
||||
}
|
||||
// Past the last set: the next batch starts the next pass.
|
||||
after = sets.length < this.batch ? '' : sets.at(-1).set_id
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, since]
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, after]
|
||||
)
|
||||
if (sets.length < BATCH) break
|
||||
if (after === '' || Date.now() > deadline) break
|
||||
}
|
||||
// Judged: no cursor (verdicts come and go); only sets with rows left.
|
||||
const { rows: judged } = await db.rawQuery(
|
||||
`SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
|
||||
WHERE v.rule = ANY(?)
|
||||
AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
|
||||
WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
|
||||
[JUNK, JUNK]
|
||||
)
|
||||
round.judged = judged.length
|
||||
const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
|
||||
for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
|
||||
return round
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { explored } from '#lib/rules/index'
|
||||
import { usableName, type Inspection } from '#lib/probe/index'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
|
||||
/** pending_releases.status (migration 6400). */
|
||||
export const PENDING = {
|
||||
waiting: 0,
|
||||
named: 1,
|
||||
discarded: 2,
|
||||
exhausted: 3,
|
||||
naming: 4,
|
||||
taken: 9,
|
||||
} as const
|
||||
|
||||
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
|
||||
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
|
||||
|
||||
export interface PendingTarget {
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
}
|
||||
|
||||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||||
export interface PendingProbe {
|
||||
/** pending_releases.release_key as hex. */
|
||||
releaseKey: string
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
tool: string
|
||||
groupId: number
|
||||
poster: string
|
||||
/** Targets probed before this one. */
|
||||
tries: number
|
||||
targets: number
|
||||
}
|
||||
|
||||
export type PendingOutcome = {
|
||||
probe: PendingProbe
|
||||
status: 'done' | 'missing' | 'error' | 'waiting'
|
||||
inspection?: Inspection
|
||||
}
|
||||
|
||||
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||||
const rank = (kind: string) => {
|
||||
const i = (EFFORT as readonly string[]).indexOf(kind)
|
||||
return i < 0 ? EFFORT.length : i
|
||||
}
|
||||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
|
||||
* signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
|
||||
*/
|
||||
class PendingLadder {
|
||||
/** A previous prober's claims back in the queue. */
|
||||
async release(): Promise<number> {
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ?`,
|
||||
[PENDING.waiting, PENDING.taken]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
}
|
||||
|
||||
/** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
|
||||
async claim(n: number): Promise<PendingProbe[]> {
|
||||
if (n <= 0) return []
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
|
||||
[PENDING.taken, PENDING.waiting, n]
|
||||
)
|
||||
return rows.map((r: any) => {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
|
||||
* end of the ladder (status 3); the finished climbs counted on their signatures.
|
||||
*/
|
||||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||||
if (outcomes.length === 0) return
|
||||
const rows = outcomes.map((o) => {
|
||||
const p = o.probe
|
||||
if (o.status === 'waiting') {
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries: p.tries,
|
||||
named: null,
|
||||
inspection: null,
|
||||
final: null,
|
||||
}
|
||||
}
|
||||
const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const tries = p.tries + 1
|
||||
const inspection = {
|
||||
kind: p.kind,
|
||||
status: o.status,
|
||||
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
|
||||
}
|
||||
if (name)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.naming,
|
||||
tries: 0,
|
||||
named: name,
|
||||
inspection,
|
||||
final: 'named',
|
||||
}
|
||||
if (tries >= p.targets)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.exhausted,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: 'exhausted',
|
||||
}
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: null,
|
||||
}
|
||||
})
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||||
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
|
||||
)
|
||||
// The signatures: the poster's own and the tool's and group's together.
|
||||
const counts = new Map<
|
||||
string,
|
||||
{ tool: string; group_id: number; poster: string; probed: number; named: number }
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const final = rows[i].final
|
||||
if (!final) continue
|
||||
for (const poster of [o.probe.poster, '']) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
poster,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
}
|
||||
entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
}
|
||||
if (counts.size === 0) return
|
||||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
|
||||
async discard(limit = 10_000): Promise<number> {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
|
||||
WHERE p.status = ? AND EXISTS (
|
||||
SELECT 1 FROM pending_signatures s
|
||||
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
|
||||
LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
)
|
||||
const keys = rows
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
if (keys.length === 0) return 0
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
|
||||
[PENDING.discarded, PENDING.waiting, keys]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
}
|
||||
}
|
||||
|
||||
export default new PendingLadder()
|
||||
@@ -0,0 +1,222 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import { unzip } from 'node:zlib'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
|
||||
import { identityKey } from '#lib/release/keys'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import Corpus from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { PENDING } from '#services/pending_ladder'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
|
||||
interface NamedRow {
|
||||
release_key: Buffer
|
||||
identity_key: Buffer
|
||||
group_id: number
|
||||
named: string
|
||||
first_article: number
|
||||
last_article: number
|
||||
file_total: number | null
|
||||
tries: number
|
||||
}
|
||||
|
||||
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
|
||||
const MAX_MOVES = 3
|
||||
|
||||
/**
|
||||
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
|
||||
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
|
||||
* connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
|
||||
* release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
|
||||
* candidates, which the prober answers from probe_cache. Runs inside the prober.
|
||||
*/
|
||||
export class PendingNamer {
|
||||
private client: XoverClient | null = null
|
||||
private usedAt = 0
|
||||
|
||||
constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
|
||||
|
||||
async run(stopped: () => boolean) {
|
||||
while (!stopped()) {
|
||||
let rows: NamedRow[] = []
|
||||
try {
|
||||
rows = await this.take(20)
|
||||
for (const row of rows) {
|
||||
if (stopped()) break
|
||||
try {
|
||||
await this.move(row)
|
||||
} catch (error: any) {
|
||||
logger.warn(
|
||||
`Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
|
||||
[row.release_key]
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Naming: ${error.message}`)
|
||||
}
|
||||
if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
}
|
||||
if (rows.length === 0)
|
||||
for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
|
||||
}
|
||||
this.client?.close()
|
||||
this.client = null
|
||||
}
|
||||
|
||||
/** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
|
||||
async take(n: number): Promise<NamedRow[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
|
||||
FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
|
||||
[PENDING.naming, MAX_MOVES, n]
|
||||
)
|
||||
return rows.map((r: any) => ({
|
||||
...r,
|
||||
group_id: Number(r.group_id),
|
||||
first_article: Number(r.first_article),
|
||||
last_article: Number(r.last_article),
|
||||
tries: Number(r.tries),
|
||||
}))
|
||||
}
|
||||
|
||||
/** One named release into the catalog. */
|
||||
async move(row: NamedRow): Promise<void> {
|
||||
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
|
||||
const range = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.where('start_id', '<=', row.first_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('id')
|
||||
.first()
|
||||
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
|
||||
const text = await this.overviewText(group.name, row)
|
||||
const rules = await RuleSets.current()
|
||||
const ix = new RangeIndexer({
|
||||
group: group.name,
|
||||
start: row.first_article,
|
||||
end: row.last_article,
|
||||
rules: { ...rules, samplePercent: 0 },
|
||||
idleChunks: 0,
|
||||
edgeArticles: 0,
|
||||
})
|
||||
ix.addText(text, row.last_article + 1)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
|
||||
if (!found)
|
||||
throw new Error(
|
||||
`its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
|
||||
)
|
||||
// The catalog row under the pending key (the identity, the group, the first article the edge saw).
|
||||
const chunk: ReleaseChunk = {
|
||||
...found,
|
||||
name: row.named,
|
||||
source: 'weak-stem',
|
||||
readable: true,
|
||||
finished: true,
|
||||
edge: false,
|
||||
firstArticle: row.first_article,
|
||||
fileTotal: row.file_total ?? found.fileTotal,
|
||||
tool: undefined,
|
||||
}
|
||||
const candidates = batch.candidates
|
||||
.filter((c) => c.releaseKey === found.key)
|
||||
.map((c) => ({ ...c, upload: found.upload }))
|
||||
await db.transaction(async (trx) => {
|
||||
const blockId = await BatchStore.blockOf(trx, range)
|
||||
await BatchStore.writeReleases(
|
||||
trx,
|
||||
{ groupId: row.group_id, blockId },
|
||||
row.first_article,
|
||||
[chunk],
|
||||
candidates
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
|
||||
[row.named.slice(0, 255), row.release_key]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
|
||||
[PENDING.named, row.release_key]
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
|
||||
private async overviewText(groupName: string, row: NamedRow): Promise<string> {
|
||||
if (this.corpusDir) {
|
||||
const stored = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.whereNotNull('stored_at')
|
||||
.where('start_id', '<=', row.last_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('start_id')
|
||||
const parts: string[] = []
|
||||
let covered = true
|
||||
for (const r of stored) {
|
||||
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
for (
|
||||
let from =
|
||||
range.start +
|
||||
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
|
||||
from <= Math.min(row.last_article, range.end);
|
||||
from += size
|
||||
) {
|
||||
const raw = await Corpus.read(
|
||||
range,
|
||||
{ from, to: Math.min(from + size - 1, range.end) },
|
||||
this.corpusDir
|
||||
)
|
||||
if (!raw) {
|
||||
covered = false
|
||||
break
|
||||
}
|
||||
parts.push((await inflate(raw)).toString('latin1'))
|
||||
}
|
||||
if (!covered) break
|
||||
}
|
||||
if (
|
||||
covered &&
|
||||
parts.length > 0 &&
|
||||
stored.length > 0 &&
|
||||
Number(stored[0].start_id) <= row.first_article &&
|
||||
Number(stored[stored.length - 1].end_id) >= row.last_article
|
||||
) {
|
||||
return parts.join('')
|
||||
}
|
||||
}
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: true,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
this.usedAt = Date.now()
|
||||
try {
|
||||
await this.client.group(groupName)
|
||||
return (await this.client.xoverRaw(row.first_article, row.last_article)).toString('latin1')
|
||||
} catch (error) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
+157
-28
@@ -3,7 +3,8 @@ import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import Corpus, { type CorpusFilter } from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
|
||||
|
||||
@@ -72,7 +73,13 @@ export class Prefetcher {
|
||||
private writingNow = 0
|
||||
private sampler?: NodeJS.Timeout
|
||||
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
|
||||
private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
|
||||
private inFlight = new Set<{
|
||||
rangeId: number
|
||||
from: number
|
||||
to: number
|
||||
since: number
|
||||
client: XoverClient
|
||||
}>()
|
||||
/** Capture ranges found complete in the corpus (not looked at again). */
|
||||
private stored = new Set<number>()
|
||||
|
||||
@@ -131,6 +138,8 @@ export class Prefetcher {
|
||||
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
|
||||
.finally(() => this.running--)
|
||||
}
|
||||
// Work left with every loop running: the pool's share could be bigger.
|
||||
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
|
||||
}
|
||||
|
||||
stop() {
|
||||
@@ -145,6 +154,9 @@ export class Prefetcher {
|
||||
/** One connection's loop: chunks until there is nothing left to fetch. */
|
||||
private async work() {
|
||||
while (!this.stopped) {
|
||||
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
|
||||
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
|
||||
if (this.running > this.loops) return
|
||||
const next = this.take()
|
||||
if (!next) return
|
||||
const { target, span } = next
|
||||
@@ -157,10 +169,14 @@ export class Prefetcher {
|
||||
}
|
||||
const corpusRange = { group: target.group, start: target.start, end: target.end }
|
||||
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
|
||||
let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||||
let data =
|
||||
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||||
const stored = data !== null
|
||||
try {
|
||||
if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
|
||||
if (!data)
|
||||
data = target.plain.has(span.from)
|
||||
? await NntpPool.usePlain(fetch)
|
||||
: await this.fetchHedged(target.id, target.group, span)
|
||||
} finally {
|
||||
this.fetchingNow--
|
||||
}
|
||||
@@ -170,7 +186,8 @@ export class Prefetcher {
|
||||
this.stats.chunks++
|
||||
this.stats.articles += span.to - span.from + 1
|
||||
this.stats.fetchMs += Date.now() - fetchedAt
|
||||
if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
|
||||
if (target.capture)
|
||||
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
|
||||
}
|
||||
if (target.corpusOnly) {
|
||||
this.stats.captured++
|
||||
@@ -187,7 +204,8 @@ export class Prefetcher {
|
||||
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
|
||||
[target.id]
|
||||
)
|
||||
if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||||
if (Number(rows[0]?.spooled_chunks) >= target.total)
|
||||
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||||
} finally {
|
||||
this.writingNow--
|
||||
this.stats.writeMs += Date.now() - writtenAt
|
||||
@@ -209,7 +227,10 @@ export class Prefetcher {
|
||||
*/
|
||||
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
|
||||
let age: number | undefined
|
||||
const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
|
||||
const request = (
|
||||
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
|
||||
first: boolean
|
||||
): Request<Buffer> | null => {
|
||||
let client: XoverClient | undefined
|
||||
let cancelled = false
|
||||
const done = start(async (c) => {
|
||||
@@ -241,7 +262,9 @@ export class Prefetcher {
|
||||
data = await c.xoverRaw(span.from, span.to)
|
||||
} catch (error) {
|
||||
if (!late) throw error
|
||||
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
|
||||
throw new Error(
|
||||
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
|
||||
)
|
||||
} finally {
|
||||
clearInterval(watch)
|
||||
this.inFlight.delete(entry)
|
||||
@@ -284,7 +307,8 @@ export class Prefetcher {
|
||||
|
||||
/** The longest-running request: range, chunk, how long, how much came in. */
|
||||
private oldest() {
|
||||
let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||||
let oldest:
|
||||
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||||
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
|
||||
if (!oldest) return 'none in flight'
|
||||
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
|
||||
@@ -309,7 +333,12 @@ export class Prefetcher {
|
||||
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
|
||||
if (chunks > 0) {
|
||||
const ages = byAge
|
||||
.map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
|
||||
.map((b, i) =>
|
||||
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
|
||||
'– min',
|
||||
'+ min'
|
||||
)
|
||||
)
|
||||
.join(', ')
|
||||
logger.info(
|
||||
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
|
||||
@@ -335,13 +364,18 @@ export class Prefetcher {
|
||||
}
|
||||
|
||||
private peek(): boolean {
|
||||
return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
|
||||
return this.targets.some(
|
||||
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether the target is under its range's and its group's limit of connections. */
|
||||
private open(target: Target) {
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||||
return (
|
||||
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
|
||||
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||||
)
|
||||
}
|
||||
|
||||
private count(target: Target, by: number) {
|
||||
@@ -372,26 +406,58 @@ export class Prefetcher {
|
||||
this.sizedAt = Date.now()
|
||||
this.spoolBytes = await Spool.size()
|
||||
}
|
||||
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
|
||||
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
|
||||
const mine = (q: any) =>
|
||||
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
|
||||
const rows = (q: any) =>
|
||||
mine(
|
||||
q
|
||||
.from('ranges as r')
|
||||
.join('groups as g', 'g.id', 'r.group_id')
|
||||
.select(
|
||||
'r.id',
|
||||
'g.name as group',
|
||||
'r.start_id',
|
||||
'r.end_id',
|
||||
'r.next_id',
|
||||
'r.plain_chunks',
|
||||
'r.capture'
|
||||
)
|
||||
)
|
||||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
|
||||
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
|
||||
const ahead = Math.max(
|
||||
coordinatorConfig.spool.minAhead,
|
||||
coordinatorConfig.spool.aheadPerLease * leased.length
|
||||
)
|
||||
const started = await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', '>', 0)
|
||||
.whereNull('r.spooled_at')
|
||||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await mine(
|
||||
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
|
||||
).count('* as ready')
|
||||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||||
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
|
||||
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
|
||||
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
|
||||
const { perRange } = coordinatorConfig.spool
|
||||
const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||||
const short =
|
||||
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||||
const want = Math.max(ahead - Number(ready), short)
|
||||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||||
const full = await CatalogSpace.full()
|
||||
const fresh = room && want > 0
|
||||
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
|
||||
: []
|
||||
const fresh =
|
||||
room && want > 0
|
||||
? await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', 0)
|
||||
.whereRaw('r.next_id <= r.end_id')
|
||||
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
|
||||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
.limit(want)
|
||||
: []
|
||||
const targets: Target[] = []
|
||||
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
|
||||
const start = Number(row.start_id)
|
||||
@@ -399,39 +465,102 @@ export class Prefetcher {
|
||||
const total = Spool.chunksOf({ start, end }).length
|
||||
const missing: ChunkSpan[] = []
|
||||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||||
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
|
||||
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span)))
|
||||
missing.push(span)
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
|
||||
targets.push({
|
||||
id: Number(row.id),
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
capture: row.capture === true,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
|
||||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
|
||||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
|
||||
targets.push(target)
|
||||
return targets
|
||||
}
|
||||
|
||||
/** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
|
||||
private filterRules: { at: number; filter: CorpusFilter } | null = null
|
||||
|
||||
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
|
||||
private async corpusFilter(): Promise<CorpusFilter | undefined> {
|
||||
if (!coordinatorConfig.corpus.filter) return undefined
|
||||
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
|
||||
const rules = await RuleSets.current()
|
||||
this.filterRules = {
|
||||
at: Date.now(),
|
||||
filter: { ruleVersion: rules.version, priorities: rules.priorities },
|
||||
}
|
||||
}
|
||||
return this.filterRules.filter
|
||||
}
|
||||
|
||||
/**
|
||||
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
|
||||
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
|
||||
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
|
||||
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
|
||||
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
|
||||
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
|
||||
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
|
||||
* gets stored_at and isn't read again.
|
||||
*/
|
||||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||||
const { dir, ahead } = coordinatorConfig.corpus
|
||||
if (!dir || ahead <= 0) return []
|
||||
const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
const ofGroup =
|
||||
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||||
const groups = new Map<string, number>()
|
||||
const candidates = await rows(db.query())
|
||||
.where('r.capture', true)
|
||||
.whereNull('r.stored_at')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
|
||||
.orWhere('r.status', 'done')
|
||||
)
|
||||
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
|
||||
.limit(ahead * 16)
|
||||
const targets: Target[] = []
|
||||
for (const row of candidates) {
|
||||
if (targets.length >= ahead) break
|
||||
const id = Number(row.id)
|
||||
if (taken.has(id) || this.stored.has(id)) continue
|
||||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const range = { group: row.group, start, end }
|
||||
if (await Corpus.complete(range)) {
|
||||
this.stored.add(id)
|
||||
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
|
||||
continue
|
||||
}
|
||||
const chunks = Spool.chunksOf({ start, end })
|
||||
const missing: ChunkSpan[] = []
|
||||
for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
|
||||
for (const span of chunks)
|
||||
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
|
||||
missing.push(span)
|
||||
if (missing.length > 0) {
|
||||
targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
|
||||
targets.push({
|
||||
id,
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total: chunks.length,
|
||||
start,
|
||||
end,
|
||||
capture: true,
|
||||
corpusOnly: true,
|
||||
})
|
||||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
return targets
|
||||
|
||||
+203
-73
@@ -6,19 +6,27 @@ import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { YencError } from '#lib/yenc/index'
|
||||
import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
|
||||
import { PendingNamer } from '#services/pending_namer'
|
||||
|
||||
interface Candidate {
|
||||
id: number
|
||||
releaseId: number
|
||||
kind: string
|
||||
messageId: string
|
||||
/** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
|
||||
pending?: PendingProbe
|
||||
}
|
||||
|
||||
type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
|
||||
|
||||
/**
|
||||
* The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
|
||||
* then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
|
||||
*/
|
||||
const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'lone', unreadable: false },
|
||||
@@ -26,8 +34,21 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'archive', unreadable: true },
|
||||
{ kind: 'par2', unreadable: false },
|
||||
{ kind: 'archive', unreadable: false },
|
||||
{ kind: '7z-tail', unreadable: true },
|
||||
{ kind: '7z-tail', unreadable: false },
|
||||
]
|
||||
const FLUSH_MS = 2000
|
||||
/** Between sweeps that discard the waiting releases of dropped signatures. */
|
||||
const SWEEP_MS = 5 * 60_000
|
||||
|
||||
/**
|
||||
* JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
|
||||
* and the whole flush failed with it (LOG.md 2026-10-18 (78)).
|
||||
*/
|
||||
function jsonb(value: unknown): string {
|
||||
return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
|
||||
}
|
||||
|
||||
/** Slices a claim takes from random points of the queue. */
|
||||
const SLICES = 20
|
||||
const REPORT_MS = 60_000
|
||||
@@ -46,18 +67,31 @@ export class Prober {
|
||||
private refilling: Promise<void> | null = null
|
||||
private stopped = false
|
||||
private stats = Prober.freshStats()
|
||||
/** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
|
||||
private lanes = new Set<Promise<void>>()
|
||||
private running = 0
|
||||
/** The last lane to look found nothing to claim: the prober wants its minimum only. */
|
||||
private dry = false
|
||||
/** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */
|
||||
private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb)
|
||||
private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections)
|
||||
|
||||
/** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
|
||||
private static freshStats() {
|
||||
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
|
||||
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
|
||||
}
|
||||
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
|
||||
private namer = new PendingNamer()
|
||||
private sweptAt = 0
|
||||
|
||||
async run() {
|
||||
// A previous run's claims: one prober at a time.
|
||||
const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
|
||||
if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
|
||||
const connections = coordinatorConfig.probe.connections
|
||||
logger.info(`Probing with ${connections} connections.`)
|
||||
const released = await PendingLadder.release()
|
||||
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
|
||||
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
|
||||
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
|
||||
const flusher = (async () => {
|
||||
while (!this.stopped || this.outcomes.length > 0) {
|
||||
await sleep(FLUSH_MS)
|
||||
@@ -67,7 +101,27 @@ export class Prober {
|
||||
this.report()
|
||||
}
|
||||
})()
|
||||
await Promise.all(Array.from({ length: connections }, () => this.lane()))
|
||||
// Lanes as the share allows: the coordinator's history comes first (app/services/nntp_shares.ts).
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
const was = this.share.granted
|
||||
const granted = await this.share.sync(this.dry ? 0 : this.share.max, this.running)
|
||||
if (granted !== was) logger.info(`Probing: ${granted} connections (was ${was}).`)
|
||||
while (this.running < granted) {
|
||||
this.running++
|
||||
const lane: Promise<void> = this.lane()
|
||||
.catch((error) => logger.error(`Probing: a lane failed: ${error.message}`))
|
||||
.finally(() => this.lanes.delete(lane))
|
||||
this.lanes.add(lane)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Probing: NNTP share failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
|
||||
}
|
||||
await Promise.all(this.lanes)
|
||||
await namer
|
||||
await this.share.leave().catch(() => {})
|
||||
// What was claimed and not probed goes back.
|
||||
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
this.stopped = true
|
||||
@@ -78,68 +132,90 @@ export class Prober {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
/** One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure. */
|
||||
/**
|
||||
* One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure; done when
|
||||
* there are more lanes than the share (its connection closed, the coordinator takes it on its next sync).
|
||||
*/
|
||||
private async lane() {
|
||||
let client: XoverClient | null = null
|
||||
while (!this.stopped) {
|
||||
const batch: Candidate[] = []
|
||||
const waited = Date.now()
|
||||
while (batch.length < coordinatorConfig.probe.pipeline) {
|
||||
const candidate = await this.next()
|
||||
if (!candidate) break
|
||||
batch.push(candidate)
|
||||
}
|
||||
this.stats.waitMs += Date.now() - waited
|
||||
if (batch.length === 0) {
|
||||
this.stats.idleMs += 30_000
|
||||
await sleep(30_000)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const fetched = Date.now()
|
||||
const bodies = await client.bodies(batch.map((c) => c.messageId))
|
||||
this.stats.fetchMs += Date.now() - fetched
|
||||
const inspected = Date.now()
|
||||
for (const [i, candidate] of batch.entries()) {
|
||||
const body = bodies[i]
|
||||
this.stats.articles++
|
||||
if (body === null) {
|
||||
// Taken down or expired: final.
|
||||
this.outcomes.push({ candidate, status: 'missing' })
|
||||
continue
|
||||
let usedAt = Date.now()
|
||||
try {
|
||||
while (!this.stopped) {
|
||||
if (this.running > this.share.granted) break
|
||||
const batch: Candidate[] = []
|
||||
const waited = Date.now()
|
||||
while (batch.length < coordinatorConfig.probe.pipeline) {
|
||||
const candidate = await this.next()
|
||||
if (!candidate) break
|
||||
batch.push(candidate)
|
||||
}
|
||||
this.stats.waitMs += Date.now() - waited
|
||||
this.dry = batch.length === 0
|
||||
if (batch.length === 0) {
|
||||
// Nothing to probe for a while: the connection closes (nntpIdleCloseMs), and opens again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (client && idleMs > 0 && Date.now() - usedAt >= idleMs) {
|
||||
client.close()
|
||||
client = null
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
this.outcomes.push({ candidate, status: 'error' })
|
||||
this.stats.idleMs += 30_000
|
||||
await sleep(30_000)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const fetched = Date.now()
|
||||
const bodies = await client.bodies(batch.map((c) => c.messageId))
|
||||
this.stats.fetchMs += Date.now() - fetched
|
||||
const inspected = Date.now()
|
||||
for (const [i, candidate] of batch.entries()) {
|
||||
const body = bodies[i]
|
||||
this.stats.articles++
|
||||
if (body === null) {
|
||||
// Taken down or expired: final.
|
||||
this.outcomes.push({ candidate, status: 'missing' })
|
||||
continue
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
this.outcomes.push({ candidate, status: 'error' })
|
||||
}
|
||||
}
|
||||
this.stats.inspectMs += Date.now() - inspected
|
||||
usedAt = Date.now()
|
||||
} catch (error: any) {
|
||||
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
|
||||
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
client?.close()
|
||||
client = null
|
||||
await sleep(5000)
|
||||
}
|
||||
this.stats.inspectMs += Date.now() - inspected
|
||||
} catch (error: any) {
|
||||
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
|
||||
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
client?.close()
|
||||
client = null
|
||||
await sleep(5000)
|
||||
}
|
||||
} finally {
|
||||
client?.close()
|
||||
this.running--
|
||||
}
|
||||
client?.close()
|
||||
}
|
||||
|
||||
/** Candidates claimed at a time: a few rounds of every lane's pipeline (a claim takes seconds; 400 lasted ~2 s at 37 lanes). */
|
||||
private get batch() {
|
||||
return Math.max(coordinatorConfig.probe.batch, this.running * coordinatorConfig.probe.pipeline * 4)
|
||||
}
|
||||
|
||||
private async next(): Promise<Candidate | undefined> {
|
||||
if (this.queue.length < coordinatorConfig.probe.batch / 2) {
|
||||
if (this.queue.length < this.batch / 2) {
|
||||
this.refilling ??= this.claim().finally(() => (this.refilling = null))
|
||||
if (this.queue.length === 0) await this.refilling
|
||||
}
|
||||
@@ -162,13 +238,55 @@ export class Prober {
|
||||
}
|
||||
|
||||
private async claimTiers() {
|
||||
// The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
|
||||
if (await this.space.short()) return
|
||||
await this.claimCatalog()
|
||||
await this.claimPending()
|
||||
}
|
||||
|
||||
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
|
||||
private async claimPending() {
|
||||
if (Date.now() - this.sweptAt > SWEEP_MS) {
|
||||
this.sweptAt = Date.now()
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
}
|
||||
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
|
||||
const probes = await PendingLadder.claim(want)
|
||||
if (probes.length === 0) return
|
||||
const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
|
||||
probes.map((p) => p.messageId),
|
||||
PROBE_VERSION,
|
||||
])
|
||||
const known = new Map(cached.map((c: any) => [c.message_id, c]))
|
||||
for (const probe of probes) {
|
||||
const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
|
||||
const hit: any = known.get(probe.messageId)
|
||||
if (hit) {
|
||||
this.stats.cached++
|
||||
this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
|
||||
} else {
|
||||
this.queue.push(candidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async claimCatalog() {
|
||||
for (const { kind, unreadable } of TIERS) {
|
||||
const want = coordinatorConfig.probe.batch - this.queue.length
|
||||
const want = this.batch - this.queue.length
|
||||
if (want <= 0) return
|
||||
// The starting points chosen here: in the query Postgres drew random() once for all slices.
|
||||
// Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
|
||||
// Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting
|
||||
// entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)).
|
||||
const values = unreadable ? [true] : [false, true]
|
||||
const ends = values
|
||||
.map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`)
|
||||
.join(' UNION ALL ')
|
||||
const [{ lo, hi }] = (
|
||||
await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
|
||||
await db.rawQuery(
|
||||
`SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`,
|
||||
values.flatMap((v) => [kind, v])
|
||||
)
|
||||
).rows
|
||||
if (lo === null) continue
|
||||
const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
|
||||
@@ -203,22 +321,31 @@ export class Prober {
|
||||
|
||||
/** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
|
||||
private async flush() {
|
||||
const outcomes = this.outcomes.splice(0)
|
||||
if (outcomes.length === 0) return
|
||||
for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
const taken = this.outcomes.splice(0)
|
||||
if (taken.length === 0) return
|
||||
for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
// The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
|
||||
const pending = taken.filter((o) => o.candidate.pending)
|
||||
const outcomes = taken.filter((o) => !o.candidate.pending)
|
||||
this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
|
||||
[JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
|
||||
)
|
||||
if (outcomes.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
|
||||
[jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
|
||||
)
|
||||
}
|
||||
await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
|
||||
// Final answers read from the news server, kept by message-id for the next catalog (a reset).
|
||||
const fresh = outcomes.filter((o) => !o.cached && o.status !== 'waiting')
|
||||
// One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
|
||||
// row twice in one statement.
|
||||
const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
|
||||
if (fresh.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO probe_cache (message_id, status, result, version)
|
||||
SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
|
||||
ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
|
||||
[PROBE_VERSION, JSON.stringify(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
|
||||
[PROBE_VERSION, jsonb(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
|
||||
)
|
||||
}
|
||||
const done = outcomes.filter((o) => o.inspection)
|
||||
@@ -228,9 +355,11 @@ export class Prober {
|
||||
SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
|
||||
ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
|
||||
// A par2 set's file list goes to par2_files, not into every inspection.
|
||||
[JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
[jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
)
|
||||
await this.storeForMerge(trx, done)
|
||||
// A par2 index read: the release's completeness by its set, when it has no counter.
|
||||
await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
|
||||
// Each touched release from all its inspections (earlier rounds may have stored others).
|
||||
const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
|
||||
const { rows } = await trx.rawQuery(
|
||||
@@ -252,7 +381,7 @@ export class Prober {
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
|
||||
[JSON.stringify(updates)]
|
||||
[jsonb(updates)]
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -265,7 +394,7 @@ export class Prober {
|
||||
`INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
|
||||
SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
|
||||
ON CONFLICT (candidate_id) DO NOTHING`,
|
||||
[JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
|
||||
[jsonb(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
|
||||
)
|
||||
}
|
||||
for (const o of done) {
|
||||
@@ -275,7 +404,7 @@ export class Prober {
|
||||
`INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
|
||||
SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
|
||||
[set.setId, jsonb(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
|
||||
)
|
||||
await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
|
||||
}
|
||||
@@ -287,7 +416,8 @@ export class Prober {
|
||||
const { articles, bytes, done, missing, error } = this.stats
|
||||
logger.info(
|
||||
`Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
|
||||
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
|
||||
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
|
||||
`the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
|
||||
`Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
|
||||
`idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
|
||||
`claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
|
||||
* release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
|
||||
* range that isn't done or doesn't start where the one before ended.
|
||||
*/
|
||||
class RangeBlocks {
|
||||
async rebuild(): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`DELETE FROM range_blocks`)
|
||||
const result = await trx.rawQuery(
|
||||
`WITH r AS (
|
||||
SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
|
||||
min(start_id) OVER (PARTITION BY group_id) AS group_first,
|
||||
lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
|
||||
FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
|
||||
runs AS (
|
||||
SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
|
||||
OVER (PARTITION BY group_id ORDER BY start_id) AS run
|
||||
FROM r)
|
||||
INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
|
||||
SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
|
||||
FROM runs WHERE done GROUP BY group_id, run`
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new RangeBlocks()
|
||||
+184
-53
@@ -1,10 +1,19 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again. */
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
|
||||
export const JUDGE_VERSION = 1
|
||||
/** Between rounds: each rule is a scan of its table. */
|
||||
const ROUND_MS = 60 * 60 * 1000
|
||||
/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
|
||||
const ROUND_MS = 2 * 60 * 1000
|
||||
/**
|
||||
* A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
|
||||
* (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
|
||||
* after a merge (repeated).
|
||||
*/
|
||||
const FULL_MS = 24 * 60 * 60 * 1000
|
||||
/** Changed releases judged at a time. */
|
||||
const CHUNK = 50_000
|
||||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||||
const REPEATED_RELEASES = 20
|
||||
|
||||
@@ -24,10 +33,15 @@ export interface Rule {
|
||||
name: string
|
||||
/** Why, in a few words (the log, the web interface). */
|
||||
says: string
|
||||
/** SELECT of the release ids the rule judges. */
|
||||
sql: string
|
||||
/** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
|
||||
sql: (scoped: boolean) => string
|
||||
}
|
||||
|
||||
/** A join to the releases in scope (an incremental round), on the given release id column. */
|
||||
const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||||
* the header pass, the prober, the merges and the spots know by now.
|
||||
@@ -36,26 +50,33 @@ export const RULES: Rule[] = [
|
||||
{
|
||||
name: 'dead',
|
||||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||||
sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
|
||||
sql: (scoped) =>
|
||||
`SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
|
||||
WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
|
||||
},
|
||||
{
|
||||
name: 'fake',
|
||||
says: 'named as a film, episode or music, but the content is a program',
|
||||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||||
// an executable by name or by its first bytes, under a media name.
|
||||
sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
|
||||
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||||
UNION
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||||
},
|
||||
{
|
||||
name: 'repeated',
|
||||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||||
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
// Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
|
||||
// (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
|
||||
: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||||
},
|
||||
{
|
||||
@@ -64,21 +85,28 @@ export const RULES: Rule[] = [
|
||||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||||
sql: `SELECT r.id FROM releases r
|
||||
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||||
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
|
||||
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
|
||||
: `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
|
||||
},
|
||||
{
|
||||
name: 'locked',
|
||||
says: 'passworded, and no spot, NZB or title gives the password',
|
||||
sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||
// Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
|
||||
sql: `SELECT r.id FROM releases r
|
||||
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||
AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
|
||||
AND r.settled_at IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||
@@ -86,15 +114,18 @@ export const RULES: Rule[] = [
|
||||
]
|
||||
|
||||
export interface JudgeRound {
|
||||
/** Per rule: releases it judges now, verdicts added, verdicts taken back. */
|
||||
/** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
|
||||
releases?: number
|
||||
/** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
|
||||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||||
}
|
||||
|
||||
/**
|
||||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
|
||||
* round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
|
||||
* matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
|
||||
* minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
|
||||
* daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
|
||||
* verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
*/
|
||||
export class ReleaseJudge {
|
||||
private stopped = false
|
||||
@@ -107,9 +138,13 @@ export class ReleaseJudge {
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
const round = await this.round()
|
||||
const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
|
||||
logger.info(`Judged: ${parts.join(', ')}.`)
|
||||
if (await this.fullDue()) {
|
||||
const round = await this.full()
|
||||
if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
|
||||
} else {
|
||||
const round = await this.changed()
|
||||
if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Judge round failed: ${error.message}`)
|
||||
}
|
||||
@@ -117,43 +152,139 @@ export class ReleaseJudge {
|
||||
}
|
||||
}
|
||||
|
||||
/** One round over every rule; with dry, only counts. */
|
||||
async round(dry = false): Promise<JudgeRound> {
|
||||
/** A full round is due: a day after the last, or the last was by another rule set. */
|
||||
async fullDue(): Promise<boolean> {
|
||||
const state = await db.from('judge_state').first()
|
||||
return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
|
||||
}
|
||||
|
||||
/**
|
||||
* Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
|
||||
* half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
|
||||
*/
|
||||
async full(dry = false): Promise<JudgeRound> {
|
||||
const started = new Date()
|
||||
const round: JudgeRound = { rules: {} }
|
||||
// Verdicts of an older rule set are judged again from scratch.
|
||||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||||
const judged = new Set<string>()
|
||||
await db.rawQuery(`TRUNCATE judge_full`)
|
||||
for (const rule of RULES) {
|
||||
if (this.stopped) break
|
||||
const { rows } = await db.rawQuery(rule.sql)
|
||||
const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
|
||||
for (const id of ids) judged.add(id)
|
||||
const stats = { matched: ids.length, added: 0, removed: 0 }
|
||||
if (!dry) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
for (let i = 0; i < ids.length; i += 50_000) {
|
||||
await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
|
||||
}
|
||||
const removed = await trx.rawQuery(
|
||||
`DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
|
||||
if (this.stopped) return round
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
|
||||
}
|
||||
if (dry || this.stopped) return round
|
||||
await db.rawQuery(`ANALYZE judge_full`)
|
||||
await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
|
||||
await db.rawQuery(
|
||||
`INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
|
||||
[started, JUDGE_VERSION]
|
||||
)
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
|
||||
* A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
|
||||
* already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
|
||||
* file in that many without them had its releases judged before.
|
||||
*/
|
||||
async changed(): Promise<JudgeRound> {
|
||||
const round: JudgeRound = { releases: 0, rules: {} }
|
||||
for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
|
||||
let more = true
|
||||
while (more && !this.stopped) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
|
||||
scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
|
||||
SELECT (SELECT count(*) FROM taken) AS taken`
|
||||
)
|
||||
const taken = Number(rows[0].taken)
|
||||
more = taken === CHUNK
|
||||
if (taken === 0) return
|
||||
// The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
|
||||
await trx.rawQuery(
|
||||
`CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_tipped
|
||||
SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
|
||||
FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT count(DISTINCT f.release_id) AS total,
|
||||
count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
|
||||
FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
|
||||
WHERE c.total >= ${REPEATED_RELEASES}`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
|
||||
WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
|
||||
ON CONFLICT DO NOTHING`
|
||||
)
|
||||
await trx.rawQuery(`ANALYZE judge_scope`)
|
||||
const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
|
||||
round.releases! += Number(scope[0].n)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
|
||||
for (const rule of RULES) {
|
||||
const { rowCount } = await trx.rawQuery(
|
||||
`INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
const added = await trx.rawQuery(
|
||||
`INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule`,
|
||||
[rule.name, JUDGE_VERSION]
|
||||
)
|
||||
stats.removed = removed.rowCount ?? 0
|
||||
stats.added = added.rowCount ?? 0
|
||||
})
|
||||
}
|
||||
round.rules[rule.name] = stats
|
||||
round.rules[rule.name].matched += rowCount ?? 0
|
||||
}
|
||||
await trx.rawQuery(`ANALYZE judged`)
|
||||
await this.apply(trx, 'judged', true, round)
|
||||
})
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
|
||||
* more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
|
||||
* insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
|
||||
*/
|
||||
private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
|
||||
const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
|
||||
for (const { rule, n } of rows) {
|
||||
round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
|
||||
round.rules[rule][field] += Number(n)
|
||||
}
|
||||
}
|
||||
const { rows: removed } = await trx.rawQuery(
|
||||
`WITH gone AS (DELETE FROM release_verdicts v
|
||||
WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
|
||||
NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
|
||||
RETURNING v.rule)
|
||||
SELECT rule, count(*) AS n FROM gone GROUP BY 1`
|
||||
)
|
||||
tally(removed, 'removed')
|
||||
const { rows: added } = await trx.rawQuery(
|
||||
`WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
|
||||
WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule
|
||||
RETURNING rule)
|
||||
SELECT rule, count(*) AS n FROM written GROUP BY 1`,
|
||||
[JUDGE_VERSION]
|
||||
)
|
||||
tally(added, 'added')
|
||||
}
|
||||
}
|
||||
|
||||
/** "dead 1,157,018 (+1,136 −712), …" for the log. */
|
||||
function describe(round: JudgeRound): string {
|
||||
return Object.entries(round.rules)
|
||||
.map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
|
||||
.join(', ')
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { SOURCES, sourceName } from '#lib/release/source'
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
@@ -51,12 +52,14 @@ export interface ReleaseSummary {
|
||||
size: number
|
||||
readable: boolean
|
||||
complete: boolean | null
|
||||
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
|
||||
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
|
||||
final: boolean
|
||||
category: number
|
||||
settled: boolean
|
||||
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
||||
passworded: boolean | null
|
||||
/** A password is known (release_passwords): it's in the NZB. */
|
||||
passwordKnown: boolean
|
||||
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
||||
innerName: string | null
|
||||
}
|
||||
@@ -89,27 +92,32 @@ class SearchService {
|
||||
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
||||
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
||||
const bindings: unknown[] = []
|
||||
// Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
|
||||
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
|
||||
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike.
|
||||
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
||||
where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
|
||||
bindings.push(contains(word), contains(word))
|
||||
where.push('r.id IN (SELECT t.release_id FROM release_texts t WHERE t.text ILIKE ?)')
|
||||
bindings.push(contains(word))
|
||||
}
|
||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||
if (params.complete) where.push('r.complete')
|
||||
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
|
||||
// Incomplete only when final: a provisional tail release may yet get its missing files.
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
|
||||
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(params.source)
|
||||
const code = (SOURCES as Record<string, number>)[params.source]
|
||||
if (code === undefined) where.push('false')
|
||||
else {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(code)
|
||||
}
|
||||
}
|
||||
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
||||
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
||||
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
||||
let posters: string[] | null = null
|
||||
let posters: number[] | null = null
|
||||
if (params.poster) {
|
||||
const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||
posters = matched.map((m: { poster: string }) => m.poster)
|
||||
const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||
posters = matched.map((m: { id: string }) => Number(m.id))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
@@ -141,14 +149,14 @@ class SearchService {
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
||||
// Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
|
||||
// bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
|
||||
// Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
|
||||
// in the order asked; then merged.
|
||||
const source =
|
||||
posters === null
|
||||
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
||||
: {
|
||||
from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||
SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||
from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||
where: 'true',
|
||||
bindings: [posters] as unknown[],
|
||||
}
|
||||
@@ -156,15 +164,17 @@ class SearchService {
|
||||
posters === null
|
||||
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
||||
: db.rawQuery(
|
||||
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||
SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||
[posters, ...bindings]
|
||||
)
|
||||
const [{ rows }, { rows: counted }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
||||
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
`SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
||||
r.settled_at IS NOT NULL AS final
|
||||
FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
|
||||
WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...source.bindings, ...bindings, perPage, offset]
|
||||
),
|
||||
counting,
|
||||
@@ -179,9 +189,9 @@ class SearchService {
|
||||
releases: (rows as any[]).map((r): ReleaseSummary => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
@@ -192,6 +202,7 @@ class SearchService {
|
||||
category: r.category,
|
||||
settled: r.settled_at !== null,
|
||||
passworded: r.passworded,
|
||||
passwordKnown: r.password_known,
|
||||
innerName: r.inner_name,
|
||||
})),
|
||||
}
|
||||
@@ -199,19 +210,24 @@ class SearchService {
|
||||
|
||||
/** A release with its files, read from its chunks. */
|
||||
async release(id: number) {
|
||||
const r = await db.from('releases').where('id', id).first()
|
||||
const {
|
||||
rows: [r],
|
||||
} = await db.rawQuery(
|
||||
`SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (!r) return null
|
||||
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
|
||||
const [files, { rows: inspections }, passwords] = await Promise.all([
|
||||
NzbService.files(id),
|
||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
||||
NzbService.passwords(id),
|
||||
])
|
||||
return {
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
@@ -219,13 +235,14 @@ class SearchService {
|
||||
segments: Number(r.segments),
|
||||
readable: r.readable,
|
||||
complete: r.complete,
|
||||
final: finality[0].final,
|
||||
final: r.settled_at !== null,
|
||||
hasPar2: r.has_par2,
|
||||
category: r.category,
|
||||
firstSeenAt: r.first_seen_at,
|
||||
lastSeenAt: r.last_seen_at,
|
||||
settledAt: r.settled_at,
|
||||
passworded: r.passworded,
|
||||
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
|
||||
passwords,
|
||||
innerName: r.inner_name,
|
||||
inspectedAt: r.inspected_at,
|
||||
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
|
||||
const QUIET_HOURS = 6
|
||||
/**
|
||||
* History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
|
||||
* neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
|
||||
* Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
|
||||
* each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
|
||||
*/
|
||||
const QUIET_HISTORY_MINUTES = 30
|
||||
const BATCH = 50_000
|
||||
|
||||
/**
|
||||
* Settles releases (REBUILD.md, part 8): once nothing has been added for
|
||||
* QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
|
||||
* [i/N] counter announced was found, unknown without a counter (phase 2
|
||||
* reads par2 sets for those). Phase 2 picks its work from settled releases.
|
||||
*/
|
||||
class Settler {
|
||||
/** Up to `batches` batches of BATCH (all there are without). */
|
||||
async run(batches = Infinity): Promise<number> {
|
||||
let settled = 0
|
||||
for (let batch = 0; batch < batches; batch++) {
|
||||
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH settled AS (
|
||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
|
||||
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
||||
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
||||
RETURNING id, readable, files, bytes, poster),
|
||||
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
|
||||
-- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
|
||||
queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
|
||||
ON CONFLICT DO NOTHING)
|
||||
SELECT count(*)::int AS n FROM settled`,
|
||||
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
|
||||
)
|
||||
const n = rows[0].n
|
||||
settled += n
|
||||
if (n < BATCH) return settled
|
||||
}
|
||||
return settled
|
||||
}
|
||||
}
|
||||
|
||||
export default new Settler()
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
|
||||
* segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
|
||||
* its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
|
||||
* it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
|
||||
* .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
|
||||
* waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
|
||||
* a piece that saw only part of it queued a segment that isn't its last.
|
||||
*/
|
||||
export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const result = await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates pc USING (
|
||||
SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
|
||||
FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
|
||||
FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
|
||||
) s
|
||||
WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
}
|
||||
@@ -6,7 +6,7 @@ const BATCH = 2000
|
||||
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
|
||||
const WINDOW_MINUTES = 10
|
||||
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
|
||||
const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||
export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||
|
||||
/** The words of a name worth comparing: 4+ letters or digits. */
|
||||
function words(text: string): Set<string> {
|
||||
@@ -53,29 +53,27 @@ export class SidecarFolder {
|
||||
|
||||
async round(): Promise<FoldRound> {
|
||||
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
|
||||
// A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
|
||||
// queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
|
||||
const { rows: queued } = await db.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
|
||||
FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||
`SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
|
||||
FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||
[BATCH]
|
||||
)
|
||||
const { rows: stale } = await db.rawQuery(
|
||||
`SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
|
||||
[BATCH]
|
||||
)
|
||||
const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
|
||||
const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
|
||||
for (const c of queued) {
|
||||
if (c.id === null) continue
|
||||
round.looked++
|
||||
done.push(Number(c.id))
|
||||
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
|
||||
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
|
||||
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
|
||||
round.sidecars++
|
||||
if (!c.posted_at) continue
|
||||
const { rows: siblings } = await db.rawQuery(
|
||||
`SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||
`SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
|
||||
AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
|
||||
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
|
||||
AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
|
||||
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
|
||||
)
|
||||
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
|
||||
if (!sibling) continue
|
||||
|
||||
@@ -9,6 +9,9 @@ import { decodeSpotBinary } from '#lib/spotnet/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/headers/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
|
||||
/** Spots claimed per round, newest first. */
|
||||
const BATCH = 20
|
||||
@@ -102,6 +105,12 @@ export class SpotInspector {
|
||||
this.client = undefined
|
||||
}
|
||||
if (!round || round.inspected === 0) {
|
||||
// Nothing to inspect for a while: closed (nntpIdleCloseMs), opened again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && IDLE_MS >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
@@ -245,8 +254,8 @@ export class SpotInspector {
|
||||
}
|
||||
if (wanted.length === 0) return 0
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
|
||||
for (const r of rows) {
|
||||
@@ -269,7 +278,7 @@ export class SpotInspector {
|
||||
await db.transaction(async (trx) => {
|
||||
// A release of the spot's own goes now that the header pass built its files' releases.
|
||||
if (spot.release_id) {
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
|
||||
if (own) {
|
||||
await trx.from('release_spots').where('release_id', own.id).delete()
|
||||
await trx.from('release_passwords').where('release_id', own.id).delete()
|
||||
@@ -288,6 +297,12 @@ export class SpotInspector {
|
||||
.ignore()
|
||||
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
|
||||
}
|
||||
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
|
||||
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
|
||||
spot.category,
|
||||
JSON.stringify(spot.subcats ?? {}),
|
||||
[...perRelease.keys()],
|
||||
])
|
||||
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
|
||||
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
|
||||
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
|
||||
@@ -323,16 +338,19 @@ export class SpotInspector {
|
||||
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
|
||||
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
|
||||
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
|
||||
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
|
||||
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
|
||||
const groupSetId = await CatalogRefs.groupSetId(db, groups)
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
release_key: `spot:${key}`,
|
||||
identity_key: key,
|
||||
release_key: releaseKey(key, groupId, null),
|
||||
identity_key: identityKey(key),
|
||||
group_id: groupId,
|
||||
group_set_id: groupSetId,
|
||||
poster_id: posterId,
|
||||
name: spot.title.slice(0, 1000),
|
||||
source: 'spot',
|
||||
poster: spot.poster,
|
||||
groups: JSON.stringify(groups),
|
||||
source: SOURCES.spot,
|
||||
posted_at: spot.posted_at,
|
||||
last_posted_at: spot.posted_at,
|
||||
files: files.length,
|
||||
@@ -341,6 +359,8 @@ export class SpotInspector {
|
||||
readable: true,
|
||||
passworded: spot.password ? true : null,
|
||||
settled_at: new Date(),
|
||||
// Kept when its title says nothing (the category trigger, migration 5200).
|
||||
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
|
||||
})
|
||||
.onConflict('release_key')
|
||||
.ignore()
|
||||
@@ -349,6 +369,8 @@ export class SpotInspector {
|
||||
const releaseId = Number(row.id)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
|
||||
// Its title, searchable (release_texts).
|
||||
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
|
||||
if (spot.password) {
|
||||
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { MissingArticle, XoverClient } from '#lib/nntp/index'
|
||||
@@ -60,6 +61,12 @@ export class Spotter {
|
||||
this.client = undefined
|
||||
}
|
||||
const wait = coordinatorConfig.spots.everySeconds * 1000
|
||||
// Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
@@ -177,7 +184,7 @@ export class Spotter {
|
||||
await trx.from('release_spots').where('spot_id', targetId).delete()
|
||||
await trx.from('release_passwords').where('spot_id', targetId).delete()
|
||||
// A release the spot made of its own goes with it.
|
||||
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete()
|
||||
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
|
||||
})
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
}
|
||||
|
||||
+60
-56
@@ -3,45 +3,45 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { Prefetcher } from '#services/prefetcher'
|
||||
import Planner from '#services/planner'
|
||||
import FragmentMerger from '#services/fragment_merger'
|
||||
import Settler from '#services/settler'
|
||||
import RangeBlocks from '#services/range_blocks'
|
||||
import Leases from '#services/leases'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
const TICK_MS = 2000
|
||||
const PLAN_EVERY_MS = 60 * 60 * 1000
|
||||
const MERGE_EVERY_MS = 60 * 1000
|
||||
const MERGE_BUDGET_MS = 30 * 1000
|
||||
const STOP_WAIT_MS = 15 * 1000
|
||||
/**
|
||||
* Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
|
||||
* rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
|
||||
* (LOG.md 2026-10-16 (67)).
|
||||
*/
|
||||
const SETTLE_EVERY_MS = 5 * 60 * 1000
|
||||
const SETTLE_BATCHES = 4
|
||||
/** range_blocks (release_final) after ranges finish. */
|
||||
const BLOCKS_EVERY_MS = 5 * 60 * 1000
|
||||
/** Leases whose worker went quiet after reading everything are finished every few minutes. */
|
||||
const ABANDONED_EVERY_MS = 5 * 60 * 1000
|
||||
|
||||
/**
|
||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
|
||||
* merging fragments of finished ranges every minute, and every hour
|
||||
* planning ranges for new articles and settling quiet releases. The API (leases, stream,
|
||||
* batches) runs in the web server.
|
||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher, the tail's ranges every few minutes,
|
||||
* new groups' history every hour, and abandoned leases. The API (leases, stream, batches) runs in the web server.
|
||||
* Voyager 3: the settler and the fragment merger are gone (the window finishes a release once; the block merger
|
||||
* joins edge releases).
|
||||
*/
|
||||
export default class CoordinatorRun extends BaseCommand {
|
||||
static commandName = 'coordinator:run'
|
||||
static description = 'Prefetches ranges into the spool and plans new ones'
|
||||
static options: CommandOptions = { startApp: true, staysAlive: true }
|
||||
|
||||
@flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
|
||||
@flags.string({
|
||||
description:
|
||||
'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles',
|
||||
})
|
||||
declare shard?: string
|
||||
|
||||
async run() {
|
||||
const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
|
||||
if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
|
||||
if (!(count >= 1 && index >= 1 && index <= count))
|
||||
throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
|
||||
const prefetcher = new Prefetcher({ index: index - 1, count })
|
||||
const share = new NntpShare(
|
||||
count > 1 ? `coordinator ${index}/${count}` : 'coordinator',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority,
|
||||
coordinatorConfig.nntpMinConnections,
|
||||
coordinatorConfig.nntpConnections
|
||||
)
|
||||
const main = index === 1
|
||||
let stopping = false
|
||||
let stopped: () => void
|
||||
@@ -53,9 +53,33 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
prefetcher.stop()
|
||||
await done
|
||||
})
|
||||
this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
|
||||
// Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
|
||||
// of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
|
||||
this.logger.info(
|
||||
count > 1
|
||||
? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).`
|
||||
: 'Coordinator started.'
|
||||
)
|
||||
// The pool's size: its share of the account's connections, the prober the rest (app/services/nntp_shares.ts).
|
||||
// Before the first connection: until then it's NNTP_CONNECTIONS.
|
||||
const first = await NntpPool.share(share)
|
||||
this.logger.info(`NNTP share: ${first.granted} connections.`)
|
||||
const sharing = (async () => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
const { was, granted } = await NntpPool.share(share)
|
||||
if (granted !== was) this.logger.info(`NNTP share: ${granted} connections (was ${was}).`)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`NNTP share failed: ${error.message}`)
|
||||
}
|
||||
for (
|
||||
let waited = 0;
|
||||
waited < coordinatorConfig.nntpShares.everyMs && !stopping;
|
||||
waited += TICK_MS
|
||||
)
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// The prefetcher's ticks in a loop of their own: planning doesn't hold them (a merge round of up to 30 s once
|
||||
// left the connections idle once their targets were fetched).
|
||||
const prefetching = (async () => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
@@ -66,22 +90,24 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
|
||||
const blocks = (async () => {
|
||||
const abandoned = (async () => {
|
||||
while (main && !stopping) {
|
||||
try {
|
||||
await RangeBlocks.rebuild()
|
||||
const finished = await Leases.finishAbandoned()
|
||||
if (finished > 0)
|
||||
this.logger.info(
|
||||
`Finished ${finished} ranges read to their end whose worker went quiet.`
|
||||
)
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Range blocks failed: ${error.message}`)
|
||||
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
|
||||
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
})()
|
||||
// History for new groups hourly; the tail every few minutes.
|
||||
let plannedAt = 0
|
||||
let tailedAt = 0
|
||||
let mergedAt = 0
|
||||
let settledAt = 0
|
||||
while (!stopping) {
|
||||
try {
|
||||
if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
|
||||
@@ -89,46 +115,24 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
const { groups, ranges } = await Planner.planAll()
|
||||
if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
|
||||
}
|
||||
if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
|
||||
settledAt = Date.now()
|
||||
const settled = await Settler.run(SETTLE_BATCHES)
|
||||
if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
|
||||
}
|
||||
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
|
||||
tailedAt = Date.now()
|
||||
const { groups, ranges } = await Planner.planTail()
|
||||
if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
|
||||
}
|
||||
if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
|
||||
mergedAt = Date.now()
|
||||
// Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
|
||||
// (Going on while only waiting files were left kept Postgres busy all the time.)
|
||||
const totals = { completed: 0, incomplete: 0, waiting: 0 }
|
||||
let decidedThisPass = 0
|
||||
while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
|
||||
const merged = await FragmentMerger.run()
|
||||
totals.completed += merged.completed
|
||||
totals.incomplete += merged.incomplete
|
||||
totals.waiting = merged.waiting
|
||||
decidedThisPass += merged.completed + merged.incomplete
|
||||
if (merged.wrapped) {
|
||||
if (decidedThisPass === 0) break
|
||||
decidedThisPass = 0
|
||||
}
|
||||
}
|
||||
if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Coordinator round failed: ${error.message}`)
|
||||
}
|
||||
await sleep(TICK_MS)
|
||||
}
|
||||
await Promise.all([prefetching, blocks])
|
||||
await Promise.all([prefetching, abandoned, sharing])
|
||||
// The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
|
||||
const waitUntil = Date.now() + STOP_WAIT_MS
|
||||
while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
|
||||
if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
|
||||
if (prefetcher.active > 0)
|
||||
this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
|
||||
await NntpPool.close()
|
||||
await share.leave().catch(() => {})
|
||||
this.logger.info('Coordinator stopped.')
|
||||
stopped!()
|
||||
await this.terminate()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { access, readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import Corpus, { FILTERED } from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/**
|
||||
* Filters corpus chunks stored as the news server sent them (before CORPUS_FILTER, 2026-10-23): each chunk without a
|
||||
* `.drops` of the planned capture ranges (--all: of every range in the corpus) written again as Corpus.write would.
|
||||
*/
|
||||
export default class CorpusFilterCommand extends BaseCommand {
|
||||
static commandName = 'corpus:filter'
|
||||
static description = 'Filters stored corpus chunks (planned capture ranges; --all: every range) as CORPUS_FILTER does'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.boolean({ description: 'Every range in the corpus, also done ones (the grouping sweeps read them raw)' })
|
||||
declare all: boolean
|
||||
|
||||
async run() {
|
||||
const dir = coordinatorConfig.corpus.dir
|
||||
if (!dir) return this.logger.error('CORPUS_DIR is not set')
|
||||
const rules = await RuleSets.current()
|
||||
const filter = { ruleVersion: rules.version, priorities: rules.priorities }
|
||||
const query = db.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').where('r.capture', true).select('g.name as group', 'r.start_id', 'r.end_id')
|
||||
const ranges = await (this.all ? query : query.where('r.status', 'planned'))
|
||||
let chunks = 0
|
||||
let before = 0
|
||||
let after = 0
|
||||
for (const r of ranges) {
|
||||
const range = { group: r.group, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
const rangeDir = join(dir, range.group, `${range.start}-${range.end}`)
|
||||
const files = await readdir(rangeDir).catch(() => [] as string[])
|
||||
const gz = files.filter((f) => f.endsWith('.gz'))
|
||||
for (const name of gz) {
|
||||
if (files.includes(`${name}.drops`)) continue
|
||||
const m = /^\d{4}-(\d+)-(\d+)\.gz$/.exec(name)
|
||||
if (!m) continue
|
||||
const data = await readFile(join(rangeDir, name))
|
||||
await Corpus.write(range, { from: Number(m[1]), to: Number(m[2]) }, data, Number.POSITIVE_INFINITY, filter)
|
||||
before += data.length
|
||||
after += (await readFile(join(rangeDir, name))).length
|
||||
chunks++
|
||||
}
|
||||
// A complete range's range.json says it's filtered now (Corpus.write only writes it at the last chunk).
|
||||
const json = join(rangeDir, 'range.json')
|
||||
if (gz.length > 0 && (await access(json).then(() => true, () => false))) await writeFile(json, JSON.stringify({ ...range, filtered: FILTERED }))
|
||||
if (chunks > 0 && chunks % 1000 < gz.length) this.logger.info(`${chunks} chunks filtered: ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
|
||||
}
|
||||
this.logger.info(`Done: ${chunks} chunks of ${ranges.length} ranges filtered, ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
|
||||
}
|
||||
}
|
||||
@@ -108,7 +108,7 @@ export default class EdgeBench extends BaseCommand {
|
||||
let text = 0
|
||||
let peakRss = 0
|
||||
let peakDetail = ''
|
||||
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
|
||||
const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, samples: 0 }
|
||||
const counts = new Map<string, number>()
|
||||
const keys = new Set<string>()
|
||||
let batches = 0
|
||||
@@ -164,7 +164,6 @@ export default class EdgeBench extends BaseCommand {
|
||||
out.payload += r.payload.length
|
||||
}
|
||||
out.candidates += batch.candidates.length
|
||||
out.fragments += batch.fragments.length
|
||||
out.samples += batch.samples.length
|
||||
for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
|
||||
const m = process.memoryUsage()
|
||||
@@ -182,7 +181,7 @@ export default class EdgeBench extends BaseCommand {
|
||||
this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
|
||||
this.logger.info(` inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
|
||||
this.logger.info(` ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
|
||||
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
|
||||
this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.samples} samples`)
|
||||
for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(` dropped ${k}: ${v.toLocaleString('en')} articles`)
|
||||
if (this.store) this.logger.info(` stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import env from '#start/env'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/**
|
||||
* What the news server does with XOVER spans around an article that made a request hang (ranges:dates: one point
|
||||
* per group timed out after 2 minutes, again on a fresh connection; LOG.md 2026-10-25 (105)). Each probe on a
|
||||
* connection of its own with a short timeout: the span asked for, the reply, how many lines, how long. First the
|
||||
* span itself and wider ones around it, then halves of it down to single articles, then a span past `high`.
|
||||
*/
|
||||
export default class NntpProbe extends BaseCommand {
|
||||
static commandName = 'nntp:probe'
|
||||
static description =
|
||||
'Probes XOVER spans around an article that hangs, bisecting to the articles at fault'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@args.string({ description: 'Group' })
|
||||
declare group: string
|
||||
|
||||
@args.string({ description: 'First article of the span that hung' })
|
||||
declare article: string
|
||||
|
||||
@flags.number({ description: 'Articles in the span (default 32)' })
|
||||
declare size?: number
|
||||
|
||||
@flags.number({ description: 'Seconds without data before a probe is given up (default 20)' })
|
||||
declare timeout?: number
|
||||
|
||||
@flags.boolean({ description: 'Only the span itself and the wider ones, no bisection' })
|
||||
declare noBisect: boolean
|
||||
|
||||
@flags.boolean({ description: 'Without XFEATURE COMPRESS GZIP' })
|
||||
declare plain: boolean
|
||||
|
||||
@flags.string({
|
||||
description: 'Only spans from the article of these sizes, e.g. "8,16,24,32,48,64"',
|
||||
})
|
||||
declare sizes?: string
|
||||
|
||||
private timeoutMs = 20_000
|
||||
private info = { count: 0, low: 0, high: 0 }
|
||||
|
||||
async run() {
|
||||
this.timeoutMs = (this.timeout ?? 20) * 1000
|
||||
const a = Number(this.article)
|
||||
const size = this.size ?? 32
|
||||
const client = await this.connect()
|
||||
this.info = await client.group(this.group)
|
||||
client.close()
|
||||
this.logger.info(
|
||||
`${this.group}: ${this.info.count} articles, ${this.info.low}–${this.info.high}${this.plain ? '; plain' : '; compressed'}.`
|
||||
)
|
||||
const span = (from: number, to: number) =>
|
||||
[Math.max(this.info.low, from), Math.min(this.info.high, to)] as const
|
||||
if (this.sizes) {
|
||||
for (const n of this.sizes.split(',').map(Number)) await this.probe(...span(a, a + n - 1))
|
||||
return
|
||||
}
|
||||
this.logger.info('The span and wider ones:')
|
||||
await this.probe(...span(a, a + size - 1))
|
||||
await this.probe(...span(a, a + size * 32 - 1))
|
||||
await this.probe(...span(a - size * 32, a + size - 1))
|
||||
await this.probe(...span(a - 50_000, a + 49_999))
|
||||
this.logger.info('Past the end, and an empty span before it:')
|
||||
await this.probe(this.info.high + 100, this.info.high + 131)
|
||||
await this.probe(this.info.high + 1, this.info.high + 1)
|
||||
if (this.noBisect) return
|
||||
this.logger.info('Halves of the span:')
|
||||
await this.bisect(a, Math.min(this.info.high, a + size - 1))
|
||||
}
|
||||
|
||||
/** Halves of a span that hangs, down to single articles; a half that answers isn't split further. */
|
||||
private async bisect(from: number, to: number) {
|
||||
const hung = await this.probe(from, to)
|
||||
if (!hung || from === to) return
|
||||
const mid = Math.floor((from + to) / 2)
|
||||
await this.bisect(from, mid)
|
||||
await this.bisect(mid + 1, to)
|
||||
}
|
||||
|
||||
/** One XOVER on a fresh connection; true when it timed out. */
|
||||
private async probe(from: number, to: number): Promise<boolean> {
|
||||
const startedAt = Date.now()
|
||||
let client: XoverClient | null = null
|
||||
try {
|
||||
client = await this.connect()
|
||||
await client.group(this.group)
|
||||
const rows = await client.xover(from, to)
|
||||
const t = client.lastTiming
|
||||
this.logger.info(
|
||||
` XOVER ${from}-${to}: ${rows.length} lines in ${Date.now() - startedAt} ms (reply ${t.replyMs.toFixed(0)}, body ${t.bodyMs.toFixed(0)})${rows.length > 0 ? `; first ${rows[0][0]}, last ${rows[rows.length - 1][0]}` : ''}`
|
||||
)
|
||||
return false
|
||||
} catch (error: any) {
|
||||
const timedOut = /timed out/i.test(error.message)
|
||||
this.logger.warning(
|
||||
` XOVER ${from}-${to}: ${error.message} after ${Date.now() - startedAt} ms${client ? `, ${client.buffered} bytes buffered: ${JSON.stringify(client.bufferedHead().toString('latin1'))} (${client.bufferedHead().toString('hex')})` : ''}`
|
||||
)
|
||||
return timedOut
|
||||
} finally {
|
||||
client?.close()
|
||||
}
|
||||
}
|
||||
|
||||
private connect() {
|
||||
return XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: !this.plain,
|
||||
timeoutMs: this.timeoutMs,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
|
||||
/** The par2 prune (Par2Pruner) in one go: a whole pass from the first set, instead of the merger's 2 minutes an hour. */
|
||||
export default class Par2Prune extends BaseCommand {
|
||||
static commandName = 'par2:prune'
|
||||
static description = 'Drops the par2 file lists the merge can no longer use: one whole pass over every set, then exits'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
async run() {
|
||||
await db.from('worker_cursors').where('name', 'par2-prune').update({ key: '' })
|
||||
const round = await new Par2Pruner().round()
|
||||
this.logger.info(`Pruned ${round.rows} par2 file rows: ${round.complete} complete sets, ${round.judged} judged, ${round.final} final.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
|
||||
const BYTES_PER_ARTICLE = 18.5
|
||||
|
||||
/**
|
||||
* Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
|
||||
* corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
|
||||
* ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
|
||||
* fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
|
||||
* missing) and reported.
|
||||
*/
|
||||
export default class RangesCapture extends BaseCommand {
|
||||
static commandName = 'ranges:capture'
|
||||
static description =
|
||||
'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'First day of the window (ISO date)', required: true })
|
||||
declare from: string
|
||||
|
||||
@flags.string({ description: 'Day after the window (ISO date)', required: true })
|
||||
declare to: string
|
||||
|
||||
@flags.number({
|
||||
description: 'Priority for the planned ranges marked (default 0: before the history)',
|
||||
})
|
||||
declare priority?: number
|
||||
|
||||
@flags.boolean({ description: 'Only report what the window holds' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const from = new Date(this.from)
|
||||
const to = new Date(this.to)
|
||||
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
|
||||
return this.logger.error('--from and --to: ISO dates, from before to')
|
||||
const priority = this.priority ?? 0
|
||||
// A range covers est_date up to the next range's; the last range of a group up to now.
|
||||
const hits = `
|
||||
WITH spans AS (
|
||||
SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
|
||||
lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
|
||||
FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
|
||||
), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
|
||||
const { rows } = await db.rawQuery(
|
||||
`${hits}
|
||||
SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
|
||||
WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
|
||||
count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
|
||||
FROM hit GROUP BY 1 ORDER BY 1`,
|
||||
[to, from]
|
||||
)
|
||||
const fetch = rows
|
||||
.filter((r: any) => r.kind === 'planned' || r.kind === 'done')
|
||||
.reduce((sum: number, r: any) => sum + Number(r.articles), 0)
|
||||
this.logger.info(`${this.from} to ${this.to}:`)
|
||||
for (const r of rows)
|
||||
this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
|
||||
this.logger.info(
|
||||
`To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
|
||||
)
|
||||
if (this.dryRun) return
|
||||
const { rows: marked } = await db.rawQuery(
|
||||
`${hits}
|
||||
UPDATE ranges r SET capture = true,
|
||||
priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
|
||||
FROM hit WHERE r.id = hit.id AND NOT hit.capture
|
||||
AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
|
||||
RETURNING r.status`,
|
||||
[to, from, priority]
|
||||
)
|
||||
const planned = marked.filter((r: any) => r.status === 'planned').length
|
||||
this.logger.info(
|
||||
`Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
|
||||
|
||||
/**
|
||||
* When each active group's articles were posted (group_dates, sampled from the news server) and from that every
|
||||
* range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
|
||||
* posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
|
||||
* articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
|
||||
*/
|
||||
export default class RangesDates extends BaseCommand {
|
||||
static commandName = 'ranges:dates'
|
||||
static description =
|
||||
"Samples when each active group's articles were posted and estimates every range's date"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
|
||||
declare group?: string
|
||||
|
||||
@flags.number({ description: 'Articles between two samples of a group (default 500M)' })
|
||||
declare every?: number
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Connections, a share of the account beside the coordinator and the prober (default 6)',
|
||||
})
|
||||
declare connections?: number
|
||||
|
||||
@flags.boolean({ description: 'Only groups without samples yet' })
|
||||
declare missing: boolean
|
||||
|
||||
@flags.boolean({
|
||||
description: 'No sampling: estimate the ranges from the samples stored already',
|
||||
})
|
||||
declare estimateOnly: boolean
|
||||
|
||||
async run() {
|
||||
if (!this.estimateOnly) await this.sample()
|
||||
await this.estimate()
|
||||
await this.report()
|
||||
}
|
||||
|
||||
private groups(missing = this.missing) {
|
||||
const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
|
||||
if (this.group) q.where('name', 'like', this.group)
|
||||
if (missing)
|
||||
q.whereNotExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
return q
|
||||
}
|
||||
|
||||
private async sample() {
|
||||
const every = this.every ?? 500_000_000
|
||||
const connections = this.connections ?? 6
|
||||
const groups = await this.groups()
|
||||
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
|
||||
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
|
||||
// room for them), as many as asked for.
|
||||
const workers = Math.max(1, Math.min(connections, groups.length))
|
||||
const share = new NntpShare(
|
||||
'dates',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority + 1,
|
||||
workers,
|
||||
workers
|
||||
)
|
||||
let open = 0
|
||||
const sync = () =>
|
||||
share
|
||||
.sync(workers, open)
|
||||
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
|
||||
await sync()
|
||||
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
|
||||
this.logger.info(
|
||||
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
|
||||
)
|
||||
const startedAt = Date.now()
|
||||
let done = 0
|
||||
let commands = 0
|
||||
let samples = 0
|
||||
const queue = [...groups]
|
||||
const worker = async () => {
|
||||
let client: XoverClient | null = null
|
||||
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
|
||||
if (!client) {
|
||||
client = await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
open++
|
||||
}
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
client.close()
|
||||
client = null
|
||||
open--
|
||||
throw error
|
||||
}
|
||||
}
|
||||
for (let g = queue.shift(); g; g = queue.shift()) {
|
||||
try {
|
||||
const result = await sampleGroup(use, g.name, { every, size: 32 })
|
||||
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('group_dates').where('group_id', g.id).delete()
|
||||
if (result.samples.length > 0)
|
||||
await trx.table('group_dates').multiInsert(
|
||||
result.samples.map((s) => ({
|
||||
group_id: g.id,
|
||||
article: s.article,
|
||||
posted_at: s.postedAt,
|
||||
}))
|
||||
)
|
||||
})
|
||||
commands += result.commands
|
||||
samples += result.samples.length
|
||||
if (result.samples.length < 2)
|
||||
this.logger.warning(
|
||||
`${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
|
||||
)
|
||||
} catch (error: any) {
|
||||
this.logger.warning(`${g.name}: ${error.message}`)
|
||||
}
|
||||
done++
|
||||
if (done % 50 === 0)
|
||||
this.logger.info(
|
||||
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
if (client) {
|
||||
;(client as XoverClient).close()
|
||||
open--
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(Array.from({ length: workers }, worker))
|
||||
} finally {
|
||||
clearInterval(sharing)
|
||||
await share.sync(0, 0).catch(() => {})
|
||||
}
|
||||
this.logger.info(
|
||||
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
|
||||
/** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
|
||||
private async estimate() {
|
||||
const groups = await this.groups(false).whereExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
let updated = 0
|
||||
for (const g of groups) {
|
||||
const rows = await db
|
||||
.from('group_dates')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('article')
|
||||
.select('article', 'posted_at')
|
||||
const samples: DateSample[] = rows.map((r) => ({
|
||||
article: Number(r.article),
|
||||
postedAt: new Date(r.posted_at),
|
||||
}))
|
||||
const ranges = await db
|
||||
.from('ranges')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('start_id')
|
||||
.select('id', 'start_id')
|
||||
for (let i = 0; i < ranges.length; i += 1000) {
|
||||
const batch = ranges.slice(i, i + 1000)
|
||||
const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
|
||||
const bindings = batch.flatMap((r) => [
|
||||
Number(r.id),
|
||||
estimateDate(samples, Number(r.start_id))!.toISOString(),
|
||||
])
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
|
||||
bindings
|
||||
)
|
||||
updated += batch.length
|
||||
}
|
||||
}
|
||||
this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
|
||||
}
|
||||
|
||||
/** The planned history by posting year: what a window would take. */
|
||||
private async report() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
|
||||
round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
|
||||
FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
|
||||
const undatedRows = await db.rawQuery(
|
||||
`SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
|
||||
)
|
||||
const undated = undatedRows.rows[0].undated
|
||||
this.logger.info(
|
||||
`By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
|
||||
)
|
||||
for (const r of rows)
|
||||
this.logger.info(
|
||||
` ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,10 @@ import { ReleaseJudge, RULES } from '#services/release_judge'
|
||||
/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
|
||||
export default class ReleasesJudge extends BaseCommand {
|
||||
static commandName = 'releases:judge'
|
||||
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable), hourly'
|
||||
static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable): the changed ones every few minutes, all daily'
|
||||
static options: CommandOptions = { startApp: true, staysAlive: true }
|
||||
|
||||
@flags.boolean({ description: 'One round, then exit' })
|
||||
@flags.boolean({ description: 'One full round, then exit' })
|
||||
declare once: boolean
|
||||
|
||||
@flags.boolean({ description: 'With --once: count only, write no verdicts' })
|
||||
@@ -17,7 +17,7 @@ export default class ReleasesJudge extends BaseCommand {
|
||||
async run() {
|
||||
const judge = new ReleaseJudge()
|
||||
if (this.once) {
|
||||
const round = await judge.round(this.dry)
|
||||
const round = await judge.full(this.dry)
|
||||
for (const rule of RULES) {
|
||||
const r = round.rules[rule.name]
|
||||
if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { Par2Merger } from '#services/par2_merger'
|
||||
|
||||
/**
|
||||
* The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
|
||||
* first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
|
||||
*/
|
||||
export default class ReleasesMergeNames extends BaseCommand {
|
||||
static commandName = 'releases:merge-names'
|
||||
static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Sets to look at (newest first); all without' })
|
||||
declare sets: number
|
||||
|
||||
@flags.boolean({ description: 'Count and show, merge nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const merger = new Par2Merger()
|
||||
// Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
|
||||
// so microseconds survive the round trip.
|
||||
let before = ['9999-12-31', '']
|
||||
let looked = 0
|
||||
let merges = 0
|
||||
let releases = 0
|
||||
const examples: string[] = []
|
||||
while (this.sets === undefined || looked < this.sets) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT set_id, at::text AS at FROM (
|
||||
SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id) x
|
||||
WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
before
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
for (const { set_id: setId } of rows) {
|
||||
looked++
|
||||
if (this.dryRun) {
|
||||
const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
|
||||
if (found) {
|
||||
merges++
|
||||
releases += found.size
|
||||
if (examples.length < 15) {
|
||||
const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
|
||||
examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const merged = await merger.mergeByName(setId.trim())
|
||||
if (merged > 0) {
|
||||
merges++
|
||||
releases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
before = [rows.at(-1).at, rows.at(-1).set_id]
|
||||
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
for (const e of examples) this.logger.info(` ${e.slice(0, 300)}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases per batch. */
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Decides completeness by par2 set (app/services/par2_completeness.ts) for the releases that had their par2 read
|
||||
* before it was built (2026-10-23): every holder of a set whose file list is still there, by release id. Once.
|
||||
*/
|
||||
export default class ReleasesPar2Complete extends BaseCommand {
|
||||
static commandName = 'releases:par2-complete'
|
||||
static description = 'Decides completeness by their par2 sets for settled releases without a counter, then exits'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
async run() {
|
||||
let after = 0
|
||||
let looked = 0
|
||||
let decided = 0
|
||||
for (;;) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT s.release_id FROM par2_sets s
|
||||
WHERE s.release_id > ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
ORDER BY s.release_id LIMIT ?`,
|
||||
[after, BATCH]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
const ids = rows.map((r: { release_id: string }) => Number(r.release_id))
|
||||
decided += await decidePar2Completeness(db, ids)
|
||||
looked += ids.length
|
||||
after = ids.at(-1)!
|
||||
if (looked % 50_000 < BATCH) this.logger.info(`${looked} releases with a par2 set looked at, ${decided} decided (up to id ${after}).`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} releases with a par2 set looked at, ${decided} decided.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
|
||||
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||
const CURSOR = 'recategorize'
|
||||
|
||||
/**
|
||||
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
|
||||
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
|
||||
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
|
||||
* the dead versions reusable) and goes on from its cursor when run again.
|
||||
*/
|
||||
export default class ReleasesRecategorize extends BaseCommand {
|
||||
static commandName = 'releases:recategorize'
|
||||
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Release ids per batch' })
|
||||
declare batch: number
|
||||
|
||||
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
|
||||
declare toId: number
|
||||
|
||||
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
|
||||
declare floorGb: number
|
||||
|
||||
@flags.boolean({ description: 'Count what would change, write nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||
declare restart: boolean
|
||||
|
||||
async run() {
|
||||
const batch = this.batch ?? 20_000
|
||||
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
|
||||
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||
const changes = new Map<string, number>()
|
||||
let changed = 0
|
||||
let looked = 0
|
||||
const started = Date.now()
|
||||
while (after < hi) {
|
||||
while (!this.dryRun && (await space.short())) await sleep(60_000)
|
||||
const to = Math.min(after + batch, hi)
|
||||
const fresh = `release_category_for(name, inner_name, groups, NULL)`
|
||||
if (this.dryRun) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
|
||||
[after, to]
|
||||
)
|
||||
for (const r of rows) {
|
||||
looked += r.n
|
||||
if (r.was === r.now) continue
|
||||
changed += r.n
|
||||
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
|
||||
}
|
||||
} else {
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
|
||||
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
|
||||
[after, to]
|
||||
)
|
||||
changed += result.rowCount ?? 0
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, String(to)]
|
||||
)
|
||||
}
|
||||
after = to
|
||||
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
|
||||
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
|
||||
}
|
||||
}
|
||||
if (this.dryRun) {
|
||||
this.logger.info(`Would change ${changed} of ${looked}:`)
|
||||
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
|
||||
return
|
||||
}
|
||||
// The spots' categories for the linked releases whose names say nothing.
|
||||
const spots = await db.rawQuery(
|
||||
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
|
||||
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
|
||||
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
|
||||
)
|
||||
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
|
||||
/**
|
||||
* What lib/release's identity makes of releases already in the catalog: a
|
||||
@@ -27,7 +28,7 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
async run() {
|
||||
for (const name of this.group ?? []) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
|
||||
[name]
|
||||
)
|
||||
const ids = rows.map((r: { release_id: string }) => r.release_id)
|
||||
@@ -54,7 +55,7 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
}
|
||||
if (this.show?.length) return
|
||||
for (const source of ['subject', 'post', 'stem']) {
|
||||
const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
|
||||
const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
|
||||
let split = 0
|
||||
let pieces = 0
|
||||
const examples: string[] = []
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { access, readdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import Leases from '#services/leases'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { fromWire, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
/**
|
||||
* Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
|
||||
* Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)). With
|
||||
* REPLAY_INTO_CATALOG=1 any database but the test one: voyager 3 deployed, history replayed from the corpus into
|
||||
* the catalog itself (deploy/replay-window.sh, LOG.md (116)).
|
||||
*/
|
||||
const SCRATCH_DB = /_(replay|scratch)$/
|
||||
const TEST_DB = /_test$/
|
||||
const NODE_NAME = 'replay'
|
||||
|
||||
interface StoredRange {
|
||||
id: number
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
|
||||
* window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
|
||||
* scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
|
||||
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
|
||||
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
|
||||
*
|
||||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
|
||||
*
|
||||
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
|
||||
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
|
||||
*/
|
||||
export default class ReplayCorpus extends BaseCommand {
|
||||
static commandName = 'replay:corpus'
|
||||
static description =
|
||||
"Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'The group (its directory under the corpus)', required: true })
|
||||
declare group: string
|
||||
|
||||
@flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
|
||||
declare dir?: string
|
||||
|
||||
@flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
|
||||
declare from?: number
|
||||
|
||||
@flags.number({
|
||||
description: "Skip ranges starting after this article (one window of the group's history)",
|
||||
})
|
||||
declare to?: number
|
||||
|
||||
@flags.number({
|
||||
description: 'Chunks of 100k articles per batch, as the worker posts them',
|
||||
default: 5,
|
||||
})
|
||||
declare batchChunks: number
|
||||
|
||||
@flags.string({
|
||||
description:
|
||||
'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
|
||||
})
|
||||
declare bench?: string
|
||||
|
||||
@flags.boolean({
|
||||
description:
|
||||
'Leave the window open at the end (the last ranges provisional), to continue later',
|
||||
})
|
||||
declare noFinish: boolean
|
||||
|
||||
async run() {
|
||||
const database = db.connection().getReadClient().client.config.connection.database as string
|
||||
const intoCatalog = process.env.REPLAY_INTO_CATALOG === '1' && !TEST_DB.test(database)
|
||||
if (!SCRATCH_DB.test(database) && !intoCatalog) {
|
||||
this.logger.error(
|
||||
`The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}"; REPLAY_INTO_CATALOG=1 allows it)`
|
||||
)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const dir = this.dir ?? coordinatorConfig.corpus.dir
|
||||
if (!dir) {
|
||||
this.logger.error('No corpus directory: --dir or CORPUS_DIR')
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const bench = await this.benchBinary()
|
||||
if (!bench) {
|
||||
this.logger.error(
|
||||
'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
|
||||
)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const groupDir = join(dir, this.group)
|
||||
const stored = await this.storedRanges(groupDir)
|
||||
if (stored.length === 0) {
|
||||
this.logger.error(`${groupDir}: no complete range`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
const nodeId = await this.nodeId()
|
||||
const groupId = await this.groupId()
|
||||
const ranges = await this.leaseRanges(groupId, nodeId, stored)
|
||||
const rules = await RuleSets.current()
|
||||
const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
|
||||
await writeFile(rulesPath, JSON.stringify(rules))
|
||||
this.logger.info(
|
||||
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
|
||||
)
|
||||
|
||||
// edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
|
||||
const args = [
|
||||
'--group',
|
||||
groupDir,
|
||||
'--rules',
|
||||
rulesPath,
|
||||
'--batch-chunks',
|
||||
String(this.batchChunks),
|
||||
'--wire',
|
||||
'-',
|
||||
]
|
||||
if (this.from) args.push('--from', String(this.from))
|
||||
if (this.to) args.push('--to', String(this.to))
|
||||
if (this.limit) args.push('--limit', String(this.limit))
|
||||
if (this.noFinish) args.push('--no-finish')
|
||||
const started = Date.now()
|
||||
const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
|
||||
const totals = {
|
||||
batches: 0,
|
||||
headers: 0,
|
||||
releases: 0,
|
||||
newReleases: 0,
|
||||
finalized: 0,
|
||||
candidates: 0,
|
||||
storeMs: 0,
|
||||
}
|
||||
let current: StoredRange | null = null
|
||||
const storeLine = async (line: string) => {
|
||||
const batch = fromWire(JSON.parse(line) as WireBatch)
|
||||
if (!current || batch.from < current.start || batch.from > current.end) {
|
||||
current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
|
||||
if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
|
||||
}
|
||||
const at = Date.now()
|
||||
const result = await BatchStore.store(current.id, nodeId, batch)
|
||||
totals.storeMs += Date.now() - at
|
||||
totals.batches++
|
||||
totals.headers += batch.headers
|
||||
totals.releases += result.releases
|
||||
totals.newReleases += result.newReleases
|
||||
totals.finalized += result.finalized
|
||||
totals.candidates += result.candidates
|
||||
if (batch.next > current.end) {
|
||||
await Leases.finish(current.id, nodeId)
|
||||
const seconds = (Date.now() - started) / 1000
|
||||
this.logger.info(
|
||||
` range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
|
||||
`(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
|
||||
)
|
||||
}
|
||||
}
|
||||
// Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
|
||||
let rest = ''
|
||||
let failure: Error | null = null
|
||||
try {
|
||||
for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
|
||||
rest += chunk.toString('utf8')
|
||||
let nl: number
|
||||
while ((nl = rest.indexOf('\n')) >= 0) {
|
||||
const line = rest.slice(0, nl)
|
||||
rest = rest.slice(nl + 1)
|
||||
if (line.length > 0) await storeLine(line)
|
||||
}
|
||||
}
|
||||
if (rest.trim().length > 0) await storeLine(rest)
|
||||
} catch (error: any) {
|
||||
failure = error
|
||||
child.kill('SIGTERM')
|
||||
}
|
||||
const code = await new Promise<number | null>((done) =>
|
||||
child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
|
||||
)
|
||||
if (failure) {
|
||||
this.logger.error(`Replay failed: ${failure.message}`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
if (code !== 0) {
|
||||
this.logger.error(`edge-bench exited with ${code}`)
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
await this.report(groupId, ranges, Date.now() - started)
|
||||
}
|
||||
|
||||
private async benchBinary(): Promise<string | null> {
|
||||
const candidates = [
|
||||
this.bench,
|
||||
process.env.EDGE_BENCH,
|
||||
resolve('edge/target/release/edge-bench'),
|
||||
'/usr/local/bin/edge-bench',
|
||||
].filter((p): p is string => !!p)
|
||||
for (const path of candidates) {
|
||||
if (
|
||||
await access(path).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
)
|
||||
return path
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
|
||||
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
|
||||
const names = await readdir(groupDir).catch(() => [] as string[])
|
||||
const ranges: { start: number; end: number }[] = []
|
||||
for (const name of names) {
|
||||
const m = /^(\d+)-(\d+)$/.exec(name)
|
||||
if (!m) continue
|
||||
const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
|
||||
if (!json) continue
|
||||
ranges.push({ start: Number(m[1]), end: Number(m[2]) })
|
||||
}
|
||||
ranges.sort((a, b) => a.start - b.start)
|
||||
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
|
||||
if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
|
||||
if (this.limit) chosen = chosen.slice(0, this.limit)
|
||||
return chosen
|
||||
}
|
||||
|
||||
private async nodeId(): Promise<number> {
|
||||
const row = await db.from('nodes').where('name', NODE_NAME).first()
|
||||
if (row) return Number(row.id)
|
||||
const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
|
||||
return Number(inserted.id)
|
||||
}
|
||||
|
||||
/** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
|
||||
private async groupId(): Promise<number> {
|
||||
const row = await db.from('groups').where('name', this.group).first()
|
||||
if (row) return Number(row.id)
|
||||
const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
|
||||
const [inserted] = await db
|
||||
.table('groups')
|
||||
.insert({ name: this.group, priority: Number(rows[0].next), active: true })
|
||||
.returning('id')
|
||||
this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
|
||||
return Number(inserted.id)
|
||||
}
|
||||
|
||||
/** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
|
||||
private async leaseRanges(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
stored: { start: number; end: number }[]
|
||||
): Promise<StoredRange[]> {
|
||||
const out: StoredRange[] = []
|
||||
const now = new Date()
|
||||
for (const r of stored) {
|
||||
const lease = {
|
||||
status: 'leased',
|
||||
node_id: nodeId,
|
||||
next_id: r.start,
|
||||
headers: 0,
|
||||
posted_low: null,
|
||||
posted_high: null,
|
||||
leased_at: now,
|
||||
heartbeat_at: now,
|
||||
done_at: null,
|
||||
error: null,
|
||||
capture: true,
|
||||
}
|
||||
const existing = await db
|
||||
.from('ranges')
|
||||
.where('group_id', groupId)
|
||||
.where('start_id', r.start)
|
||||
.first()
|
||||
if (existing) {
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', existing.id)
|
||||
.update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
|
||||
out.push({ id: Number(existing.id), start: r.start, end: r.end })
|
||||
continue
|
||||
}
|
||||
const [inserted] = await db
|
||||
.table('ranges')
|
||||
.insert({
|
||||
...lease,
|
||||
group_id: groupId,
|
||||
start_id: r.start,
|
||||
end_id: r.end,
|
||||
priority: 0,
|
||||
attempts: 1,
|
||||
})
|
||||
.returning('id')
|
||||
out.push({ id: Number(inserted.id), start: r.start, end: r.end })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** What the replay left in the catalog (this group's releases; the sizes are the whole catalog's), per article. */
|
||||
private async report(groupId: number, ranges: StoredRange[], elapsedMs: number) {
|
||||
const ids = ranges.map((r) => r.id)
|
||||
const { rows: counts } = await db.rawQuery(
|
||||
`SELECT count(*)::bigint AS releases,
|
||||
count(settled_at)::bigint AS finished,
|
||||
count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
|
||||
count(*) FILTER (WHERE edge)::bigint AS edges,
|
||||
count(*) FILTER (WHERE complete)::bigint AS complete,
|
||||
count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
|
||||
count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
|
||||
count(*) FILTER (WHERE readable)::bigint AS readable,
|
||||
count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
|
||||
coalesce(sum(files), 0)::bigint AS files,
|
||||
coalesce(sum(segments), 0)::bigint AS segments,
|
||||
coalesce(sum(bytes), 0)::bigint AS bytes
|
||||
FROM releases WHERE group_id = ?`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: sources } = await db.rawQuery(
|
||||
`SELECT source, count(*)::bigint AS n FROM releases WHERE group_id = ? GROUP BY source ORDER BY source`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: pending } = await db.rawQuery(
|
||||
`SELECT count(*)::bigint AS n, coalesce(sum(segments), 0)::bigint AS segments, count(*) FILTER (WHERE jsonb_array_length(candidates) = 0)::bigint AS unprobable
|
||||
FROM pending_releases WHERE group_id = ?`,
|
||||
[groupId]
|
||||
)
|
||||
const { rows: headerRows } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
|
||||
[ids]
|
||||
)
|
||||
const headers = Number(headerRows[0].headers)
|
||||
const { rows: sizes } = await db.rawQuery(
|
||||
`SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
|
||||
sum(pg_total_relation_size(c.oid))::bigint AS bytes
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'r'
|
||||
AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks', 'pending_releases')
|
||||
OR c.relname LIKE 'release_chunks%')
|
||||
GROUP BY 1 ORDER BY 2 DESC`
|
||||
)
|
||||
// On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
|
||||
const hdd = new Set([
|
||||
'release_chunks',
|
||||
'probe_candidates',
|
||||
'release_inspections',
|
||||
'pending_releases',
|
||||
])
|
||||
let nvme = 0
|
||||
let other = 0
|
||||
const c = counts[0]
|
||||
const fmt = (n: unknown) => Number(n).toLocaleString('en')
|
||||
this.logger.info(
|
||||
`Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
|
||||
)
|
||||
this.logger.info(
|
||||
`Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
|
||||
`complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
|
||||
`${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
|
||||
)
|
||||
this.logger.info(
|
||||
`By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
|
||||
)
|
||||
this.logger.info(
|
||||
`Pending (unnamed, outside the catalog): ${fmt(pending[0].n)} releases, ${fmt(pending[0].segments)} segments, ${fmt(pending[0].unprobable)} without a probe target.`
|
||||
)
|
||||
for (const s of sizes) {
|
||||
const bytes = Number(s.bytes)
|
||||
if (hdd.has(s.name)) other += bytes
|
||||
else nvme += bytes
|
||||
this.logger.info(
|
||||
` ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
|
||||
)
|
||||
}
|
||||
this.logger.info(
|
||||
`NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header (the whole catalog's sizes over this group's headers: only the first group's are its own).`
|
||||
)
|
||||
}
|
||||
}
|
||||
+40
-2
@@ -31,8 +31,21 @@ export default {
|
||||
},
|
||||
/** A leased range without a heartbeat for this long goes back to the queue. */
|
||||
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
||||
/** NNTP connections the coordinator fetches headers with. */
|
||||
/** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
|
||||
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
|
||||
nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
|
||||
/**
|
||||
* The account's connections, shared by the coordinator and the prober by priority (app/services/nntp_shares.ts):
|
||||
* all of them less the reserved ones (the spotter's and the spot inspector's 1 each, the coordinator's plain one,
|
||||
* and 2 for a hedged request's or a reconnect's new connection while the old one closes).
|
||||
*/
|
||||
nntpShares: {
|
||||
total: env.get('NNTP_ACCOUNT_CONNECTIONS', 50) - env.get('NNTP_RESERVED_CONNECTIONS', 5),
|
||||
everyMs: 15_000,
|
||||
/** Headers before probes: history takes what it can use, the prober the rest. */
|
||||
coordinatorPriority: 2,
|
||||
proberPriority: 1,
|
||||
},
|
||||
/**
|
||||
* A reply with no data for this long fails, and its connection is replaced.
|
||||
* Eweka sometimes stops answering on long-lived connections (all 40 quiet,
|
||||
@@ -68,17 +81,36 @@ export default {
|
||||
* under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
|
||||
*/
|
||||
nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
|
||||
/**
|
||||
* A connection with nothing to do for this long is closed, and opened again
|
||||
* when there's work. 0: kept. Eweka drops connections that stay quiet, and
|
||||
* the next command on one failed ("Spot inspection failed: NNTP connection
|
||||
* closed"); idle ones also take the account's connections from downloads.
|
||||
*/
|
||||
nntpIdleCloseMs: env.get('NNTP_IDLE_CLOSE_SECONDS', 60) * 1000,
|
||||
probe: {
|
||||
/**
|
||||
* NNTP connections the prober reads articles with, besides the
|
||||
* At most (its share, nntpShares): NNTP connections the prober reads articles with, besides the
|
||||
* coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
|
||||
* couple stay free for downloads. ~7 MB/s each (~10 articles/s).
|
||||
*/
|
||||
connections: env.get('PROBE_CONNECTIONS', 6),
|
||||
/** Of them, always (its share's minimum; nntpShares). */
|
||||
minConnections: env.get('PROBE_MIN_CONNECTIONS', 2),
|
||||
/** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */
|
||||
minFreeGb: env.get('PROBE_MIN_FREE_GB', 0),
|
||||
/** Candidates claimed at a time. */
|
||||
batch: 400,
|
||||
/** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
|
||||
pipeline: env.get('PROBE_PIPELINE', 8),
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f; app/services/pending_ladder.ts): the share of each claim taken
|
||||
* from pending_releases at least (the catalog's own candidates come first, the rest of the claim is theirs).
|
||||
*/
|
||||
pendingShare: env.get('PENDING_SHARE', 0.25),
|
||||
/** A signature (tool, group, poster) with this many climbs and at most pendingMaxNamed of them named is dropped. */
|
||||
pendingMinProbes: env.get('PENDING_MIN_PROBES', 100),
|
||||
pendingMaxNamed: env.get('PENDING_MAX_NAMED', 0.005),
|
||||
},
|
||||
/**
|
||||
* The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
|
||||
@@ -107,6 +139,12 @@ export default {
|
||||
* ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
|
||||
*/
|
||||
ahead: env.get('CORPUS_AHEAD', 16),
|
||||
/**
|
||||
* Stored without the lines the edge worker drops on their own (crossposts another indexed group owns, obfuscated
|
||||
* subjects; ~48% of the indexed ranges' headers), what was left out in each chunk's `.drops`: twice the history
|
||||
* in the same space (LOG.md 2026-10-23 (101)). Off: as the news server sent them.
|
||||
*/
|
||||
filter: env.get('CORPUS_FILTER', true),
|
||||
},
|
||||
spool: {
|
||||
dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* releases.sidecar_only: set by the batch store when it inserts a small release (at most 3 files, under 1 MB) whose
|
||||
* files are all sidecars (.nfo, .sfv, .nzb, …). The settler queues only those for the sidecar fold: it queued every
|
||||
* small release, 13.5M (single images mostly), and the fold's queue scans held the merger back (LOG.md 2026-10-17 (72)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS sidecar_only boolean`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS sidecar_only`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* worker_cursors.key: a tie-breaker after `at`. The par2 pruner's cursor was a timestamp alone, read back to the
|
||||
* millisecond: 20,000 sets of one prober flush share created_at 2026-10-15 14:38:39.883984, and every round took the
|
||||
* same batch again (LOG.md 2026-10-17 (73)). Now (at to the microsecond, set_id).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE worker_cursors ADD COLUMN IF NOT EXISTS key text`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE worker_cursors DROP COLUMN IF EXISTS key`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* nntp_shares: the account's connections shared by the processes that read articles (app/services/nntp_shares.ts).
|
||||
* Each process's row says what it could use and has open, and what it may open; a row without a heartbeat for a
|
||||
* minute is a process gone (its connections closed with it). The split was fixed per process: with history paused
|
||||
* the coordinator's 38 stood idle while the prober's 8 were the limit (LOG.md 2026-10-18 (75)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE IF NOT EXISTS nntp_shares (
|
||||
name text PRIMARY KEY,
|
||||
priority integer NOT NULL,
|
||||
min integer NOT NULL,
|
||||
want integer NOT NULL,
|
||||
open integer NOT NULL,
|
||||
granted integer NOT NULL,
|
||||
heartbeat_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS nntp_shares`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* probe_cache's primary key in the catalog's tablespace (default_tablespace: nvme on homelab); the table stays on the
|
||||
* HDD. Each claim looks its candidates up by message_id (`message_id = ANY(…)`): with the index on the HDD that took
|
||||
* 1.27 s a claim, 48 s of each minute, the prober's lanes waiting on claims (LOG.md 2026-10-18 (76)). ~97% aren't in it,
|
||||
* so only the index is read. Homelab moved it by hand (prober stopped); a no-op where it's there already, or without a
|
||||
* default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE c.relname = 'probe_cache_pkey' AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||
[target]
|
||||
)
|
||||
if (rows.length > 0) await db.rawQuery(`ALTER INDEX probe_cache_pkey SET TABLESPACE ${target}`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Was: release_inspections' rows on the HDD (pg_default), for the NVMe's room during the probe backlog (LOG.md
|
||||
* 2026-10-18 (79)). The judge's fake rule reads all of it every round: on the HDD that took 17–18 min, was ~3
|
||||
* (LOG.md 2026-10-19 (81)). Migration 4800 moves it back; nothing here now.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The junk judge's locked rule by an index of the passworded releases (1.2M of 74M), not a scan of all of releases
|
||||
* every round (30 s; LOG.md 2026-10-19 (80)). And the posters of the spot inspector's own releases (source `spot`),
|
||||
* which it didn't add to posters: poster search, and the judge's scraped rule by posters, missed them.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_passworded_idx ON releases (id) WHERE passworded`)
|
||||
this.schema.raw(
|
||||
`INSERT INTO posters (poster) SELECT DISTINCT left(s.poster, 200) FROM spots s
|
||||
JOIN releases r ON r.id = s.release_id AND r.source = 'spot' WHERE s.poster IS NOT NULL ON CONFLICT DO NOTHING`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_passworded_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* release_inspections back in the catalog's tablespace (default_tablespace: nvme on homelab), where migration 4600 had
|
||||
* moved it to the HDD: the judge's fake rule reads all of it every round, 17–18 min on the HDD, ~3 on the NVMe (LOG.md
|
||||
* 2026-10-19 (81)). Homelab moved it by hand (prober, merger and judge stopped); a no-op where it's there already, or
|
||||
* without a default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT 1 FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE c.relname = 'release_inspections' AND c.relkind = 'r' AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||
[target]
|
||||
)
|
||||
if (rows.length > 0) await db.rawQuery(`ALTER TABLE release_inspections SET TABLESPACE ${target}`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The junk judge by what changed (LOG.md 2026-10-19 (83)): a full round over every release took 35–38 min of each
|
||||
* hour, longer with every inspection the prober added. Triggers note each release whose facts a rule reads changed
|
||||
* (judge_dirty: its probes' statuses, inspections, file hashes, passwords, spots, par2 sets, merges, its own judged
|
||||
* columns); the judge judges those every few minutes, and every release in a daily full round (judge_state: when the
|
||||
* last one was, by which rule set), its matches gathered in judge_full. judge_dirty is unlogged: what a crash loses
|
||||
* the next full round judges.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_dirty (release_id bigint NOT NULL)`)
|
||||
this.schema.raw(`CREATE UNLOGGED TABLE IF NOT EXISTS judge_full (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL)`)
|
||||
this.schema.raw(
|
||||
`CREATE TABLE IF NOT EXISTS judge_state (id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1), full_at timestamptz NOT NULL, version integer NOT NULL)`
|
||||
)
|
||||
// The verdicts are an hourly full round's: the next full one a day from now.
|
||||
this.schema.raw(`INSERT INTO judge_state (full_at, version) VALUES (now(), 1) ON CONFLICT DO NOTHING`)
|
||||
|
||||
// Rows of a table with release_id: the releases of the rows inserted, deleted, or updated (both, when moved).
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_by_release_id() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('INSERT', 'UPDATE') THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
||||
END IF;
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT release_id FROM old_rows WHERE release_id IS NOT NULL;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
// probe_candidates: only a status that counts for the dead rule, or a candidate moved by a merge (not the claims'
|
||||
// 'taken', the settler's ready).
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_probes() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
INSERT INTO judge_dirty (release_id)
|
||||
SELECT DISTINCT unnest(ARRAY[n.release_id, o.release_id]) FROM new_rows n JOIN old_rows o ON o.id = n.id
|
||||
WHERE n.release_id IS DISTINCT FROM o.release_id
|
||||
OR (n.status IS DISTINCT FROM o.status AND (n.status IN ('done', 'missing', 'error') OR o.status IN ('done', 'missing', 'error')));
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_merges() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
INSERT INTO judge_dirty (release_id) SELECT DISTINCT into_id FROM new_rows;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION judge_dirty_releases() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT id FROM new_rows;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
INSERT INTO judge_dirty (release_id) SELECT id FROM old_rows;
|
||||
ELSE
|
||||
INSERT INTO judge_dirty (release_id) VALUES (NEW.id);
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
|
||||
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets']) {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON ${table}`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON ${table}`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON ${table}`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON ${table} REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON ${table} REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON ${table} REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
}
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON probe_candidates`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON probe_candidates`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON probe_candidates REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_probes()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON probe_candidates REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON release_merges`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_merges REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_merges()`
|
||||
)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_insert ON releases`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_update ON releases`)
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_delete ON releases`)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
// Per row, and only when a column a rule reads changed: the tail and the settler update releases all the time.
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
|
||||
WHEN (OLD.name IS DISTINCT FROM NEW.name OR OLD.poster IS DISTINCT FROM NEW.poster OR OLD.source IS DISTINCT FROM NEW.source
|
||||
OR OLD.file_total IS DISTINCT FROM NEW.file_total OR OLD.passworded IS DISTINCT FROM NEW.passworded
|
||||
OR OLD.settled_at IS DISTINCT FROM NEW.settled_at OR OLD.readable IS DISTINCT FROM NEW.readable
|
||||
OR OLD.inner_name IS DISTINCT FROM NEW.inner_name OR OLD.group_id IS DISTINCT FROM NEW.group_id
|
||||
OR OLD.first_article IS DISTINCT FROM NEW.first_article OR OLD.last_article IS DISTINCT FROM NEW.last_article
|
||||
OR OLD.posted_at IS DISTINCT FROM NEW.posted_at OR OLD.last_posted_at IS DISTINCT FROM NEW.last_posted_at
|
||||
OR OLD.complete IS DISTINCT FROM NEW.complete)
|
||||
EXECUTE FUNCTION judge_dirty_releases()`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const table of ['release_inspections', 'file_hashes', 'release_passwords', 'release_spots', 'par2_sets', 'probe_candidates', 'release_merges', 'releases']) {
|
||||
for (const event of ['insert', 'update', 'delete']) this.schema.raw(`DROP TRIGGER IF EXISTS judge_dirty_${event} ON ${table}`)
|
||||
}
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS judge_dirty_by_release_id(), judge_dirty_probes(), judge_dirty_merges(), judge_dirty_releases()`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS judge_dirty, judge_full, judge_state`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* When a group's articles were posted, sampled from the news server (ranges:dates; app/services/group_dates.ts), and
|
||||
* from those each range's estimated posting date, so ranges can be chosen by era: ranges:capture --from --to marks a
|
||||
* posting-time window of every group for the corpus (voyager 3's test data; REBUILD.md, corpus). `stored_at`: the
|
||||
* range is complete in the corpus (the prefetcher saw its range.json), so the corpus fill skips it without looking.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE IF NOT EXISTS group_dates (
|
||||
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
|
||||
article bigint NOT NULL,
|
||||
posted_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (group_id, article)
|
||||
)`)
|
||||
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS est_date timestamptz`)
|
||||
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS stored_at timestamptz`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS stored_at`)
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS est_date`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS group_dates`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* file_hashes by (hash16k, size, release_id), in place of (hash16k): how many releases have a file, from the index
|
||||
* alone. The judge counts that for every file of each changed release (repeated); with the row fetched for size and
|
||||
* release_id per entry, a file posted in 100k releases cost 100k heap reads every round (24–32 s a chunk; LOG.md
|
||||
* 2026-10-19 (84)). The merger and pruner join on (hash16k, size) too.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_size_release_idx ON file_hashes (hash16k, size, release_id)`)
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_idx`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_idx ON file_hashes (hash16k)`)
|
||||
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_size_release_idx`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Probe candidates added to a release after it settled (a later range of its upload, a merge into it) stayed not
|
||||
* ready: the settler marks a release's candidates only when it settles it (LOG.md 2026-10-20 (87)). 93,642 waited
|
||||
* since 2026-10-13 (17k lone, 31k par2, 45k archive). The batch store and the merge mark them now; these are made
|
||||
* claimable once.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.status = 'waiting' AND NOT pc.ready AND r.id = pc.release_id AND r.settled_at IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release's category from more than its name (REBUILD.md, part 9; LOG.md 2026-10-23 (95)). Migration 1200 made it
|
||||
* a generated column of release_category(name), and knex took the `?` in its regexes for bindings: the function held
|
||||
* `blu-$1ray`, `web-$2dl`, `h\.$326[45]`, so BluRay, WEB-DL and H.264 names without a resolution were Misc. Now:
|
||||
* - release_category(name), fixed (no `?` in any regex here: `{0,1}`) and with XXX, audiobooks, console and PC games,
|
||||
* and TV codes the old one missed (`S01e07[`, `S04E01E02`);
|
||||
* - the inner name the prober found (an obfuscated release's real name) when the name says nothing;
|
||||
* - else a category given before (a spot's, below), else the groups' (erotica, sounds, ebooks, …);
|
||||
* - a spot's category (spot_category: its head category and subcategories), set by the spot inspector.
|
||||
* `category` becomes a plain column (DROP EXPRESSION keeps the values, no rewrite) set by a trigger; existing rows are
|
||||
* recategorized by `releases:recategorize` in batches, not here (a rewrite of the 70 GB table).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- XXX first: its names carry years, resolutions and episode-like codes too.
|
||||
WHEN name ~* '(^|[^a-z0-9])xxx([^a-z0-9]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 6045
|
||||
WHEN name ~* '720p|1080p|x26[45]|h\.{0,1}26[45]|hevc' THEN 6040
|
||||
WHEN name ~* 'imageset|picset|(^|[._ -])pics([._ -]|$)' THEN 6060 ELSE 6070 END
|
||||
-- TV: S01E02 (also S01e02[, S04E01E02), 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[^a-z0-9])(s[0-9]{1,2}[ ._-]{0,1}e[0-9]{1,3}|[0-9]{1,2}x[0-9]{2})([^0-9]|$)' OR name ~* '(^|[._ -])s[0-9]{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)[0-9]{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-{0,1}ray|web-{0,1}dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.{0,1}26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-{0,1}ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: audiobooks, lossless, mp3, or scene music names (-WEB-2026-GROUP).
|
||||
WHEN name ~* '(audiobook|audioboek|luisterboek|h(oe|ö)rbuch)' THEN 3030
|
||||
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)[0-9]{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN name ~* '(^|[._ -])(epub|ebook|e-book|mobi|azw3)([._ -]|$)' THEN 7020
|
||||
-- Games: console tags, PC scene groups and repackers.
|
||||
WHEN name ~* '(^|[._ -])(nsw|ps4|ps5|xboxone|xbox360|wiiu|3ds|nds)([._ -]|$)' THEN 1000
|
||||
WHEN name ~* '-(codex|plaza|skidrow|reloaded|flt|doge|tenoke|rune|empress|razor1911|tinyiso|darksiders|simplex)$' OR name ~* '(fitgirl|dodi|elamigos)[._ -]{0,1}repack' THEN 4050
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_group_category(groups jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
WHEN g ~* '(erotica|xxx|porn|erotic|nude|naked)' THEN CASE WHEN g ~* '(pictures|pics|images|photos)' THEN 6060 ELSE 6070 END
|
||||
WHEN g ~* 'anime' THEN 5070
|
||||
WHEN g ~* 'audiobook' THEN 3030
|
||||
WHEN g ~* '(sounds|mp3|music)' THEN CASE WHEN g ~* '(lossless|flac)' THEN 3040 ELSE 3010 END
|
||||
WHEN g ~* 'comics' THEN 7030
|
||||
WHEN g ~* 'e-{0,1}books{0,1}' THEN 7020
|
||||
WHEN g ~* '(games|nintendo|xbox|playstation|wii)' THEN 1000
|
||||
WHEN g ~* '(^|[. ])(tv|hdtv|teevee|tvseries)([. ]|$)' THEN 5000
|
||||
WHEN g ~* '(^|[. ])(movies|dvd|bluray|dvdr)([. ]|$)' THEN 2000
|
||||
ELSE 8010
|
||||
END
|
||||
FROM (SELECT coalesce(string_agg(value, ' '), '') AS g FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(groups) = 'array' THEN groups ELSE '[]' END)) x
|
||||
$$`)
|
||||
// The name's, else the inner name's, else one given before (a spot's), else the groups'.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION release_category_for(name text, inner_name text, groups jsonb, given integer) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE WHEN n <> 8010 THEN n WHEN i <> 8010 THEN i WHEN coalesce(given, 8010) <> 8010 THEN given ELSE release_group_category(groups) END
|
||||
FROM (SELECT release_category(name) AS n, coalesce(release_category(inner_name), 8010) AS i) x
|
||||
$$`)
|
||||
// Whether a spot's subcategories of one type letter ({"a": [9], "z": [0]}) hold one of `vals`.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_has(subcats jsonb, letter text, vals integer[]) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((SELECT bool_or((e #>> '{}')::integer = ANY(vals)) FROM jsonb_array_elements(
|
||||
CASE WHEN jsonb_typeof(subcats->letter) = 'array' THEN subcats->letter ELSE '[]' END) e), false)
|
||||
$$`)
|
||||
// A spot's category (lib/spotnet/categories.ts) in Newznab's; null for pictures and the unknown.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_category(category integer, subcats jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE category
|
||||
WHEN 0 THEN CASE
|
||||
WHEN spot_has(subcats, 'z', '{3}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 6045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 6040
|
||||
WHEN spot_has(subcats, 'a', '{12,13}') THEN 6060 ELSE 6070 END
|
||||
WHEN spot_has(subcats, 'z', '{1}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 5045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 5040 ELSE 5030 END
|
||||
WHEN spot_has(subcats, 'z', '{2}') OR spot_has(subcats, 'a', '{5,11}') THEN 7020
|
||||
WHEN spot_has(subcats, 'z', '{4}') THEN NULL
|
||||
ELSE CASE WHEN spot_has(subcats, 'a', '{15}') THEN 2045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 2040 ELSE 2030 END
|
||||
END
|
||||
WHEN 1 THEN CASE WHEN spot_has(subcats, 'z', '{3}') THEN 3030 WHEN spot_has(subcats, 'a', '{2,4,7,8}') OR spot_has(subcats, 'c', '{8}') THEN 3040 ELSE 3010 END
|
||||
WHEN 2 THEN CASE WHEN spot_has(subcats, 'a', '{0,1,2}') THEN 4050 ELSE 1000 END
|
||||
WHEN 3 THEN 4000
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, NEW.groups, CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`)
|
||||
// ACCESS EXCLUSIVE for a moment, while the workers write: a few tries instead of a queue behind them.
|
||||
this.schema.raw(`
|
||||
DO $$
|
||||
BEGIN
|
||||
FOR attempt IN 1..30 LOOP
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'releases' AND column_name = 'category' AND is_generated = 'ALWAYS') THEN
|
||||
ALTER TABLE releases ALTER COLUMN category DROP EXPRESSION;
|
||||
END IF;
|
||||
DROP TRIGGER IF EXISTS releases_category ON releases;
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, groups ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category();
|
||||
RETURN;
|
||||
EXCEPTION WHEN lock_not_available THEN
|
||||
PERFORM pg_sleep(2);
|
||||
END;
|
||||
END LOOP;
|
||||
RAISE EXCEPTION 'releases: no lock for the category trigger in 30 tries';
|
||||
END
|
||||
$$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS releases_category ON releases`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS releases_set_category()`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_category(integer, jsonb)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_has(jsonb, text, integer[])`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_category_for(text, text, jsonb, integer)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_group_category(jsonb)`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
|
||||
* place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
|
||||
* inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
|
||||
* groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
|
||||
* with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
|
||||
* - posters get an id; group_sets holds each distinct group combination once (~140k); releases is created anew
|
||||
* without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
|
||||
* columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
|
||||
* (null: a tail release still provisional). The category and judge triggers follow the new columns;
|
||||
* releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts
|
||||
* (migration 6200), which holds the name and every searchable string learned later.
|
||||
* - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
|
||||
* release_final() goes: a release is final once finished.
|
||||
* - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
|
||||
* - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
|
||||
* release, not by search.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
|
||||
if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
|
||||
throw new Error(
|
||||
`The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
|
||||
)
|
||||
}
|
||||
})
|
||||
this.schema.raw(`
|
||||
TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
|
||||
release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
|
||||
judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
|
||||
// Spots matched to releases that are gone: the spot inspector matches them again.
|
||||
this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
|
||||
this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)
|
||||
|
||||
this.schema.raw(`DROP TABLE IF EXISTS fragments`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
|
||||
this.schema.raw(
|
||||
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
|
||||
)
|
||||
|
||||
// Posters by id (a release's poster_id); searched by trigram as before (poster_search).
|
||||
this.schema.raw(`DROP TABLE IF EXISTS posters`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE posters (
|
||||
id serial PRIMARY KEY,
|
||||
poster text NOT NULL UNIQUE
|
||||
)`)
|
||||
this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
|
||||
// Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
|
||||
this.schema.raw(`
|
||||
CREATE TABLE group_sets (
|
||||
id serial PRIMARY KEY,
|
||||
groups jsonb NOT NULL UNIQUE
|
||||
)`)
|
||||
|
||||
this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE releases (
|
||||
id bigserial PRIMARY KEY,
|
||||
-- The upload's posting window (first and last article's Date).
|
||||
posted_at timestamptz,
|
||||
last_posted_at timestamptz,
|
||||
first_article bigint,
|
||||
last_article bigint,
|
||||
segments bigint NOT NULL DEFAULT 0,
|
||||
bytes bigint NOT NULL DEFAULT 0,
|
||||
-- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
|
||||
settled_at timestamptz,
|
||||
inspected_at timestamptz,
|
||||
first_seen_at timestamptz NOT NULL DEFAULT now(),
|
||||
-- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
|
||||
group_id integer,
|
||||
group_set_id integer NOT NULL,
|
||||
poster_id integer,
|
||||
file_total integer,
|
||||
files integer NOT NULL DEFAULT 0,
|
||||
chunks integer NOT NULL DEFAULT 0,
|
||||
category integer,
|
||||
-- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
|
||||
source smallint NOT NULL,
|
||||
readable boolean NOT NULL DEFAULT false,
|
||||
has_par2 boolean NOT NULL DEFAULT false,
|
||||
complete boolean,
|
||||
passworded boolean,
|
||||
sidecar_only boolean,
|
||||
-- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
|
||||
edge boolean NOT NULL DEFAULT false,
|
||||
-- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
|
||||
release_key bytea NOT NULL,
|
||||
identity_key bytea NOT NULL,
|
||||
name text NOT NULL,
|
||||
inner_name varchar(255)
|
||||
)`)
|
||||
this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
|
||||
this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
|
||||
// No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
|
||||
)
|
||||
|
||||
// The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
|
||||
CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
|
||||
// The judge's dirty marks (migration 4900), on the columns its rules read.
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
this.schema.raw(`
|
||||
CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
|
||||
WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
|
||||
OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
|
||||
OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
|
||||
OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
|
||||
OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
|
||||
OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
|
||||
OR old.complete IS DISTINCT FROM new.complete)
|
||||
EXECUTE FUNCTION judge_dirty_releases()`)
|
||||
|
||||
// Chunks, rule counts and samples by the block that wrote them.
|
||||
this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(
|
||||
`ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
|
||||
this.schema.raw(
|
||||
`ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
|
||||
)
|
||||
|
||||
// Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
|
||||
const hdd = env.get('CHUNK_TABLESPACE')
|
||||
if (hdd) {
|
||||
for (const table of ['release_inspections', 'probe_candidates']) {
|
||||
this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`
|
||||
DO $$ DECLARE i record; BEGIN
|
||||
FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
|
||||
EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
|
||||
END LOOP;
|
||||
END $$`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive
|
||||
* ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high),
|
||||
* read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the
|
||||
* prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and
|
||||
* counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE blocks (
|
||||
id bigserial PRIMARY KEY,
|
||||
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
|
||||
-- The first range's start and the last range's end.
|
||||
start_id bigint NOT NULL,
|
||||
end_id bigint NOT NULL,
|
||||
-- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were).
|
||||
next_id bigint NOT NULL,
|
||||
-- The posting time the block covers: estimated when planned, the worker's measured span once done.
|
||||
posted_low timestamptz,
|
||||
posted_high timestamptz,
|
||||
-- planned, leased, done, failed.
|
||||
status varchar(8) NOT NULL DEFAULT 'planned',
|
||||
-- As ranges': the tail before everything, then the history by era.
|
||||
priority integer NOT NULL DEFAULT 0,
|
||||
node_id integer,
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
rule_version integer,
|
||||
headers bigint NOT NULL DEFAULT 0,
|
||||
releases integer NOT NULL DEFAULT 0,
|
||||
leased_at timestamptz,
|
||||
heartbeat_at timestamptz,
|
||||
done_at timestamptz,
|
||||
error text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (group_id, start_id)
|
||||
)`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`)
|
||||
this.schema.raw(
|
||||
`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL`
|
||||
)
|
||||
this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`)
|
||||
// The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture,
|
||||
// est_date, stored_at, plain_chunks and the measured posted_low/high stay.
|
||||
this.schema.raw(`
|
||||
UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0,
|
||||
spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL,
|
||||
rule_version = NULL
|
||||
WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`)
|
||||
this.schema.raw(`DROP TABLE IF EXISTS blocks`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* release_texts (REBUILD.md, "Voyager 3 … Schema"): every string search matches against, one table, one trigram
|
||||
* index. The release name goes in (kind 1) with whatever anyone learns later: the prober's file names inside an
|
||||
* archive, NFO titles, par2 set names, media titles (lib/release/text_kind.ts). Appending is an insert, never an
|
||||
* update of the wide releases row; a repeat insert is a no-op by the primary key; the judge's dirty mark follows
|
||||
* (judge_dirty_by_release_id, migration 4900), so a rule can react to a name arriving later. On the NVMe: search
|
||||
* reads it. releases keeps name and inner_name for display, without trigram indexes (migration 6000).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE release_texts (
|
||||
release_id bigint NOT NULL,
|
||||
-- lib/release/text_kind.ts: 1 name, 2 file, 3 nfo, 4 par2, 5 media.
|
||||
kind smallint NOT NULL,
|
||||
text varchar(255) NOT NULL,
|
||||
PRIMARY KEY (release_id, kind, text)
|
||||
)`)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX release_texts_trgm ON release_texts USING gin (text gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_texts REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE TRIGGER judge_dirty_delete AFTER DELETE ON release_texts REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS release_texts`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* pending_releases (REBUILD.md decision f, 2026-10-26): nothing enters the catalog without a usable name. A release
|
||||
* the window closed on whose files all have random names is written here instead of releases: its keys (as the
|
||||
* catalog would store them), its counts and article span, the upload tool from its message-id (with the group its
|
||||
* signature), and its probe targets by effort (the edge's candidates: a par2, an archive start, a lone file, a 7z
|
||||
* tail), no chunk. The pending ladder probes them cheapest first and moves a named one into the catalog, reading its
|
||||
* segments again from the corpus or the server; a signature that names nothing is discarded. On CHUNK_TABLESPACE
|
||||
* (the HDD): written once, read by the ladder, never by search.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE pending_releases (
|
||||
-- lib/release/keys.ts: the identity, the group and the first article; the identity alone.
|
||||
release_key bytea PRIMARY KEY,
|
||||
identity_key bytea NOT NULL,
|
||||
group_id integer NOT NULL,
|
||||
-- The stem or post the edge keyed it by (a random name: no release name, but the inner-stem merge's handle).
|
||||
name varchar(255) NOT NULL,
|
||||
poster text,
|
||||
posted_at timestamptz,
|
||||
last_posted_at timestamptz,
|
||||
first_article bigint NOT NULL,
|
||||
last_article bigint NOT NULL,
|
||||
files integer NOT NULL DEFAULT 0,
|
||||
segments bigint NOT NULL DEFAULT 0,
|
||||
bytes bigint NOT NULL DEFAULT 0,
|
||||
-- A file misses parts.
|
||||
partial boolean NOT NULL DEFAULT false,
|
||||
-- False: a tail release still open; a later chunk of its key adds its counts up.
|
||||
finished boolean NOT NULL DEFAULT true,
|
||||
-- lib/rules uploadTool: ngPost, nyuu, hex32, random, other, …
|
||||
tool varchar(16) NOT NULL,
|
||||
-- [{kind, messageId, file}] by effort (lib/edge ProbeCandidate); empty: nothing to read, only a signature.
|
||||
candidates jsonb NOT NULL DEFAULT '[]',
|
||||
-- 0 waiting, 1 named (in the catalog), 2 discarded (its signature's verdict, or every probe gave nothing).
|
||||
status smallint NOT NULL DEFAULT 0,
|
||||
tries smallint NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
probed_at timestamptz
|
||||
)`)
|
||||
// The ladder's order: a group's and tool's waiting releases, oldest first.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX pending_releases_ladder_idx ON pending_releases (group_id, tool, status, created_at)`
|
||||
)
|
||||
const hdd = env.get('CHUNK_TABLESPACE')
|
||||
if (hdd) {
|
||||
this.schema.raw(`ALTER TABLE pending_releases SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`ALTER INDEX pending_releases_pkey SET TABLESPACE ${hdd}`)
|
||||
this.schema.raw(`ALTER INDEX pending_releases_ladder_idx SET TABLESPACE ${hdd}`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS pending_releases`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f, 2026-10-26): what the prober needs to work pending_releases cheapest
|
||||
* first and to learn which signatures never name anything.
|
||||
* - pending_releases: `file_total` (the counter's, for the catalog row once named), `inspections` (what each probe
|
||||
* showed, by kind), `named` (the usable name a probe found; status 4 until the namer moved it into the catalog).
|
||||
* Status: 0 waiting, 1 named and in the catalog, 2 discarded (its signature's verdict), 3 exhausted (every
|
||||
* target probed, no name), 4 named, awaiting the move, 9 taken by a running prober.
|
||||
* - pending_signatures: per (tool, group, poster) how many pending releases were probed to the end and how many
|
||||
* got a name; `dropped` once probed ≥ PENDING_MIN_PROBES and named ≤ PENDING_MAX_NAMED of them (voyager 2's
|
||||
* garbage learner, LOG.md 2026-10-08 (5)). The poster '' row is the tool's and group's together.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS file_total integer`)
|
||||
this.schema.raw(
|
||||
`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS inspections jsonb NOT NULL DEFAULT '[]'`
|
||||
)
|
||||
this.schema.raw(`ALTER TABLE pending_releases ADD COLUMN IF NOT EXISTS named varchar(255)`)
|
||||
// The ladder's claim: the oldest waiting ones; the namer's: the named ones.
|
||||
this.schema.raw(
|
||||
`CREATE INDEX IF NOT EXISTS pending_releases_waiting_idx ON pending_releases (created_at) WHERE status = 0`
|
||||
)
|
||||
this.schema.raw(
|
||||
`CREATE INDEX IF NOT EXISTS pending_releases_naming_idx ON pending_releases (probed_at) WHERE status = 4`
|
||||
)
|
||||
this.schema.raw(`
|
||||
CREATE TABLE pending_signatures (
|
||||
tool varchar(16) NOT NULL,
|
||||
group_id integer NOT NULL,
|
||||
poster text NOT NULL DEFAULT '',
|
||||
probed integer NOT NULL DEFAULT 0,
|
||||
named integer NOT NULL DEFAULT 0,
|
||||
dropped boolean NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (tool, group_id, poster)
|
||||
)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS pending_signatures`)
|
||||
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_naming_idx`)
|
||||
this.schema.raw(`DROP INDEX IF EXISTS pending_releases_waiting_idx`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS named`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS inspections`)
|
||||
this.schema.raw(`ALTER TABLE pending_releases DROP COLUMN IF EXISTS file_total`)
|
||||
}
|
||||
}
|
||||
+28
-7
@@ -115,11 +115,14 @@ services:
|
||||
<<: *app-env
|
||||
NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38}
|
||||
|
||||
# Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
|
||||
# connections of its own. The account has 50: the coordinator's 38 (+ its
|
||||
# plain one), the prober's 8, the spotter's and the spot inspector's 1 each,
|
||||
# 2 to spare (a hedged request closes its connection and opens another)
|
||||
# (with 10 the spot inspector got 502 Too many connections).
|
||||
# Phase 2's prober: reads the probe queue's articles over connections of its
|
||||
# own. The account has 50 (NNTP_ACCOUNT_CONNECTIONS): 5 reserved (the
|
||||
# coordinator's plain one, the spotter's and the spot inspector's 1 each, 2
|
||||
# to spare: a hedged request closes its connection and opens another; with
|
||||
# 10 the spot inspector got 502 Too many connections), the other 45 shared
|
||||
# by the coordinator (NNTP_MIN_CONNECTIONS–NNTP_CONNECTIONS) and the prober
|
||||
# (PROBE_MIN_CONNECTIONS–PROBE_CONNECTIONS), headers first
|
||||
# (app/services/nntp_shares.ts).
|
||||
prober:
|
||||
<<: *app
|
||||
command: ['node', 'ace', 'probe:run']
|
||||
@@ -127,6 +130,9 @@ services:
|
||||
environment:
|
||||
<<: *app-env
|
||||
PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8}
|
||||
# No claims under this much free on the NVMe (its overlay root): 2 GB above the 4 GB floor (2026-10-24), for
|
||||
# what the tail, the merger and the judge write meanwhile.
|
||||
PROBE_MIN_FREE_GB: ${PROBE_MIN_FREE_GB:-6}
|
||||
|
||||
# The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP.
|
||||
merger:
|
||||
@@ -134,8 +140,8 @@ services:
|
||||
command: ['node', 'ace', 'releases:merge']
|
||||
volumes: []
|
||||
|
||||
# The junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable);
|
||||
# verdicts hide them from search. No NNTP.
|
||||
# The junk judge: releases judged by what their probes showed (dead, fake, repeated, locked, unusable), the changed
|
||||
# ones every 2 minutes, all daily; verdicts hide them from search. No NNTP.
|
||||
judge:
|
||||
<<: *app
|
||||
command: ['node', 'ace', 'releases:judge']
|
||||
@@ -184,6 +190,21 @@ services:
|
||||
node-api:
|
||||
condition: service_started
|
||||
|
||||
# The replay harness into the scratch catalog usenet_replay on bulk (REBUILD.md, "The replay harness"):
|
||||
# docker compose run --rm --no-deps replay --group alt.binaries.ath
|
||||
replay:
|
||||
<<: *app
|
||||
profiles: [replay]
|
||||
restart: 'no'
|
||||
entrypoint: ['node', 'ace', 'replay:corpus']
|
||||
command: ['--help']
|
||||
environment:
|
||||
<<: *app-env
|
||||
DB_DATABASE: usenet_replay
|
||||
volumes:
|
||||
- ./corpus:/corpus:ro
|
||||
depends_on: []
|
||||
|
||||
# One-off ace commands: docker compose run --rm ace releases:duplicates
|
||||
ace:
|
||||
<<: *app
|
||||
|
||||
+8
-1
@@ -51,7 +51,14 @@ if [ -e "$CACHE/paused" ]; then
|
||||
docker compose up -d --no-deps node-api web
|
||||
else
|
||||
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
||||
docker compose up -d --no-deps edge
|
||||
if [ -e "$CACHE/no-edge" ]; then
|
||||
# Voyager 3 (2026-10-26, LOG.md (116)): the worker indexes range by range, without the carried window, and
|
||||
# would split releases at every range edge with nothing to join them; history comes from the corpus through
|
||||
# deploy/replay-window.sh until the window worker is built. Remove the marker to start it again.
|
||||
echo "== no-edge: the edge worker left stopped"
|
||||
else
|
||||
docker compose up -d --no-deps edge
|
||||
fi
|
||||
fi
|
||||
echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
|
||||
# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
|
||||
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
|
||||
# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
|
||||
# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
|
||||
# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
|
||||
# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
|
||||
# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
|
||||
# from the deployment root:
|
||||
# setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
|
||||
# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
|
||||
# DB=usenet_rebuild replays into the catalog itself (voyager 3 deployed, 2026-10-26: history comes from the corpus
|
||||
# this way until the window worker is built); the harness then needs REPLAY_INTO_CATALOG=1, passed here. OUT keeps
|
||||
# a run's logs and markers apart (.cache/replay-catalog for the catalog's).
|
||||
set -u
|
||||
ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
|
||||
PRIORITY=${PRIORITY:-0}
|
||||
PARALLEL=${PARALLEL:-4}
|
||||
MEMORY=${MEMORY:-12g}
|
||||
IMAGE=${IMAGE:-usenet-voyager3:latest}
|
||||
DB=${DB:-usenet_replay}
|
||||
OUT=${OUT:-$ROOT/.cache/replay}
|
||||
mkdir -p "$OUT"
|
||||
cd "$ROOT" || exit 1
|
||||
|
||||
say() { echo "$(date -Is) $*"; }
|
||||
catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }
|
||||
|
||||
# The window's groups: name, first article, last article, whether every range is stored; smallest first.
|
||||
window() {
|
||||
catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
|
||||
FROM ranges r JOIN groups g ON g.id = r.group_id
|
||||
WHERE r.capture AND r.priority = $PRIORITY
|
||||
GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
|
||||
}
|
||||
|
||||
replay() {
|
||||
local group=$1 from=$2 to=$3
|
||||
say "$group: replaying $from–$to"
|
||||
if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
|
||||
-e NODE_ENV=production -e HOST=0.0.0.0 -e PORT=3334 -e APP_URL=http://web:3334 -e SPOOL_DIR=/spool -e CORPUS_DIR=/corpus \
|
||||
-e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE="$DB" -e CHUNK_TABLESPACE=pg_default -e REPLAY_INTO_CATALOG=1 \
|
||||
-v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
|
||||
node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
|
||||
touch "$OUT/$group.done"
|
||||
say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
|
||||
else
|
||||
touch "$OUT/$group.failed"
|
||||
say "$group: FAILED (see $OUT/$group.log)"
|
||||
fi
|
||||
rm -f "$OUT/$group.running"
|
||||
}
|
||||
|
||||
while true; do
|
||||
todo=0
|
||||
while read -r group from to complete; do
|
||||
[ -n "$group" ] || continue
|
||||
[ -e "$OUT/$group.done" ] && continue
|
||||
[ -e "$OUT/$group.failed" ] && continue
|
||||
todo=$((todo + 1))
|
||||
[ "$complete" = 1 ] || continue
|
||||
[ -e "$OUT/$group.running" ] && continue
|
||||
[ -e "$ROOT/.cache/paused" ] && continue
|
||||
while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
|
||||
touch "$OUT/$group.running"
|
||||
replay "$group" "$from" "$to" &
|
||||
sleep 5
|
||||
done <<< "$(window)"
|
||||
if [ "$todo" = 0 ]; then
|
||||
wait
|
||||
say "every group of the window at priority $PRIORITY is replayed"
|
||||
exit 0
|
||||
fi
|
||||
sleep 300
|
||||
done
|
||||
@@ -7,5 +7,10 @@ flock 9
|
||||
rm -f .cache/paused
|
||||
date "+%F %T resuming"
|
||||
docker compose up -d --no-deps coordinator prober merger judge spotter spot-inspector
|
||||
docker compose up -d --no-deps edge
|
||||
if [ -e .cache/no-edge ]; then
|
||||
# Voyager 3 without its window worker yet (deploy.sh, LOG.md (116)): the edge stays stopped.
|
||||
echo "no-edge: the edge worker left stopped"
|
||||
else
|
||||
docker compose up -d --no-deps edge
|
||||
fi
|
||||
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
name = "edge"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)"
|
||||
description = "The edge worker (edge-node) and its benchmark (edge-bench): voyager 3's window indexing in Rust"
|
||||
|
||||
[[bin]]
|
||||
name = "edge-bench"
|
||||
|
||||
+33
-11
@@ -6,18 +6,21 @@ from the node API, reads their prefetched XOVER chunks, finds releases and
|
||||
posts batches. `edge-bench` (`src/main.rs`) replays a captured range through
|
||||
the same indexing (`src/lib.rs`) for timing and for comparing.
|
||||
|
||||
The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the
|
||||
reference: `check.ts` runs both on a capture and compares every output field
|
||||
by field. An identity or rule change goes into both and has to come out
|
||||
identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test`
|
||||
and `check-fixtures.sh`, which compares both on the committed captures in
|
||||
`fixtures/` (25k overview lines each of a.b.models, a.b.ath and
|
||||
a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they
|
||||
match.
|
||||
Since voyager 3 (2026-10-25) the Rust indexer is the one implementation of
|
||||
the window (REBUILD.md, "Voyager 3: window indexing"; `src/indexer.rs`):
|
||||
files and releases are finished when the frontier (the median Date of the
|
||||
last 1,024 plausibly dated articles) is 24 h past their last part, or at
|
||||
the block's end (provisionally, as edges), and written once. The
|
||||
TypeScript `RangeIndexer` (`lib/edge`) keeps the range path for the corpus
|
||||
scanner and the tests; the field-by-field comparison (`check.ts`) went
|
||||
with the parity. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs
|
||||
`cargo test` and `bench-fixtures.sh`, which replays the committed captures
|
||||
in `fixtures/` (25k overview lines each of a.b.ath, a.b.boneless, a.b.fta
|
||||
and a.b.models, in 5 chunks).
|
||||
|
||||
```sh
|
||||
cargo test --release --manifest-path edge/Cargo.toml
|
||||
edge/check-fixtures.sh # from the repository root
|
||||
edge/bench-fixtures.sh # from the repository root
|
||||
```
|
||||
|
||||
In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores
|
||||
@@ -110,11 +113,30 @@ files, segments, payload size, candidates, fragments and samples. The
|
||||
summary can hide compensating differences, so compare the outputs
|
||||
themselves.
|
||||
|
||||
## Comparing the output
|
||||
## Replaying a group through one window
|
||||
|
||||
A corpus group directory (`<corpus>/<group>/<start>-<end>/`, each range with
|
||||
its `range.json`) is replayed in article order with the window carried from
|
||||
range to range, the end finished after the last range (REBUILD.md, "The
|
||||
replay harness"):
|
||||
|
||||
```sh
|
||||
./target/release/edge-bench --group /corpus/alt.binaries.ath --rules rules.json --batch-chunks 5 --wire - > batches.jsonl
|
||||
# --from <article> skips ranges ending before it, --to <article> ranges starting after it (one window of the
|
||||
# group's history), --limit <n> stops after n ranges, --no-finish leaves the window open
|
||||
```
|
||||
|
||||
`--rules` takes the lease's rules JSON (`src/lease.rs`: the catalog's rule
|
||||
set and group priorities, as `replay:corpus` writes it); without it the
|
||||
defaults with only the group indexed apply. `--wire -` writes each batch as
|
||||
the worker posts it to stdout, one JSON per line, and the report to stderr;
|
||||
`node ace replay:corpus` reads them into the batch store. Each range end
|
||||
prints the frontier, the releases out and the releases and files still open.
|
||||
|
||||
## The dump
|
||||
|
||||
```sh
|
||||
./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl
|
||||
cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl
|
||||
```
|
||||
|
||||
`--dump` writes every output of every batch as JSON lines:
|
||||
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# The Rust indexer over the committed captures (edge/fixtures: 25k overview lines each of a.b.ath, a.b.boneless,
|
||||
# a.b.fta and a.b.models, in 5 chunks): a smoke test of the window on real headers, exiting non-zero when the bench
|
||||
# fails on one. CI runs it after cargo test; run it from the repository root after a change to the indexer.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
|
||||
for capture in edge/fixtures/*/; do
|
||||
echo "== $(basename "$capture")"
|
||||
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 | tail -1
|
||||
done
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/sh
|
||||
# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k
|
||||
# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared
|
||||
# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from
|
||||
# the repository root after any change to either side.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
|
||||
out=$(mktemp -d)
|
||||
trap 'rm -rf "$out"' EXIT
|
||||
status=0
|
||||
for capture in edge/fixtures/*/; do
|
||||
name=$(basename "$capture")
|
||||
edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1
|
||||
echo "== $name"
|
||||
node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1
|
||||
done
|
||||
exit $status
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
/**
|
||||
* Compares edge-bench's output (--dump) with the TypeScript RangeIndexer
|
||||
* on the same captured range, item by item: releases (every field, and the
|
||||
* payload decoded with lib/chunks), samples, probe candidates, rule counts,
|
||||
* fragments and batch boundaries.
|
||||
*
|
||||
* node --import=@poppinss/ts-exec edge/check.ts <dir> <dump.jsonl> [--wire batches.jsonl] [--batch-chunks 10] [--idle 5]
|
||||
*
|
||||
* With --wire (edge-bench --wire): also each batch as the Rust worker
|
||||
* posts it, against toWire of the TypeScript batch, field by field.
|
||||
*/
|
||||
import { readFile, readdir } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { RangeIndexer, toWire, type EdgeBatch } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
|
||||
const [dir, dumpPath] = process.argv.slice(2)
|
||||
const flag = (name: string, fallback: number) => {
|
||||
const i = process.argv.indexOf(name)
|
||||
return i === -1 ? fallback : Number(process.argv[i + 1])
|
||||
}
|
||||
const batchChunks = flag('--batch-chunks', 10)
|
||||
const idle = flag('--idle', 5)
|
||||
|
||||
const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
|
||||
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
|
||||
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
|
||||
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
|
||||
const batches: EdgeBatch[] = []
|
||||
for (const [i, file] of files.entries()) {
|
||||
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
|
||||
indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
|
||||
const last = i === files.length - 1
|
||||
if (last) indexer.finish()
|
||||
if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
|
||||
}
|
||||
|
||||
/** Canonical lines per kind, in a stable order, from both sides. */
|
||||
type Item = Record<string, unknown>
|
||||
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
|
||||
const payloadBytes = { ts: 0, rs: 0 }
|
||||
let side: 'ts' | 'rs' = 'ts'
|
||||
const release = (r: Item, b: number) => {
|
||||
const { payload, ...rest } = r as Item & { payload: Buffer }
|
||||
// The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
|
||||
// differently); what the payload decodes to is.
|
||||
payloadBytes[side] += payload.length
|
||||
return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
|
||||
}
|
||||
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
batches.forEach((batch, b) => {
|
||||
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers, postedLow: batch.postedLow, postedHigh: batch.postedHigh })
|
||||
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
||||
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
||||
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
||||
for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
|
||||
for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
|
||||
})
|
||||
|
||||
const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||
side = 'rs'
|
||||
let b = -1
|
||||
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
|
||||
if (!line) continue
|
||||
const { t, ...item } = JSON.parse(line)
|
||||
if (t === 'b') b++
|
||||
if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
|
||||
else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
|
||||
else rs[t].push({ b, ...item })
|
||||
}
|
||||
|
||||
const keyOf: Record<string, (i: Item) => string> = {
|
||||
b: (i) => String(i.b),
|
||||
r: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
s: (i) => `${i.b} ${i.key} ${i.upload}`,
|
||||
c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
|
||||
n: (i) => `${i.b} ${i.rule} ${i.key}`,
|
||||
f: (i) => `${i.b} ${i.fileKey}`,
|
||||
}
|
||||
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
|
||||
let failed = false
|
||||
for (const kind of Object.keys(keyOf)) {
|
||||
const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
|
||||
const missing = [...a.keys()].filter((k) => !z.has(k))
|
||||
const extra = [...z.keys()].filter((k) => !a.has(k))
|
||||
const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
|
||||
const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
|
||||
failed ||= !ok
|
||||
console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
|
||||
for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(` only one side: ${k}`)
|
||||
for (const k of differ.slice(0, 2)) {
|
||||
const x = a.get(k)!
|
||||
const y = z.get(k)!
|
||||
const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
|
||||
console.log(` ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
|
||||
}
|
||||
}
|
||||
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
|
||||
|
||||
const wireIndex = process.argv.indexOf('--wire')
|
||||
if (wireIndex !== -1) {
|
||||
// Order within a batch doesn't count; payloads by what they decode to.
|
||||
const by = (f: (x: any) => string) => (a: any, b: any) => (f(a) < f(b) ? -1 : f(a) > f(b) ? 1 : 0)
|
||||
const normal = (w: any) => ({
|
||||
...w,
|
||||
releases: w.releases.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
samples: w.samples.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
|
||||
candidates: [...w.candidates].sort(by((c) => JSON.stringify(c))),
|
||||
counts: [...w.counts].sort(by((c) => `${c.rule} ${c.key}`)),
|
||||
fragments: [...w.fragments].sort(by((f) => `${f.fileKey} ${f.segments[0]?.[1]}`)),
|
||||
})
|
||||
const rust = (await readFile(process.argv[wireIndex + 1], 'utf8')).split('\n').filter(Boolean).map((l) => normal(JSON.parse(l)))
|
||||
const typescript = batches.map((b) => normal(JSON.parse(JSON.stringify(toWire(b)))))
|
||||
let wireSame = rust.length === typescript.length
|
||||
const firstDiff = (x: any, y: any, path: string): string | null => {
|
||||
if (isDeepStrictEqual(x, y)) return null
|
||||
if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null) return `${path}: ${JSON.stringify(x)?.slice(0, 120)} vs ${JSON.stringify(y)?.slice(0, 120)}`
|
||||
for (const k of new Set([...Object.keys(x), ...Object.keys(y)])) {
|
||||
const d = firstDiff(x[k], y[k], `${path}.${k}`)
|
||||
if (d) return d
|
||||
}
|
||||
return path
|
||||
}
|
||||
for (let i = 0; i < Math.max(rust.length, typescript.length); i++) {
|
||||
const d = firstDiff(typescript[i], rust[i], `batch ${i}`)
|
||||
if (d) {
|
||||
wireSame = false
|
||||
console.log(`DIFF wire: ${d}`)
|
||||
break
|
||||
}
|
||||
}
|
||||
if (wireSame) console.log(`same wire batches: ${typescript.length} TypeScript, ${rust.length} Rust`)
|
||||
failed ||= !wireSame
|
||||
}
|
||||
process.exit(failed ? 1 : 0)
|
||||
@@ -9,7 +9,7 @@
|
||||
//! (the next worker resumes at the last stored batch).
|
||||
|
||||
use edge::indexer::{Batch, RangeIndexer};
|
||||
use edge::names::Rules;
|
||||
use edge::lease::LeaseJson;
|
||||
use edge::wire;
|
||||
use flate2::write::GzEncoder;
|
||||
use serde::Deserialize;
|
||||
@@ -34,63 +34,6 @@ fn log(level: &str, message: &str) {
|
||||
println!("[ {level} ] {message}");
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LeaseJson {
|
||||
range: RangeJson,
|
||||
rules: RulesJson,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RangeJson {
|
||||
id: i64,
|
||||
group: String,
|
||||
start: i64,
|
||||
end: i64,
|
||||
next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RulesJson {
|
||||
version: i64,
|
||||
priorities: HashMap<String, u32>,
|
||||
#[serde(default)]
|
||||
dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
per_article_max_parts: Option<u32>,
|
||||
#[serde(default)]
|
||||
sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
dropped_posters: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
fn rules(self) -> Rules {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
if let Some(n) = self.per_article_max_parts {
|
||||
rules.per_article_max_parts = n;
|
||||
}
|
||||
if let Some(p) = self.sample_percent {
|
||||
rules.sample_percent = p;
|
||||
}
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StoredJson {
|
||||
|
||||
+731
-146
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,65 @@
|
||||
//! A lease as the coordinator sends it (lib/edge/lease.ts): the range and the rules, JSON with the TypeScript
|
||||
//! names. The edge-node worker takes one per range; edge-bench's replay takes the rules alone (`--rules`).
|
||||
|
||||
use crate::names::Rules;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct LeaseJson {
|
||||
pub range: RangeJson,
|
||||
pub rules: RulesJson,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct RangeJson {
|
||||
pub id: i64,
|
||||
pub group: String,
|
||||
pub start: i64,
|
||||
pub end: i64,
|
||||
pub next: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RulesJson {
|
||||
pub version: i64,
|
||||
pub priorities: HashMap<String, u32>,
|
||||
#[serde(default)]
|
||||
pub dropped_signatures: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub dropped_tools: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub per_article_max_parts: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub dropped_posters: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
/// The rules with today's defaults under them (a field the stored set doesn't have keeps the default).
|
||||
pub fn rules(self) -> Rules {
|
||||
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
||||
rules.version = self.version;
|
||||
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
||||
if let Some(tools) = self.dropped_tools {
|
||||
rules.dropped_tools = tools.into_iter().collect();
|
||||
}
|
||||
if let Some(n) = self.per_article_max_parts {
|
||||
rules.per_article_max_parts = n;
|
||||
}
|
||||
if let Some(p) = self.sample_percent {
|
||||
rules.sample_percent = p;
|
||||
}
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
@@ -5,5 +5,6 @@
|
||||
pub mod codec;
|
||||
pub mod indexer;
|
||||
pub mod js;
|
||||
pub mod lease;
|
||||
pub mod names;
|
||||
pub mod wire;
|
||||
+192
-90
@@ -1,10 +1,11 @@
|
||||
//! The edge worker's indexing in Rust, on a range captured by `edge:bench
|
||||
//! --capture` (REBUILD.md, part 2: TypeScript or Rust). Same work as
|
||||
//! edge:bench's replay: inflate each chunk, scan it, aggregate files, cut off
|
||||
//! idle ones, and every N chunks judge and encode the releases. `--dump`
|
||||
//! writes every output as JSON lines for edge/check.ts.
|
||||
//! The Rust indexer over stored chunks (REBUILD.md, part 2 and "Voyager 3"): a range captured by `edge:bench
|
||||
//! --capture` or stored in the corpus, or a group's corpus ranges in order through one window (`--group`, the
|
||||
//! replay harness behind `replay:corpus`). Same work as the edge-node worker: inflate each chunk, scan it,
|
||||
//! aggregate files, finish releases through the window, and every N chunks judge and encode them. `--dump`
|
||||
//! writes every output as JSON lines; `--wire` the batches as the worker posts them (`-`: stdout).
|
||||
//!
|
||||
//! edge-bench <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl]
|
||||
//! edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json]
|
||||
//! [--batch-chunks 10] [--idle 5] [--no-finish] [--dump out.jsonl] [--wire batches.jsonl|-] [--releases out.jsonl]
|
||||
|
||||
use base64::Engine;
|
||||
use edge::indexer::{self, Batch, RangeIndexer};
|
||||
@@ -123,7 +124,7 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(",\"postedLast\":");
|
||||
opt_num(r.posted_last, line);
|
||||
line.push_str(&format!(
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"finished\":{},\"edge\":{},\"partial\":{},\"payload\":\"{}\"",
|
||||
r.files,
|
||||
r.segments,
|
||||
r.bytes,
|
||||
@@ -131,6 +132,9 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
r.last_article,
|
||||
r.readable,
|
||||
r.has_par2,
|
||||
r.finished,
|
||||
r.edge,
|
||||
r.partial,
|
||||
b64.encode(&r.payload)
|
||||
));
|
||||
};
|
||||
@@ -162,122 +166,220 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
|
||||
emit(&mut line);
|
||||
}
|
||||
for f in &batch.fragments {
|
||||
line.push_str("{\"t\":\"f\",\"fileKey\":");
|
||||
json_str(&f.file_key, &mut line);
|
||||
line.push_str(",\"name\":");
|
||||
json_str(&f.name, &mut line);
|
||||
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
|
||||
opt_num(f.posted_at, &mut line);
|
||||
line.push_str(",\"segments\":[");
|
||||
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
|
||||
if i > 0 {
|
||||
line.push(',');
|
||||
}
|
||||
line.push_str(&format!("[{n},"));
|
||||
json_str(id, &mut line);
|
||||
line.push_str(&format!(",{bytes},{article}]"));
|
||||
}
|
||||
|
||||
/// A stored range: its directory and article span (range.json).
|
||||
struct Range {
|
||||
dir: String,
|
||||
start: i64,
|
||||
end: i64,
|
||||
}
|
||||
|
||||
/// The range in a directory with a range.json, and its group.
|
||||
fn range_in(dir: &str) -> Option<(String, Range)> {
|
||||
let json = std::fs::read_to_string(format!("{dir}/range.json")).ok()?;
|
||||
Some((json_field(&json, "group"), Range { dir: dir.to_string(), start: json_field(&json, "start").parse().ok()?, end: json_field(&json, "end").parse().ok()? }))
|
||||
}
|
||||
|
||||
/// A group directory's complete ranges in article order (the corpus layout: `<group>/<start>-<end>/`).
|
||||
fn ranges_in(group_dir: &str) -> (String, Vec<Range>) {
|
||||
let mut group = String::new();
|
||||
let mut ranges: Vec<Range> = Vec::new();
|
||||
for entry in std::fs::read_dir(group_dir).unwrap_or_else(|e| panic!("{group_dir}: {e}")) {
|
||||
let path = entry.unwrap().path();
|
||||
if !path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
line.push_str("]}");
|
||||
emit(&mut line);
|
||||
if let Some((g, range)) = range_in(path.to_str().unwrap()) {
|
||||
group = g;
|
||||
ranges.push(range);
|
||||
}
|
||||
}
|
||||
ranges.sort_by_key(|r| r.start);
|
||||
(group, ranges)
|
||||
}
|
||||
|
||||
fn date(ms: Option<i64>) -> String {
|
||||
match ms.and_then(chrono::DateTime::from_timestamp_millis) {
|
||||
Some(t) => t.format("%Y-%m-%d %H:%M").to_string(),
|
||||
None => "-".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Files waiting in finished releases after which a batch goes out whatever its chunks (edge-node's limit).
|
||||
const MAX_PENDING_FILES: usize = 100_000;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let dir = args.get(1).expect("usage: edge-bench <dir> [--batch-chunks N] [--idle N] [--dump file]").clone();
|
||||
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
let has = |name: &str| args.iter().any(|a| a == name);
|
||||
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
|
||||
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
|
||||
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: a file, or stdout (`-`) for replay:corpus.
|
||||
let wire_path = flag("--wire");
|
||||
let wire_stdout = wire_path.as_deref() == Some("-");
|
||||
let mut wire_out: Option<Box<dyn Write>> = match wire_path.as_deref() {
|
||||
Some("-") => Some(Box::new(std::io::BufWriter::with_capacity(1 << 20, std::io::stdout().lock()))),
|
||||
Some(p) => Some(Box::new(std::io::BufWriter::new(std::fs::File::create(p).unwrap()))),
|
||||
None => None,
|
||||
};
|
||||
// One line per release chunk without its payload, with its first subjects (deploy/corpus_sweep.py).
|
||||
let mut releases_out = flag("--releases").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
|
||||
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
|
||||
let group = json_field(&range, "group");
|
||||
let start: i64 = json_field(&range, "start").parse().unwrap();
|
||||
let end: i64 = json_field(&range, "end").parse().unwrap();
|
||||
let mut files: Vec<String> = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||
files.sort();
|
||||
// One range, or a group's ranges in order through one window (the window carries over from range to range;
|
||||
// the end is only finished after the last one, unless --no-finish: a replay to be continued).
|
||||
let (group, ranges) = match flag("--group") {
|
||||
Some(dir) => {
|
||||
let (group, mut ranges) = ranges_in(&dir);
|
||||
// --from: ranges ending before it are skipped (a replay continued); --to: ranges starting after it (one
|
||||
// window of the group's history, the corpus holding others); --limit: this many at most.
|
||||
if let Some(from) = flag("--from").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.end >= from);
|
||||
}
|
||||
if let Some(to) = flag("--to").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.start <= to);
|
||||
}
|
||||
if let Some(limit) = flag("--limit").map(|v| v.parse::<usize>().unwrap()) {
|
||||
ranges.truncate(limit);
|
||||
}
|
||||
(group, ranges)
|
||||
}
|
||||
None => {
|
||||
let positional = (1..args.len()).find(|&i| !args[i].starts_with("--") && !args[i - 1].starts_with("--")).map(|i| args[i].clone());
|
||||
let dir = positional.expect(usage);
|
||||
let (group, range) = range_in(&dir).unwrap_or_else(|| panic!("{dir}: no range.json"));
|
||||
(group, vec![range])
|
||||
}
|
||||
};
|
||||
if ranges.is_empty() {
|
||||
eprintln!("no complete range found");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let finish = !has("--no-finish");
|
||||
let rules = match flag("--rules") {
|
||||
Some(path) => {
|
||||
let json = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
|
||||
serde_json::from_str::<edge::lease::RulesJson>(&json).unwrap_or_else(|e| panic!("{path}: {e}")).rules()
|
||||
}
|
||||
None => {
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
names::Rules::defaults(priorities)
|
||||
}
|
||||
};
|
||||
// The report goes to stderr when the wire takes stdout.
|
||||
let say = |line: String| {
|
||||
if wire_stdout {
|
||||
eprintln!("{line}");
|
||||
} else {
|
||||
println!("{line}");
|
||||
}
|
||||
};
|
||||
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, names::Rules::defaults(priorities), idle);
|
||||
let (start, end) = (ranges[0].start, ranges[ranges.len() - 1].end);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, rules, idle);
|
||||
|
||||
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
|
||||
let (mut compressed, mut text_bytes, mut chunks) = (0usize, 0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, samples
|
||||
let mut finished = 0usize;
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
let mut decompressor = libdeflater::Decompressor::new();
|
||||
let wall = Instant::now();
|
||||
for (i, file) in files.iter().enumerate() {
|
||||
let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
|
||||
compressed += raw.len();
|
||||
let t = Instant::now();
|
||||
let data = inflate(&raw, &mut decompressor);
|
||||
inflate_s += t.elapsed().as_secs_f64();
|
||||
text_bytes += data.len();
|
||||
let t = Instant::now();
|
||||
let text = String::from_utf8_lossy(&data);
|
||||
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||
indexer.add_text(&text, next);
|
||||
let last = i + 1 == files.len();
|
||||
if last {
|
||||
indexer.finish();
|
||||
let mut since_batch = 0usize;
|
||||
for (ri, range) in ranges.iter().enumerate() {
|
||||
let range_wall = Instant::now();
|
||||
let headers_before = out.0;
|
||||
// After a gap between stored ranges the batch starts at this range (every range ends with a batch taken).
|
||||
if ri > 0 {
|
||||
indexer.skip_to(range.start);
|
||||
}
|
||||
if !last && (i + 1) % batch_chunks != 0 {
|
||||
let mut files: Vec<String> = std::fs::read_dir(&range.dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).filter(|f| f.ends_with(".gz")).collect();
|
||||
files.sort();
|
||||
for (i, file) in files.iter().enumerate() {
|
||||
let raw = std::fs::read(format!("{}/{file}", range.dir)).unwrap();
|
||||
compressed += raw.len();
|
||||
let t = Instant::now();
|
||||
let data = inflate(&raw, &mut decompressor);
|
||||
inflate_s += t.elapsed().as_secs_f64();
|
||||
text_bytes += data.len();
|
||||
chunks += 1;
|
||||
let t = Instant::now();
|
||||
let text = String::from_utf8_lossy(&data);
|
||||
let next: i64 = file.split('-').nth(2).unwrap().trim_end_matches(".gz").parse::<i64>().unwrap() + 1;
|
||||
indexer.add_text(&text, next);
|
||||
since_batch += 1;
|
||||
let range_end = i + 1 == files.len();
|
||||
let last = range_end && ri + 1 == ranges.len();
|
||||
if last && finish {
|
||||
indexer.finish();
|
||||
}
|
||||
if !range_end && since_batch < batch_chunks && indexer.pending() < MAX_PENDING_FILES {
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
continue;
|
||||
}
|
||||
let batch = indexer.take();
|
||||
since_batch = 0;
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
continue;
|
||||
out.0 += batch.headers;
|
||||
out.1 += batch.releases.len();
|
||||
for r in &batch.releases {
|
||||
keys.insert(r.key.clone());
|
||||
out.2 += r.files;
|
||||
out.3 += r.segments;
|
||||
out.4 += r.payload.len();
|
||||
finished += r.finished as usize;
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
w.flush().unwrap();
|
||||
}
|
||||
if let Some(w) = releases_out.as_mut() {
|
||||
dump_releases(&batch, &group, w);
|
||||
}
|
||||
}
|
||||
let batch = indexer.take();
|
||||
index_s += t.elapsed().as_secs_f64();
|
||||
out.0 += batch.headers;
|
||||
out.1 += batch.releases.len();
|
||||
for r in &batch.releases {
|
||||
keys.insert(r.key.clone());
|
||||
out.2 += r.files;
|
||||
out.3 += r.segments;
|
||||
out.4 += r.payload.len();
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.fragments.len();
|
||||
out.7 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
}
|
||||
if let Some(w) = releases_out.as_mut() {
|
||||
dump_releases(&batch, &group, w);
|
||||
if ranges.len() > 1 {
|
||||
eprintln!(
|
||||
"{} {}–{}: {} headers in {:.1} s; frontier {}; {} releases out ({} finished), {} releases and {} files open; RSS {} MB",
|
||||
group,
|
||||
range.start,
|
||||
range.end,
|
||||
out.0 - headers_before,
|
||||
range_wall.elapsed().as_secs_f64(),
|
||||
date(indexer.frontier()),
|
||||
out.1,
|
||||
finished,
|
||||
indexer.open_releases(),
|
||||
indexer.open_files(),
|
||||
peak_rss_mb()
|
||||
);
|
||||
}
|
||||
}
|
||||
let total = inflate_s + index_s;
|
||||
#[allow(static_mut_refs)]
|
||||
let p = unsafe { indexer::PROF };
|
||||
eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
|
||||
println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
|
||||
println!(
|
||||
"{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",
|
||||
out.0,
|
||||
total,
|
||||
out.0 as f64 / total,
|
||||
wall.elapsed().as_secs_f64()
|
||||
);
|
||||
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
|
||||
println!(
|
||||
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
|
||||
say(format!("{group} {start}–{end}: {} ranges, {chunks} chunks, {} MB compressed, {} MB text", ranges.len(), compressed / 1_000_000, text_bytes / 1_000_000));
|
||||
say(format!("{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)", out.0, total, out.0 as f64 / total, wall.elapsed().as_secs_f64()));
|
||||
say(format!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb()));
|
||||
say(format!(
|
||||
" kept: {} release chunks ({} releases, {} finished), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} samples",
|
||||
out.1,
|
||||
keys.len(),
|
||||
finished,
|
||||
out.2,
|
||||
out.3,
|
||||
out.4 as f64 / 1e6,
|
||||
out.4 as f64 / out.3.max(1) as f64,
|
||||
out.5,
|
||||
out.6,
|
||||
out.7
|
||||
);
|
||||
out.6
|
||||
));
|
||||
}
|
||||
+16
-5
@@ -40,11 +40,11 @@ static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||||
(re(r"(?i)\.tar$"), ""),
|
||||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$"), ""),
|
||||
]
|
||||
});
|
||||
|
||||
@@ -147,10 +147,13 @@ fn strip_volume_digits(s: &[u8]) -> Option<usize> {
|
||||
((2..=3).contains(&d) && n > d && s[n - d - 1] == b'.').then(|| n - d - 1)
|
||||
}
|
||||
|
||||
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||||
const SPLIT_EXTS: [&[u8]; 23] = [
|
||||
b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob", b"wmv", b"mov", b"mpg", b"mpeg", b"divx", b"webm", b"flv", b"m2v",
|
||||
b"mts",
|
||||
];
|
||||
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||||
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||||
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||||
const MEDIA_EXTS: [&[u8]; 16] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf", b"wmv", b"mov", b"mpg", b"mpeg"];
|
||||
|
||||
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||||
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||||
@@ -1030,7 +1033,7 @@ static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(bin|zip)$"));
|
||||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||||
@@ -1234,6 +1237,11 @@ pub fn explored(key: &str, percent: f64) -> bool {
|
||||
}
|
||||
|
||||
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||||
|
||||
/// A sidecar by its extension (the judge's SIDECAR): no release of its own, and no album's member.
|
||||
pub fn is_sidecar(name: &str) -> bool {
|
||||
test(&SIDECAR, name)
|
||||
}
|
||||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||
|
||||
pub struct JudgedFile<'a> {
|
||||
@@ -1326,6 +1334,9 @@ mod tests {
|
||||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||||
assert_eq!(file_stem("a.7z.002"), "a");
|
||||
assert_eq!(file_stem("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
assert_eq!(file_stem("clip.mov.12"), "clip");
|
||||
assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+10
-42
@@ -1,7 +1,7 @@
|
||||
//! A batch as the coordinator takes it (lib/edge/wire.ts toWire): JSON with
|
||||
//! the TypeScript names, chunk payloads in base64.
|
||||
|
||||
use crate::indexer::{Batch, Candidate, Fragment, ReleaseChunk};
|
||||
use crate::indexer::{Batch, Candidate, ReleaseChunk};
|
||||
use base64::Engine;
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -24,6 +24,11 @@ struct WireRelease<'a> {
|
||||
last_article: i64,
|
||||
readable: bool,
|
||||
has_par2: bool,
|
||||
finished: bool,
|
||||
edge: bool,
|
||||
partial: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool: Option<&'a str>,
|
||||
payload: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
rule: Option<&'a str>,
|
||||
@@ -49,30 +54,6 @@ struct WireCount<'a> {
|
||||
files: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragmentRelease<'a> {
|
||||
key: &'a str,
|
||||
name: &'a str,
|
||||
source: &'a str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
file_total: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireFragment<'a> {
|
||||
file_key: &'a str,
|
||||
name: &'a str,
|
||||
poster: &'a str,
|
||||
subject: &'a str,
|
||||
total_parts: u32,
|
||||
groups: &'a [String],
|
||||
release: Option<WireFragmentRelease<'a>>,
|
||||
posted_at: Option<i64>,
|
||||
segments: Vec<(u32, &'a str, i64, i64)>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct WireBatch<'a> {
|
||||
@@ -85,7 +66,6 @@ struct WireBatch<'a> {
|
||||
samples: Vec<WireRelease<'a>>,
|
||||
candidates: Vec<WireCandidate<'a>>,
|
||||
counts: Vec<WireCount<'a>>,
|
||||
fragments: Vec<WireFragment<'a>>,
|
||||
}
|
||||
|
||||
fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRelease<'a> {
|
||||
@@ -106,6 +86,10 @@ fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRel
|
||||
last_article: r.last_article,
|
||||
readable: r.readable,
|
||||
has_par2: r.has_par2,
|
||||
finished: r.finished,
|
||||
edge: r.edge,
|
||||
partial: r.partial,
|
||||
tool: r.tool,
|
||||
payload: base64::engine::general_purpose::STANDARD.encode(&r.payload),
|
||||
rule: rule.map(|(rule, _)| rule),
|
||||
rule_key: rule.map(|(_, key)| key),
|
||||
@@ -116,21 +100,6 @@ fn candidate(c: &Candidate) -> WireCandidate<'_> {
|
||||
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
|
||||
}
|
||||
|
||||
fn fragment(f: &Fragment) -> WireFragment<'_> {
|
||||
WireFragment {
|
||||
file_key: &f.file_key,
|
||||
name: &f.name,
|
||||
poster: &f.poster,
|
||||
subject: &f.subject,
|
||||
total_parts: f.total_parts,
|
||||
groups: &f.groups,
|
||||
// A named file always has a release; fileTotal only when truthy, as fragmentOf.
|
||||
release: Some(WireFragmentRelease { key: &f.release.key, name: &f.release.name, source: f.release.source, file_total: f.release.file_total_truthy() }),
|
||||
posted_at: f.posted_at,
|
||||
segments: f.segments.iter().map(|(n, id, bytes, article)| (*n, id.as_str(), *bytes, *article)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// toWire(batch) as JSON bytes.
|
||||
pub fn to_json(batch: &Batch) -> Vec<u8> {
|
||||
let wire = WireBatch {
|
||||
@@ -143,7 +112,6 @@ pub fn to_json(batch: &Batch) -> Vec<u8> {
|
||||
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
||||
candidates: batch.candidates.iter().map(candidate).collect(),
|
||||
counts: batch.counts.iter().map(|(rule, key, articles, files)| WireCount { rule, key, articles: *articles, files: *files }).collect(),
|
||||
fragments: batch.fragments.iter().map(fragment).collect(),
|
||||
};
|
||||
serde_json::to_vec(&wire).unwrap()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { SegmentScanner, scanOverviewText } from './scan.js'
|
||||
|
||||
/**
|
||||
* What a corpus chunk left out (REBUILD.md, corpus; LOG.md 2026-10-23 (101)): the overview lines the edge worker only
|
||||
* counts and never indexes, decided per line as RangeIndexer.line does: a crosspost an indexed group with a lower
|
||||
* priority number owns (by owner), and a subject that names no file part (obfuscated). Article numbers ascending,
|
||||
* stored as differences. Release-level rules (garbage, per-article, executable, poster) judge files put together
|
||||
* across chunks, so they aren't applied here.
|
||||
*/
|
||||
export interface ChunkDrops {
|
||||
version: 1
|
||||
/** The rule set (rule_sets.version) and group priorities the decision was made with. */
|
||||
ruleVersion: number
|
||||
priorities: Record<string, number>
|
||||
/** Overview lines in the chunk, and kept. */
|
||||
lines: number
|
||||
kept: number
|
||||
/** By owning group, article numbers as differences (the first from 0). */
|
||||
crosspost: Record<string, number[]>
|
||||
obfuscated: number[]
|
||||
}
|
||||
|
||||
const deltas = (numbers: number[]) => numbers.map((n, i) => (i === 0 ? n : n - numbers[i - 1]))
|
||||
|
||||
/** The article numbers back from ChunkDrops differences. */
|
||||
export function undelta(differences: number[]): number[] {
|
||||
let at = 0
|
||||
return differences.map((d) => (at += d))
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk's overview text without the lines the edge worker drops on their own, and what was dropped. Lines kept
|
||||
* byte for byte (with their line ends); lines the worker skips uncounted (no number or message-id) are kept.
|
||||
*/
|
||||
export function filterOverviews(text: string, group: string, priorities: Record<string, number>, ruleVersion: number): { kept: string; drops: ChunkDrops } {
|
||||
const scanner = new SegmentScanner()
|
||||
const crosspost = new Map<string, number[]>()
|
||||
const obfuscated: number[] = []
|
||||
const dropped = new Set<number>()
|
||||
let lines = 0
|
||||
scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => {
|
||||
lines++
|
||||
if (!Number.isFinite(n) || !messageId) return
|
||||
const owner = xref ? scanner.owner(xref, group, priorities) : group
|
||||
if (owner !== group) {
|
||||
const list = crosspost.get(owner)
|
||||
if (list) list.push(n)
|
||||
else crosspost.set(owner, [n])
|
||||
dropped.add(n)
|
||||
} else if (!scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) {
|
||||
obfuscated.push(n)
|
||||
dropped.add(n)
|
||||
}
|
||||
})
|
||||
const out: string[] = []
|
||||
let start = 0
|
||||
while (start < text.length) {
|
||||
let end = text.indexOf('\n', start)
|
||||
end = end === -1 ? text.length : end + 1
|
||||
const tab = text.indexOf('\t', start)
|
||||
const n = tab !== -1 && tab < end ? Number.parseInt(text.slice(start, tab), 10) : Number.NaN
|
||||
if (!dropped.has(n)) out.push(text.slice(start, end))
|
||||
start = end
|
||||
}
|
||||
const sorted = (list: number[]) => [...list].sort((a, b) => a - b)
|
||||
return {
|
||||
kept: out.join(''),
|
||||
drops: {
|
||||
version: 1,
|
||||
ruleVersion,
|
||||
priorities: Object.fromEntries(Object.entries(priorities).filter(([name]) => crosspost.has(name) || name === group)),
|
||||
lines,
|
||||
kept: lines - dropped.size,
|
||||
crosspost: Object.fromEntries([...crosspost].map(([owner, list]) => [owner, deltas(sorted(list))])),
|
||||
obfuscated: deltas(sorted(obfuscated)),
|
||||
},
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -2,6 +2,6 @@ export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsO
|
||||
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
|
||||
export { toWire, fromWire, type WireBatch } from './wire.js'
|
||||
export type { Lease } from './lease.js'
|
||||
export { mergeFragments } from './merge.js'
|
||||
export type { EdgeSegment, FileMeta } from './scan.js'
|
||||
export { throttle } from './throttle.js'
|
||||
export { filterOverviews, undelta, type ChunkDrops } from './corpus_filter.js'
|
||||
@@ -1,35 +0,0 @@
|
||||
import type { AggregatedFile } from '../files/index.js'
|
||||
import type { Fragment } from './range_indexer.js'
|
||||
import type { EdgeSegment, FileMeta } from './scan.js'
|
||||
|
||||
/**
|
||||
* The fragments of one file from neighbouring ranges, put together
|
||||
* (REBUILD.md, fragments): the complete file when every part from 1 to its
|
||||
* total is there (the first article seen of a part wins), else null.
|
||||
*/
|
||||
export function mergeFragments(fragments: Fragment[]): AggregatedFile<EdgeSegment> | null {
|
||||
if (fragments.length === 0) return null
|
||||
const first = fragments[0]
|
||||
const meta: FileMeta = {
|
||||
fileKey: first.fileKey,
|
||||
name: first.name,
|
||||
poster: first.poster,
|
||||
subject: first.subject,
|
||||
totalParts: first.totalParts,
|
||||
groups: first.groups,
|
||||
release: first.release ? { ...first.release, source: first.release.source as any } : null,
|
||||
}
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const fragment of fragments) {
|
||||
const postedAt = fragment.postedAt ?? Number.NaN
|
||||
for (const [number, messageId, bytes, articleNumber] of fragment.segments) {
|
||||
const known = parts.get(number)
|
||||
if (!known || articleNumber < known.articleNumber) {
|
||||
parts.set(number, { fileKey: meta.fileKey, number, totalParts: meta.totalParts, messageId, bytes, articleNumber, postedAt, file: meta })
|
||||
}
|
||||
}
|
||||
}
|
||||
if (parts.size < meta.totalParts) return null
|
||||
for (let n = 1; n <= meta.totalParts; n++) if (!parts.has(n)) return null
|
||||
return { fileKey: meta.fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }
|
||||
}
|
||||
@@ -26,6 +26,20 @@ export interface ReleaseChunk {
|
||||
/** A file with a readable name; a par2 file. */
|
||||
readable: boolean
|
||||
hasPar2: boolean
|
||||
/**
|
||||
* Voyager 3: the window closed on the release (default), or it is a tail release sent provisionally, finished by a
|
||||
* later chunk of the same release key (the totals then are the release's; the payload only the files not sent yet).
|
||||
*/
|
||||
finished?: boolean
|
||||
/** Still open at its block's end, or posted within the window of its start: the block merger joins it with the neighbour block's part. */
|
||||
edge?: boolean
|
||||
/** A file of the release misses parts (the window closed on it): not complete, whatever the counter says. */
|
||||
partial?: boolean
|
||||
/**
|
||||
* Voyager 3, an unnamed release (readable false): the upload tool from its first article's message-id, its
|
||||
* signature with the group; the payload is empty (no chunk until a probe names it: pending_releases).
|
||||
*/
|
||||
tool?: string
|
||||
payload: Buffer
|
||||
}
|
||||
|
||||
@@ -88,7 +102,8 @@ export interface EdgeBatch {
|
||||
samples: DroppedChunk[]
|
||||
candidates: ProbeCandidate[]
|
||||
counts: RuleCount[]
|
||||
fragments: Fragment[]
|
||||
/** The range path's cut-off files (the TypeScript indexer); the Rust edge's window has none, the store ignores them. */
|
||||
fragments?: Fragment[]
|
||||
}
|
||||
|
||||
export interface RangeIndexerOptions {
|
||||
@@ -203,7 +218,7 @@ export class RangeIndexer {
|
||||
|
||||
/** Files completed and fragments cut since the last take: how big the next batch is. */
|
||||
get pending(): number {
|
||||
let files = this.batch.fragments.length
|
||||
let files = this.batch.fragments!.length
|
||||
for (const list of this.completed.values()) files += list.length
|
||||
return files
|
||||
}
|
||||
@@ -287,7 +302,7 @@ export class RangeIndexer {
|
||||
fragment.segments.push([s.number, s.messageId, s.bytes, s.articleNumber])
|
||||
if (Number.isFinite(s.postedAt) && (fragment.postedAt === null || s.postedAt < fragment.postedAt)) fragment.postedAt = s.postedAt
|
||||
}
|
||||
this.batch.fragments.push(fragment)
|
||||
this.batch.fragments!.push(fragment)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,6 +449,8 @@ export function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[],
|
||||
if (!tail || volume > tail.volume) tail = { volume, file: f }
|
||||
continue
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if (volume === 0 && f.segments.length > 1 && !tail) tail = { volume, file: f }
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name })
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ interface Counter {
|
||||
function findCounter(subject: string, after: number): Counter | null {
|
||||
let open = subject.length
|
||||
while (true) {
|
||||
// lastIndexOf('(', -1) searches from 0 again: a "(" at the start that isn't a counter looped forever.
|
||||
if (open === 0) return null
|
||||
open = subject.lastIndexOf('(', open - 1)
|
||||
if (open <= after) return null
|
||||
let i = open + 1
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001: its type; the listing is at
|
||||
* its end, the 7z-tail), ZIP (its first entries) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
if (/\.(bin|zip)$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */
|
||||
/** Newznab categories this indexer assigns (release_category_for() and spot_category() in SQL, migration …5200). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 1000, name: 'Console', subcats: [] },
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3030, name: 'Audiobook' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [{ id: 4050, name: 'Games' }] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }, { id: 5070, name: 'Anime' }] },
|
||||
{ id: 6000, name: 'XXX', subcats: [{ id: 6040, name: 'x264' }, { id: 6045, name: 'UHD' }, { id: 6060, name: 'ImgSet' }, { id: 6070, name: 'Other' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }, { id: 7030, name: 'Comics' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
|
||||
+8
-1
@@ -1,4 +1,11 @@
|
||||
export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
|
||||
export {
|
||||
XoverClient,
|
||||
MissingArticle,
|
||||
TruncatedReply,
|
||||
parseOverviews,
|
||||
type Overview,
|
||||
type XoverClientOptions,
|
||||
} from './xover_client.js'
|
||||
export { parseXrefGroups } from './xref.js'
|
||||
export { parseGroupResponse, type GroupInfo } from './group.js'
|
||||
export { hedge, type Request, type Hedged } from './hedge.js'
|
||||
+71
-11
@@ -36,6 +36,14 @@ const DOT_LINE = Buffer.from('.\r\n')
|
||||
*/
|
||||
/** The server doesn't have the article (430): taken down or expired. */
|
||||
export class MissingArticle extends Error {}
|
||||
/**
|
||||
* A compressed reply the server cut short: it ends its compressed output at the first ".\r\n" in it, as if that
|
||||
* were the terminator, and sends nothing more (Eweka; "78 01 2e 0d 0a" for 16 of 14k small XOVERs, LOG.md
|
||||
* 2026-10-25 (105)). The same request without compression answers.
|
||||
*/
|
||||
export class TruncatedReply extends Error {}
|
||||
/** How long a compressed reply that doesn't inflate may stay silent before it counts as cut short. */
|
||||
const TRUNCATED_MS = 2_000
|
||||
|
||||
export class XoverClient {
|
||||
private chunks: Buffer[] = []
|
||||
@@ -54,6 +62,10 @@ export class XoverClient {
|
||||
get buffered() {
|
||||
return this.length
|
||||
}
|
||||
/** The first `n` bytes received and not consumed yet (what a reply that stalled left; nntp:probe). */
|
||||
bufferedHead(n = 32): Buffer {
|
||||
return this.buffer().subarray(0, n)
|
||||
}
|
||||
private lastDataAt = 0
|
||||
private longestGap = 0
|
||||
|
||||
@@ -85,7 +97,11 @@ export class XoverClient {
|
||||
|
||||
static async connect(options: XoverClientOptions): Promise<XoverClient> {
|
||||
const sock = options.secure
|
||||
? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host })
|
||||
? tls.connect({
|
||||
host: options.host,
|
||||
port: options.port,
|
||||
servername: options.servername || options.host,
|
||||
})
|
||||
: net.connect({ host: options.host, port: options.port })
|
||||
const client = new XoverClient(sock, options.timeoutMs ?? 120_000)
|
||||
// The greeting and login are short: a server holding a connection back (too many open) fails it in 30 s, not timeoutMs.
|
||||
@@ -94,10 +110,14 @@ export class XoverClient {
|
||||
await client.expect(await client.line(), ['200', '201'])
|
||||
if (options.user) {
|
||||
const reply = await client.command(`AUTHINFO USER ${options.user}`)
|
||||
if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281'])
|
||||
if (reply.startsWith('381'))
|
||||
await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), [
|
||||
'281',
|
||||
])
|
||||
else await client.expect(reply, ['281'])
|
||||
}
|
||||
if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
if (options.compress)
|
||||
client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
} catch (error) {
|
||||
client.close()
|
||||
throw error
|
||||
@@ -127,7 +147,9 @@ export class XoverClient {
|
||||
const reply = await this.command(`LIST ACTIVE ${pattern}`)
|
||||
await this.expect(reply, ['215'])
|
||||
// With XFEATURE COMPRESS GZIP on, Eweka compresses this list too ("215 NewsGroups Follow [COMPRESS=GZIP]").
|
||||
const body = reply.includes('COMPRESS=GZIP') ? (await this.compressedRaw(true)).text : await this.plainBody()
|
||||
const body = reply.includes('COMPRESS=GZIP')
|
||||
? (await this.compressedRaw(true)).text
|
||||
: await this.plainBody()
|
||||
return body
|
||||
.split('\r\n')
|
||||
.filter(Boolean)
|
||||
@@ -154,7 +176,11 @@ export class XoverClient {
|
||||
return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
|
||||
} finally {
|
||||
this.measuring = false
|
||||
this.lastTiming = { replyMs: repliedAt - sentAt, bodyMs: performance.now() - repliedAt, longestGapMs: this.longestGap }
|
||||
this.lastTiming = {
|
||||
replyMs: repliedAt - sentAt,
|
||||
bodyMs: performance.now() - repliedAt,
|
||||
longestGapMs: this.longestGap,
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -180,7 +206,9 @@ export class XoverClient {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return []
|
||||
await this.expect(reply, ['224'])
|
||||
const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody()
|
||||
const data = reply.includes('COMPRESS=GZIP')
|
||||
? await this.compressedBody()
|
||||
: await this.plainBody()
|
||||
return parseOverviews(data)
|
||||
}
|
||||
|
||||
@@ -199,7 +227,8 @@ export class XoverClient {
|
||||
}
|
||||
|
||||
private async expect(reply: string, codes: string[]) {
|
||||
if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
if (!codes.some((code) => reply.startsWith(code)))
|
||||
throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
}
|
||||
|
||||
private buffer(): Buffer {
|
||||
@@ -223,10 +252,28 @@ export class XoverClient {
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
}
|
||||
|
||||
/** As more(), but false when nothing came within `ms`. */
|
||||
private async moreWithin(ms: number): Promise<boolean> {
|
||||
if (this.failure) throw this.failure
|
||||
const came = await new Promise<boolean>((resolve) => {
|
||||
const timer = setTimeout(() => resolve(false), ms)
|
||||
this.wake = () => {
|
||||
clearTimeout(timer)
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
this.wake = undefined
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
return came
|
||||
}
|
||||
|
||||
private async line(): Promise<string> {
|
||||
while (true) {
|
||||
const at = this.buffer().indexOf('\r\n')
|
||||
if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1')
|
||||
if (at !== -1)
|
||||
return this.take(at + 2)
|
||||
.subarray(0, at)
|
||||
.toString('latin1')
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
@@ -267,7 +314,9 @@ export class XoverClient {
|
||||
*/
|
||||
async bodies(messageIds: string[]): Promise<(Buffer | null)[]> {
|
||||
return this.timed(async () => {
|
||||
this.sock.write(messageIds.map((m) => `BODY ${m.startsWith('<') ? m : `<${m}>`}\r\n`).join(''))
|
||||
this.sock.write(
|
||||
messageIds.map((m) => `BODY ${m.startsWith('<') ? m : `<${m}>`}\r\n`).join('')
|
||||
)
|
||||
const out: (Buffer | null)[] = []
|
||||
for (let i = 0; i < messageIds.length; i++) {
|
||||
const reply = await this.line()
|
||||
@@ -309,7 +358,11 @@ export class XoverClient {
|
||||
return ''
|
||||
}
|
||||
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
|
||||
if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.')
|
||||
if (at !== -1)
|
||||
return this.take(at + TERMINATOR.length)
|
||||
.subarray(0, at + 2)
|
||||
.toString('utf8')
|
||||
.replace(/^\.\./gm, '.')
|
||||
searched = all.length
|
||||
await this.more()
|
||||
}
|
||||
@@ -359,7 +412,14 @@ export class XoverClient {
|
||||
this.take(all.length)
|
||||
return { raw, text: inflated.toString('utf8') }
|
||||
} catch {
|
||||
// ".\r\n" inside the compressed data: wait for more.
|
||||
// ".\r\n" inside the compressed data: wait for more, but not for the reply timeout: the server may have
|
||||
// cut its output there (TruncatedReply), and then nothing comes. Raw, the chunk is returned as it is
|
||||
// and reported by the worker; the text is wanted here.
|
||||
if (!(await this.moreWithin(TRUNCATED_MS))) {
|
||||
this.take(all.length)
|
||||
throw new TruncatedReply(`NNTP: compressed reply cut short after ${all.length} bytes`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
await this.more()
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
export { inspect, innerName, MAX_NAMES, PROBE_VERSION, type Inspection } from './inspect.js'
|
||||
export { inspect, innerName, usableName, MAX_NAMES, PROBE_VERSION, type Inspection } from './inspect.js'
|
||||
+77
-9
@@ -2,7 +2,10 @@ import { createHash } from 'node:crypto'
|
||||
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||
import { parseZipStart } from '#lib/zip/index'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
import { fileStem, looksRandom } from '#lib/release/index'
|
||||
|
||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||
export interface Inspection {
|
||||
@@ -23,7 +26,10 @@ export interface Inspection {
|
||||
*/
|
||||
hash16k?: string
|
||||
/** A par2 body's recovery set: every file it describes (not kept in release_inspections; par2_files). */
|
||||
par2?: { setId: string; files: { fileId: string; hash16k: string; length: number; name: string }[] }
|
||||
par2?: {
|
||||
setId: string
|
||||
files: { fileId: string; hash16k: string; length: number; name: string }[]
|
||||
}
|
||||
}
|
||||
|
||||
const HASH_BYTES = 16 * 1024
|
||||
@@ -55,12 +61,22 @@ function largestFirst(items: { name: string; size: number }[]): string[] {
|
||||
.map((i) => i.name.slice(0, MAX_NAME_LENGTH))
|
||||
}
|
||||
|
||||
/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
|
||||
export function inspect(body: Buffer): Inspection {
|
||||
/**
|
||||
* One article's body: its yEnc header, what the first bytes are, and for par2, RAR, ZIP and 7z what's inside.
|
||||
* Throws when it isn't yEnc. `kind` is the probe candidate's: a 7z-tail's article ends the archive.
|
||||
*/
|
||||
export function inspect(body: Buffer, kind?: string): Inspection {
|
||||
const segment = decodeBody(body)
|
||||
const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
|
||||
const result: Inspection = {
|
||||
yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH),
|
||||
size: segment.meta.begin.size,
|
||||
type: sniffContentType(segment.data),
|
||||
}
|
||||
// The file's first 16 KiB, or all of a smaller file.
|
||||
if (segment.offset === 0 && (segment.data.length >= HASH_BYTES || segment.data.length === result.size)) {
|
||||
if (
|
||||
segment.offset === 0 &&
|
||||
(segment.data.length >= HASH_BYTES || segment.data.length === result.size)
|
||||
) {
|
||||
result.hash16k = createHash('md5').update(segment.data.subarray(0, HASH_BYTES)).digest('hex')
|
||||
}
|
||||
if (isPar2(segment.data)) {
|
||||
@@ -71,7 +87,12 @@ export function inspect(body: Buffer): Inspection {
|
||||
result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
|
||||
result.par2 = {
|
||||
setId: set.recoverySetId,
|
||||
files: set.files.map((f) => ({ fileId: f.fileId, hash16k: f.md5_16k, length: f.length, name: f.name.slice(0, MAX_NAME_LENGTH) })),
|
||||
files: set.files.map((f) => ({
|
||||
fileId: f.fileId,
|
||||
hash16k: f.md5_16k,
|
||||
length: f.length,
|
||||
name: f.name.slice(0, MAX_NAME_LENGTH),
|
||||
})),
|
||||
}
|
||||
}
|
||||
return result
|
||||
@@ -85,19 +106,66 @@ export function inspect(body: Buffer): Inspection {
|
||||
result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
|
||||
}
|
||||
}
|
||||
if (result.type === 'zip') {
|
||||
const start = parseZipStart(segment.data)
|
||||
if (start) {
|
||||
result.entries = start.files.length
|
||||
result.names = largestFirst(start.files)
|
||||
result.passworded = start.encrypted
|
||||
}
|
||||
}
|
||||
// 7z keeps its listing at the end: the last segment of the last volume (a 7z-tail), or a small archive whole.
|
||||
if (
|
||||
(kind === '7z-tail' || result.type === '7z') &&
|
||||
segment.offset + segment.data.length === result.size
|
||||
) {
|
||||
const listing = parseSevenZipTail(segment.data)
|
||||
if (listing) {
|
||||
result.entries = listing.files.length
|
||||
if (listing.files.length > 0) result.names = largestFirst(listing.files)
|
||||
result.passworded = listing.encrypted
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* The usable name one probe of an unnamed release found (the pending ladder, REBUILD.md decision f): the largest
|
||||
* named file of a par2 set or an archive's listing, or a lone file's yEnc name, when it isn't packaging and isn't
|
||||
* random itself (an obfuscated upload's par2 names its random volumes; its yEnc name is mostly the random name
|
||||
* again). Null: this probe named nothing, the next target is tried.
|
||||
*/
|
||||
export function usableName(inspection: Inspection, kind: string): string | null {
|
||||
const usable = (n: string | undefined) => {
|
||||
if (!n) return null
|
||||
const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null
|
||||
}
|
||||
const named = (inspection.names ?? []).map(usable).find((n) => n)
|
||||
if (named) return named
|
||||
return kind === 'lone' || !inspection.names?.length ? usable(inspection.yencName) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* A release's best inner name from its inspections (by probe kind): the
|
||||
* largest named entry of an archive, else the largest file of a par2 set
|
||||
* that isn't packaging, else a lone file's yEnc name. Null when none says
|
||||
* more than the release's own name.
|
||||
*/
|
||||
export function innerName(inspections: { kind: string; result: Inspection }[], releaseName = ''): string | null {
|
||||
export function innerName(
|
||||
inspections: { kind: string; result: Inspection }[],
|
||||
releaseName = ''
|
||||
): string | null {
|
||||
const of = (kind: string) => inspections.filter((i) => i.kind === kind).map((i) => i.result)
|
||||
const content = (kind: string) => of(kind).flatMap((i) => i.names ?? []).find((n) => !PACKAGING.test(n))
|
||||
const pick = content('archive') ?? content('par2') ?? of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const content = (kind: string) =>
|
||||
of(kind)
|
||||
.flatMap((i) => i.names ?? [])
|
||||
.find((n) => !PACKAGING.test(n))
|
||||
const pick =
|
||||
content('archive') ??
|
||||
content('7z-tail') ??
|
||||
content('par2') ??
|
||||
of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const name = pick?.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
// The release's own name with an extension says nothing new (a random-named file's yEnc name is mostly its random name).
|
||||
return name && !name.toLowerCase().startsWith(releaseName.toLowerCase() || '\0') ? name : null
|
||||
|
||||
@@ -40,12 +40,13 @@ const SUFFIXES = [
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r\d{2,3}))\.\d{2,3}$/i,
|
||||
// Also a split rar volume ("rld-gtre.r13.002": 1,620 single files of usenet-town's posts), and other video
|
||||
// ("Morning Lovers.wmv.001" to ".058" were 58 releases; LOG.md 2026-10-23 (97)).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r\d{2,3}))\.\d{2,3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* The catalog's keys as voyager 3 stores them (migration 6000): 20-byte SHA-1s in bytea columns.
|
||||
* - identity_key: the identity's hash (lib/release/identity.ts gives it as 40 hex characters).
|
||||
* - release_key: one upload of an identity, unique: the identity, the owner group and the first article the window
|
||||
* saw of it. The same release sent twice (a tail release provisionally, then finished) hits the same row; a repost
|
||||
* starts at another article and is a release of its own (decision d). Deterministic, so no numbering under a lock.
|
||||
*/
|
||||
const HEX40 = /^[0-9a-f]{40}$/i
|
||||
|
||||
function sha1(text: string): Buffer {
|
||||
return createHash('sha1').update(text).digest()
|
||||
}
|
||||
|
||||
/** The identity key's bytes: 40 hex characters decoded, any other text (a spot's, a test's) hashed. */
|
||||
export function identityKey(key: string): Buffer {
|
||||
return HEX40.test(key) ? Buffer.from(key, 'hex') : sha1(key)
|
||||
}
|
||||
|
||||
/** The release key's bytes for an identity in a group, first seen at an article (null for releases without one: a spot's). */
|
||||
export function releaseKey(
|
||||
identity: string,
|
||||
groupId: number | null,
|
||||
firstArticle: number | null
|
||||
): Buffer {
|
||||
return sha1(`release\n${identity}\n${groupId ?? ''}\n${firstArticle ?? ''}`)
|
||||
}
|
||||
|
||||
/** A stored key as hex, for logs, the API and release_merges.from_key. */
|
||||
export function keyHex(key: Buffer | Uint8Array | string | null | undefined): string | null {
|
||||
if (key === null || key === undefined) return null
|
||||
return typeof key === 'string' ? key : Buffer.from(key).toString('hex')
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* How a release got its name, as releases.source stores it (a smallint; voyager 3, migration 6000). The names are
|
||||
* what the edge, the spot inspector and the API use; the numbers never change meaning once written.
|
||||
*/
|
||||
export const SOURCES = {
|
||||
/** A name the subject announces ("name" yEnc (k/N)). */
|
||||
subject: 1,
|
||||
/** The post's files, named after the first file (no release name in the subject). */
|
||||
post: 2,
|
||||
/** The files' shared stem. */
|
||||
stem: 3,
|
||||
/** A Spotnet spot's files that matched no release. */
|
||||
spot: 4,
|
||||
/** An NZB's files. */
|
||||
nzb: 5,
|
||||
/** One poster's files that differ only by a number sequence, or share a prefix (voyager 3). */
|
||||
album: 6,
|
||||
/** An obfuscated post's stem, read from the yEnc body (identity.ts: weak). */
|
||||
'weak-stem': 7,
|
||||
} as const
|
||||
|
||||
export type Source = keyof typeof SOURCES
|
||||
|
||||
const NAMES = Object.fromEntries(
|
||||
Object.entries(SOURCES).map(([name, code]) => [code, name])
|
||||
) as Record<number, Source>
|
||||
|
||||
/** The code of a source name; throws for an unknown one (a typo would write a wrong number for good). */
|
||||
export function sourceCode(name: string): number {
|
||||
const code = SOURCES[name as Source]
|
||||
if (code === undefined) throw new Error(`Unknown release source "${name}"`)
|
||||
return code
|
||||
}
|
||||
|
||||
/** The name of a stored code; "unknown" for one not listed (never written, but a row might be read by an older build). */
|
||||
export function sourceName(code: number | string | null | undefined): Source | 'unknown' {
|
||||
return NAMES[Number(code)] ?? 'unknown'
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* What a searchable string of a release is, as release_texts.kind stores it (a smallint; voyager 3, migration 6200).
|
||||
* The table holds every string search matches against (the release name too), appended by whoever learns one: the
|
||||
* batch store the name, the prober the file names inside an archive, later passes NFO titles, par2 set names,
|
||||
* media titles. The numbers never change meaning once written.
|
||||
*/
|
||||
export const TEXT_KINDS = {
|
||||
/** The release name (releases.name, once more for the one trigram index). */
|
||||
name: 1,
|
||||
/** A file name inside an archive (the prober's inner names). */
|
||||
file: 2,
|
||||
/** A title read from an NFO. */
|
||||
nfo: 3,
|
||||
/** A par2 set's name when it differs from the release name. */
|
||||
par2: 4,
|
||||
/** A title from a media container's metadata. */
|
||||
media: 5,
|
||||
/** An album's member, by its own file name (the edge's albums: one poster's files differing by a number sequence). */
|
||||
member: 6,
|
||||
} as const
|
||||
|
||||
export type TextKind = keyof typeof TEXT_KINDS
|
||||
|
||||
/** The longest string stored (release_texts.text is a varchar(255)); longer ones are cut, not refused. */
|
||||
export const TEXT_MAX = 255
|
||||
/** The most strings one writer appends per release and kind (an archive of two thousand files is not two thousand rows). */
|
||||
export const TEXTS_PER_KIND = 20
|
||||
|
||||
const NAMES = Object.fromEntries(
|
||||
Object.entries(TEXT_KINDS).map(([name, code]) => [code, name])
|
||||
) as Record<number, TextKind>
|
||||
|
||||
/** The code of a kind; throws for an unknown one (a typo would write a wrong number for good). */
|
||||
export function textKindCode(name: string): number {
|
||||
const code = TEXT_KINDS[name as TextKind]
|
||||
if (code === undefined) throw new Error(`Unknown release text kind "${name}"`)
|
||||
return code
|
||||
}
|
||||
|
||||
/** The name of a stored code; "unknown" for one not listed. */
|
||||
export function textKindName(code: number | string | null | undefined): TextKind | 'unknown' {
|
||||
return NAMES[Number(code)] ?? 'unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* The strings a writer appends for one kind: trimmed, cut to TEXT_MAX, empty ones and repeats dropped, the first
|
||||
* TEXTS_PER_KIND kept in the order given (the writer puts the most telling first).
|
||||
*/
|
||||
export function textsToStore(texts: Iterable<string | null | undefined>): string[] {
|
||||
const seen = new Set<string>()
|
||||
const out: string[] = []
|
||||
for (const raw of texts) {
|
||||
const text = (raw ?? '').trim().slice(0, TEXT_MAX)
|
||||
if (!text || seen.has(text)) continue
|
||||
seen.add(text)
|
||||
out.push(text)
|
||||
if (out.length === TEXTS_PER_KIND) break
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { parseSevenZipTail, type SevenZipListing } from '#lib/sevenzip/parse'
|
||||
export { lzmaDecode, LzmaError } from '#lib/sevenzip/lzma'
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* An LZMA decoder (raw stream, as a 7z coder stores it: the properties apart, the unpacked size known), after the
|
||||
* LZMA SDK's reference decoder (LzmaSpec.cpp). For 7z headers: those are small, so the whole output is one buffer.
|
||||
*/
|
||||
|
||||
const MODEL_BITS = 11
|
||||
const BIT_MODEL_TOTAL = 1 << MODEL_BITS
|
||||
const MOVE_BITS = 5
|
||||
const TOP = 1 << 24
|
||||
const POS_BITS_MAX = 4
|
||||
const STATES = 12
|
||||
const LEN_TO_POS_STATES = 4
|
||||
const ALIGN_BITS = 4
|
||||
const END_POS_MODEL_INDEX = 14
|
||||
const FULL_DISTANCES = 1 << (END_POS_MODEL_INDEX >> 1)
|
||||
const MATCH_MIN_LEN = 2
|
||||
|
||||
export class LzmaError extends Error {}
|
||||
|
||||
const probs = (n: number) => new Uint16Array(n).fill(BIT_MODEL_TOTAL >>> 1)
|
||||
|
||||
class RangeDecoder {
|
||||
private range = 0xffffffff
|
||||
private code = 0
|
||||
|
||||
constructor(
|
||||
private input: Uint8Array,
|
||||
private pos: number
|
||||
) {
|
||||
if (this.byte() !== 0) throw new LzmaError('LZMA: bad stream start')
|
||||
for (let i = 0; i < 4; i++) this.code = ((this.code << 8) | this.byte()) >>> 0
|
||||
if (this.code === this.range) throw new LzmaError('LZMA: corrupt stream')
|
||||
}
|
||||
|
||||
private byte(): number {
|
||||
if (this.pos >= this.input.length) throw new LzmaError('LZMA: input ended')
|
||||
return this.input[this.pos++]
|
||||
}
|
||||
|
||||
private normalize() {
|
||||
if (this.range < TOP) {
|
||||
this.range = (this.range << 8) >>> 0
|
||||
this.code = ((this.code << 8) | this.byte()) >>> 0
|
||||
}
|
||||
}
|
||||
|
||||
direct(bits: number): number {
|
||||
let result = 0
|
||||
do {
|
||||
this.range >>>= 1
|
||||
this.code = (this.code - this.range) >>> 0
|
||||
const t = this.code >>> 31 ? 0xffffffff : 0
|
||||
this.code = (this.code + (this.range & t)) >>> 0
|
||||
this.normalize()
|
||||
result = ((result << 1) | (t ? 0 : 1)) >>> 0
|
||||
} while (--bits)
|
||||
return result
|
||||
}
|
||||
|
||||
bit(p: Uint16Array, i: number): number {
|
||||
const v = p[i]
|
||||
const bound = ((this.range >>> MODEL_BITS) * v) >>> 0
|
||||
let symbol: number
|
||||
if (this.code < bound) {
|
||||
p[i] = v + ((BIT_MODEL_TOTAL - v) >>> MOVE_BITS)
|
||||
this.range = bound
|
||||
symbol = 0
|
||||
} else {
|
||||
p[i] = v - (v >>> MOVE_BITS)
|
||||
this.code = (this.code - bound) >>> 0
|
||||
this.range = (this.range - bound) >>> 0
|
||||
symbol = 1
|
||||
}
|
||||
this.normalize()
|
||||
return symbol
|
||||
}
|
||||
|
||||
tree(p: Uint16Array, offset: number, bits: number): number {
|
||||
let m = 1
|
||||
for (let i = 0; i < bits; i++) m = (m << 1) + this.bit(p, offset + m)
|
||||
return m - (1 << bits)
|
||||
}
|
||||
|
||||
reverseTree(p: Uint16Array, offset: number, bits: number): number {
|
||||
let m = 1
|
||||
let symbol = 0
|
||||
for (let i = 0; i < bits; i++) {
|
||||
const bit = this.bit(p, offset + m)
|
||||
m = (m << 1) + bit
|
||||
symbol |= bit << i
|
||||
}
|
||||
return symbol
|
||||
}
|
||||
}
|
||||
|
||||
class LenDecoder {
|
||||
private choice = probs(2)
|
||||
private low = probs((1 << POS_BITS_MAX) << 3)
|
||||
private mid = probs((1 << POS_BITS_MAX) << 3)
|
||||
private high = probs(1 << 8)
|
||||
|
||||
decode(rc: RangeDecoder, posState: number): number {
|
||||
if (rc.bit(this.choice, 0) === 0) return rc.tree(this.low, posState << 3, 3)
|
||||
if (rc.bit(this.choice, 1) === 0) return 8 + rc.tree(this.mid, posState << 3, 3)
|
||||
return 16 + rc.tree(this.high, 0, 8)
|
||||
}
|
||||
}
|
||||
|
||||
/** Decodes `size` bytes of a raw LZMA stream with its 5 property bytes (lc/lp/pb, dictionary size). */
|
||||
export function lzmaDecode(properties: Uint8Array, input: Uint8Array, size: number): Buffer {
|
||||
if (properties.length < 5) throw new LzmaError('LZMA: short properties')
|
||||
let d = properties[0]
|
||||
if (d >= 9 * 5 * 5) throw new LzmaError('LZMA: bad properties')
|
||||
const lc = d % 9
|
||||
d = Math.floor(d / 9)
|
||||
const lp = d % 5
|
||||
const pb = Math.floor(d / 5)
|
||||
const dictSize = Math.max(properties[1] | (properties[2] << 8) | (properties[3] << 16) | (properties[4] << 24), 1 << 12) >>> 0
|
||||
|
||||
const out = Buffer.alloc(size)
|
||||
let pos = 0
|
||||
const rc = new RangeDecoder(input, 0)
|
||||
const literals = probs(0x300 << (lc + lp))
|
||||
const posSlot = probs(LEN_TO_POS_STATES << 6)
|
||||
const align = probs(1 << ALIGN_BITS)
|
||||
const posDecoders = probs(1 + FULL_DISTANCES - END_POS_MODEL_INDEX)
|
||||
const isMatch = probs(STATES << POS_BITS_MAX)
|
||||
const isRep = probs(STATES)
|
||||
const isRepG0 = probs(STATES)
|
||||
const isRepG1 = probs(STATES)
|
||||
const isRepG2 = probs(STATES)
|
||||
const isRep0Long = probs(STATES << POS_BITS_MAX)
|
||||
const lenDecoder = new LenDecoder()
|
||||
const repLenDecoder = new LenDecoder()
|
||||
|
||||
const distance = (len: number): number => {
|
||||
const lenState = Math.min(len, LEN_TO_POS_STATES - 1)
|
||||
const slot = rc.tree(posSlot, lenState << 6, 6)
|
||||
if (slot < 4) return slot
|
||||
const directBits = (slot >>> 1) - 1
|
||||
let dist = ((2 | (slot & 1)) * 2 ** directBits) >>> 0
|
||||
if (slot < END_POS_MODEL_INDEX) {
|
||||
dist = (dist + rc.reverseTree(posDecoders, dist - slot, directBits)) >>> 0
|
||||
} else {
|
||||
dist = (dist + rc.direct(directBits - ALIGN_BITS) * 2 ** ALIGN_BITS) >>> 0
|
||||
dist = (dist + rc.reverseTree(align, 0, ALIGN_BITS)) >>> 0
|
||||
}
|
||||
return dist
|
||||
}
|
||||
|
||||
let state = 0
|
||||
let rep0 = 0
|
||||
let rep1 = 0
|
||||
let rep2 = 0
|
||||
let rep3 = 0
|
||||
while (pos < size) {
|
||||
const posState = pos & ((1 << pb) - 1)
|
||||
if (rc.bit(isMatch, (state << POS_BITS_MAX) + posState) === 0) {
|
||||
const prev = pos > 0 ? out[pos - 1] : 0
|
||||
const base = 0x300 * (((pos & ((1 << lp) - 1)) << lc) + (prev >>> (8 - lc)))
|
||||
let symbol = 1
|
||||
if (state >= 7) {
|
||||
let match = out[pos - rep0 - 1]
|
||||
do {
|
||||
const matchBit = (match >>> 7) & 1
|
||||
match <<= 1
|
||||
const bit = rc.bit(literals, base + ((1 + matchBit) << 8) + symbol)
|
||||
symbol = (symbol << 1) | bit
|
||||
if (matchBit !== bit) break
|
||||
} while (symbol < 0x100)
|
||||
}
|
||||
while (symbol < 0x100) symbol = (symbol << 1) | rc.bit(literals, base + symbol)
|
||||
out[pos++] = symbol - 0x100
|
||||
state = state < 4 ? 0 : state < 10 ? state - 3 : state - 6
|
||||
continue
|
||||
}
|
||||
let len: number
|
||||
if (rc.bit(isRep, state) !== 0) {
|
||||
if (pos === 0) throw new LzmaError('LZMA: repeat before any output')
|
||||
if (rc.bit(isRepG0, state) === 0) {
|
||||
if (rc.bit(isRep0Long, (state << POS_BITS_MAX) + posState) === 0) {
|
||||
state = state < 7 ? 9 : 11
|
||||
out[pos] = out[pos - rep0 - 1]
|
||||
pos++
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
let dist: number
|
||||
if (rc.bit(isRepG1, state) === 0) {
|
||||
dist = rep1
|
||||
} else {
|
||||
if (rc.bit(isRepG2, state) === 0) {
|
||||
dist = rep2
|
||||
} else {
|
||||
dist = rep3
|
||||
rep3 = rep2
|
||||
}
|
||||
rep2 = rep1
|
||||
}
|
||||
rep1 = rep0
|
||||
rep0 = dist
|
||||
}
|
||||
len = repLenDecoder.decode(rc, posState)
|
||||
state = state < 7 ? 8 : 11
|
||||
} else {
|
||||
rep3 = rep2
|
||||
rep2 = rep1
|
||||
rep1 = rep0
|
||||
len = lenDecoder.decode(rc, posState)
|
||||
state = state < 7 ? 7 : 10
|
||||
rep0 = distance(len)
|
||||
// The end marker: allowed only where the output is complete, which the loop already ended at.
|
||||
if (rep0 === 0xffffffff) throw new LzmaError('LZMA: end marker before the end')
|
||||
if (rep0 >= dictSize || rep0 >= pos) throw new LzmaError('LZMA: distance out of range')
|
||||
}
|
||||
len += MATCH_MIN_LEN
|
||||
if (len > size - pos) throw new LzmaError('LZMA: match past the end')
|
||||
for (let i = 0; i < len; i++, pos++) out[pos] = out[pos - rep0 - 1]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
import { lzmaDecode } from '#lib/sevenzip/lzma'
|
||||
|
||||
/**
|
||||
* What a 7z archive's header says (its listing): read from the archive's end, where 7z keeps it, so the last
|
||||
* segment of the last volume is enough (the `7z-tail` probe; REBUILD.md, phase 2). The header there is mostly
|
||||
* an encoded one: a pointer to the real header, LZMA-packed just before it, or AES-encrypted (`7z -mhe`), which
|
||||
* hides the names and means a password.
|
||||
*/
|
||||
export interface SevenZipListing {
|
||||
/** Files, not directories, with their unpacked size (0 for an empty one). */
|
||||
files: { name: string; size: number }[]
|
||||
/** Entries or the header encrypted (AES). */
|
||||
encrypted: boolean
|
||||
/** The header itself encrypted: no names known. */
|
||||
headerEncrypted: boolean
|
||||
}
|
||||
|
||||
const AES = '06f10701'
|
||||
const LZMA = '030101'
|
||||
/** A header bigger than this isn't unpacked (headers list names; 64 MiB is ~a million files). */
|
||||
const MAX_HEADER = 64 * 1024 * 1024
|
||||
|
||||
const ID = {
|
||||
end: 0x00,
|
||||
header: 0x01,
|
||||
archiveProperties: 0x02,
|
||||
additionalStreamsInfo: 0x03,
|
||||
mainStreamsInfo: 0x04,
|
||||
filesInfo: 0x05,
|
||||
packInfo: 0x06,
|
||||
unpackInfo: 0x07,
|
||||
subStreamsInfo: 0x08,
|
||||
size: 0x09,
|
||||
crc: 0x0a,
|
||||
folder: 0x0b,
|
||||
codersUnpackSize: 0x0c,
|
||||
numUnpackStream: 0x0d,
|
||||
emptyStream: 0x0e,
|
||||
emptyFile: 0x0f,
|
||||
name: 0x11,
|
||||
encodedHeader: 0x17,
|
||||
dummy: 0x19,
|
||||
}
|
||||
|
||||
class Malformed extends Error {}
|
||||
|
||||
interface Coder {
|
||||
id: string
|
||||
properties: Uint8Array
|
||||
inStreams: number
|
||||
outStreams: number
|
||||
}
|
||||
interface Folder {
|
||||
coders: Coder[]
|
||||
bindOut: Set<number>
|
||||
unpackSizes: number[]
|
||||
crc?: number
|
||||
}
|
||||
interface StreamsInfo {
|
||||
packPos: number
|
||||
packSizes: number[]
|
||||
folders: Folder[]
|
||||
/** Per folder, how many files' streams it holds, and every stream's size in order. */
|
||||
streamsPerFolder: number[]
|
||||
streamSizes: number[]
|
||||
}
|
||||
|
||||
class Reader {
|
||||
constructor(
|
||||
private buf: Uint8Array,
|
||||
public pos = 0
|
||||
) {}
|
||||
|
||||
get done() {
|
||||
return this.pos >= this.buf.length
|
||||
}
|
||||
|
||||
byte(): number {
|
||||
if (this.pos >= this.buf.length) throw new Malformed('7z: header ended')
|
||||
return this.buf[this.pos++]
|
||||
}
|
||||
|
||||
bytes(n: number): Uint8Array {
|
||||
if (n < 0 || this.pos + n > this.buf.length) throw new Malformed('7z: header ended')
|
||||
const out = this.buf.subarray(this.pos, this.pos + n)
|
||||
this.pos += n
|
||||
return out
|
||||
}
|
||||
|
||||
/** 7z's variable-length number: the first byte's high bits say how many bytes follow. */
|
||||
number(): number {
|
||||
const first = this.byte()
|
||||
let mask = 0x80
|
||||
let value = 0
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * 2 ** (8 * i)
|
||||
break
|
||||
}
|
||||
value += this.byte() * 2 ** (8 * i)
|
||||
mask >>>= 1
|
||||
}
|
||||
if (value > Number.MAX_SAFE_INTEGER) throw new Malformed('7z: number too large')
|
||||
return value
|
||||
}
|
||||
|
||||
uint32(): number {
|
||||
const b = this.bytes(4)
|
||||
return (b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)) >>> 0
|
||||
}
|
||||
|
||||
bits(n: number): boolean[] {
|
||||
const out: boolean[] = []
|
||||
let byte = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (i % 8 === 0) byte = this.byte()
|
||||
out.push((byte & (0x80 >>> i % 8)) !== 0)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** "All defined" or a bit vector, then a CRC per defined item. */
|
||||
digests(n: number): (number | undefined)[] {
|
||||
const allDefined = this.byte() !== 0
|
||||
const defined = allDefined ? Array(n).fill(true) : this.bits(n)
|
||||
return defined.map((d) => (d ? this.uint32() : undefined))
|
||||
}
|
||||
|
||||
expect(id: number) {
|
||||
if (this.byte() !== id) throw new Malformed(`7z: expected property ${id}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Counts that come from the data: a garbage position makes huge ones, which shouldn't allocate. */
|
||||
function count(n: number, max = 1_000_000): number {
|
||||
if (n > max) throw new Malformed('7z: implausible count')
|
||||
return n
|
||||
}
|
||||
|
||||
function readFolder(r: Reader): Folder {
|
||||
const coders: Coder[] = []
|
||||
const numCoders = count(r.number(), 64)
|
||||
if (numCoders === 0) throw new Malformed('7z: folder without coders')
|
||||
for (let i = 0; i < numCoders; i++) {
|
||||
const flags = r.byte()
|
||||
if (flags & 0x80) throw new Malformed('7z: alternative coder methods')
|
||||
const id = Buffer.from(r.bytes(flags & 0x0f)).toString('hex')
|
||||
const complex = (flags & 0x10) !== 0
|
||||
const inStreams = complex ? count(r.number(), 64) : 1
|
||||
const outStreams = complex ? count(r.number(), 64) : 1
|
||||
const properties = flags & 0x20 ? r.bytes(count(r.number(), 1 << 16)) : new Uint8Array(0)
|
||||
coders.push({ id, properties, inStreams, outStreams })
|
||||
}
|
||||
const totalOut = coders.reduce((n, c) => n + c.outStreams, 0)
|
||||
const totalIn = coders.reduce((n, c) => n + c.inStreams, 0)
|
||||
const bindOut = new Set<number>()
|
||||
for (let i = 0; i < totalOut - 1; i++) {
|
||||
r.number()
|
||||
bindOut.add(r.number())
|
||||
}
|
||||
const packed = totalIn - (totalOut - 1)
|
||||
if (packed < 1) throw new Malformed('7z: folder without packed streams')
|
||||
if (packed > 1) for (let i = 0; i < packed; i++) r.number()
|
||||
return { coders, bindOut, unpackSizes: [] }
|
||||
}
|
||||
|
||||
/** The folder's output: the one out stream no bind pair consumes. */
|
||||
function unpackSize(folder: Folder): number {
|
||||
for (let i = folder.unpackSizes.length - 1; i >= 0; i--) if (!folder.bindOut.has(i)) return folder.unpackSizes[i]
|
||||
return 0
|
||||
}
|
||||
|
||||
function readStreamsInfo(r: Reader): StreamsInfo {
|
||||
const info: StreamsInfo = { packPos: 0, packSizes: [], folders: [], streamsPerFolder: [], streamSizes: [] }
|
||||
let id = r.byte()
|
||||
if (id === ID.packInfo) {
|
||||
info.packPos = r.number()
|
||||
const n = count(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.size) for (let i = 0; i < n; i++) info.packSizes.push(r.number())
|
||||
else if (id === ID.crc) r.digests(n)
|
||||
else throw new Malformed('7z: unexpected property in pack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.unpackInfo) {
|
||||
r.expect(ID.folder)
|
||||
const n = count(r.number())
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external folders')
|
||||
for (let i = 0; i < n; i++) info.folders.push(readFolder(r))
|
||||
r.expect(ID.codersUnpackSize)
|
||||
for (const f of info.folders) for (const c of f.coders) for (let i = 0; i < c.outStreams; i++) f.unpackSizes.push(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.crc) r.digests(n).forEach((crc, i) => (info.folders[i].crc = crc))
|
||||
else throw new Malformed('7z: unexpected property in unpack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
info.streamsPerFolder = info.folders.map(() => 1)
|
||||
if (id === ID.subStreamsInfo) {
|
||||
id = r.byte()
|
||||
if (id === ID.numUnpackStream) {
|
||||
info.streamsPerFolder = info.folders.map(() => count(r.number()))
|
||||
id = r.byte()
|
||||
}
|
||||
const hasSizes = id === ID.size
|
||||
info.folders.forEach((f, i) => {
|
||||
const n = info.streamsPerFolder[i]
|
||||
if (n === 0) return
|
||||
let sum = 0
|
||||
for (let j = 0; j < n - 1 && hasSizes; j++) {
|
||||
const size = r.number()
|
||||
info.streamSizes.push(size)
|
||||
sum += size
|
||||
}
|
||||
info.streamSizes.push(unpackSize(f) - sum)
|
||||
})
|
||||
if (hasSizes) id = r.byte()
|
||||
while (id !== ID.end) {
|
||||
if (id === ID.crc) {
|
||||
// CRCs for the streams whose folder had none (a folder of one stream with a CRC has its own).
|
||||
const unknown = info.folders.reduce((n, f, i) => n + (info.streamsPerFolder[i] === 1 && f.crc !== undefined ? 0 : info.streamsPerFolder[i]), 0)
|
||||
r.digests(unknown)
|
||||
} else {
|
||||
throw new Malformed('7z: unexpected property in substreams info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
id = r.byte()
|
||||
} else {
|
||||
info.streamSizes = info.folders.map(unpackSize)
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: streams info not ended')
|
||||
return info
|
||||
}
|
||||
|
||||
function readFilesInfo(r: Reader, streams: StreamsInfo | null): { name: string; size: number }[] {
|
||||
const n = count(r.number())
|
||||
let emptyStream: boolean[] = Array(n).fill(false)
|
||||
let emptyFile: boolean[] = []
|
||||
let names: string[] = []
|
||||
for (let id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
const size = r.number()
|
||||
const end = r.pos + size
|
||||
if (id === ID.emptyStream) {
|
||||
emptyStream = r.bits(n)
|
||||
} else if (id === ID.emptyFile) {
|
||||
emptyFile = r.bits(emptyStream.filter(Boolean).length)
|
||||
} else if (id === ID.name) {
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external names')
|
||||
const text = Buffer.from(r.bytes(size - 1)).toString('utf16le')
|
||||
names = text.split('\0').slice(0, n)
|
||||
} else if (id !== ID.dummy && id > 0x19) {
|
||||
throw new Malformed('7z: unknown file property')
|
||||
}
|
||||
r.pos = end
|
||||
}
|
||||
if (names.length !== n) throw new Malformed('7z: names missing')
|
||||
const out: { name: string; size: number }[] = []
|
||||
let stream = 0
|
||||
let empty = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (!emptyStream[i]) {
|
||||
out.push({ name: names[i], size: streams?.streamSizes[stream++] ?? 0 })
|
||||
continue
|
||||
}
|
||||
// An empty stream is a directory unless it's marked an empty file.
|
||||
if (emptyFile[empty++]) out.push({ name: names[i], size: 0 })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function readHeader(r: Reader): SevenZipListing {
|
||||
let id = r.byte()
|
||||
if (id === ID.archiveProperties) {
|
||||
for (let type = r.byte(); type !== 0; type = r.byte()) r.bytes(r.number())
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.additionalStreamsInfo) {
|
||||
readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let streams: StreamsInfo | null = null
|
||||
if (id === ID.mainStreamsInfo) {
|
||||
streams = readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let files: { name: string; size: number }[] = []
|
||||
if (id === ID.filesInfo) {
|
||||
files = readFilesInfo(r, streams)
|
||||
id = r.byte()
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: header not ended')
|
||||
const encrypted = (streams?.folders ?? []).some((f) => f.coders.some((c) => c.id === AES))
|
||||
return { files, encrypted, headerEncrypted: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* The header that ends exactly at `end` in `data`, starting at `start`: a plain one, or an encoded one whose packed
|
||||
* data lies just before it (where 7z writes it). Throws when it isn't one.
|
||||
*/
|
||||
function headerAt(data: Uint8Array, start: number, end: number): SevenZipListing {
|
||||
const r = new Reader(data.subarray(0, end), start)
|
||||
const id = r.byte()
|
||||
if (id === ID.header) {
|
||||
const listing = readHeader(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the header')
|
||||
return listing
|
||||
}
|
||||
if (id !== ID.encodedHeader) throw new Malformed('7z: not a header')
|
||||
const info = readStreamsInfo(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the encoded header')
|
||||
if (info.folders.length !== 1 || info.packSizes.length < 1) throw new Malformed('7z: unexpected encoded header')
|
||||
const folder = info.folders[0]
|
||||
if (folder.coders.some((c) => c.id === AES)) return { files: [], encrypted: true, headerEncrypted: true }
|
||||
if (folder.coders.length !== 1 || folder.coders[0].id !== LZMA) throw new Malformed(`7z: header coder ${folder.coders[0].id} not supported`)
|
||||
const packSize = info.packSizes[0]
|
||||
const size = unpackSize(folder)
|
||||
if (packSize > start || size > MAX_HEADER) throw new Malformed('7z: packed header not in the data')
|
||||
const header = lzmaDecode(folder.coders[0].properties, data.subarray(start - packSize, start), size)
|
||||
if (folder.crc !== undefined && crc32(header) >>> 0 !== folder.crc) throw new Malformed('7z: header CRC mismatch')
|
||||
return headerAt(header, 0, header.length)
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing of the 7z archive that `data` is the end of (the last segment of its last volume, or a whole small
|
||||
* archive); null when no header ends there.
|
||||
*/
|
||||
export function parseSevenZipTail(data: Uint8Array): SevenZipListing | null {
|
||||
const end = data.length
|
||||
// An encoded header is a few dozen bytes; a plain one (7z -mhc=off) can be large.
|
||||
for (let start = end - 2; start >= Math.max(0, end - 1024); start--) {
|
||||
if (data[start] !== ID.encodedHeader || data[start + 1] !== ID.packInfo) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
for (let start = end - 2; start >= 0; start--) {
|
||||
if (data[start] !== ID.header || ![ID.archiveProperties, ID.additionalStreamsInfo, ID.mainStreamsInfo, ID.filesInfo, ID.end].includes(data[start + 1])) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { parseZipStart, type ZipStart } from '#lib/zip/parse'
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* The entries a ZIP's start shows: its local file headers, one before each entry's data, as far as the data
|
||||
* holds them (the first segment of a .zip; REBUILD.md, phase 2). The full list is in the central directory at
|
||||
* the end; the start names the first entries and says whether they're encrypted, as a RAR's first volume does.
|
||||
*/
|
||||
export interface ZipStart {
|
||||
files: { name: string; size: number }[]
|
||||
encrypted: boolean
|
||||
}
|
||||
|
||||
const LOCAL = 0x04034b50
|
||||
/** Traditional PKWARE encryption (bit 0 of the flags), or WinZip AES (method 99). */
|
||||
const ENCRYPTED_FLAG = 0x1
|
||||
const DATA_DESCRIPTOR_FLAG = 0x8
|
||||
const AES_METHOD = 99
|
||||
const MAX_ENTRIES = 10_000
|
||||
|
||||
export function parseZipStart(data: Buffer): ZipStart | null {
|
||||
if (data.length < 30 || data.readUInt32LE(0) !== LOCAL) return null
|
||||
const result: ZipStart = { files: [], encrypted: false }
|
||||
let pos = 0
|
||||
while (pos + 30 <= data.length && data.readUInt32LE(pos) === LOCAL && result.files.length < MAX_ENTRIES) {
|
||||
const flags = data.readUInt16LE(pos + 6)
|
||||
const method = data.readUInt16LE(pos + 8)
|
||||
let compressed = data.readUInt32LE(pos + 18)
|
||||
let size = data.readUInt32LE(pos + 22)
|
||||
const nameLength = data.readUInt16LE(pos + 26)
|
||||
const extraLength = data.readUInt16LE(pos + 28)
|
||||
const nameEnd = pos + 30 + nameLength
|
||||
if (nameEnd > data.length) break
|
||||
// Bit 11: UTF-8 names; others are mostly CP437, read as latin1.
|
||||
const name = data.toString(flags & 0x800 ? 'utf8' : 'latin1', pos + 30, nameEnd)
|
||||
// Zip64: the real sizes in extra field 0x0001.
|
||||
for (let e = nameEnd; e + 4 <= Math.min(nameEnd + extraLength, data.length); ) {
|
||||
const id = data.readUInt16LE(e)
|
||||
const length = data.readUInt16LE(e + 2)
|
||||
if (id === 0x0001 && e + 4 + length <= data.length && length >= 16) {
|
||||
size = Number(data.readBigUInt64LE(e + 4))
|
||||
compressed = Number(data.readBigUInt64LE(e + 12))
|
||||
}
|
||||
e += 4 + length
|
||||
}
|
||||
if (flags & ENCRYPTED_FLAG || method === AES_METHOD) result.encrypted = true
|
||||
if (!name.endsWith('/')) result.files.push({ name, size })
|
||||
// Sizes after the data (a streamed zip): where the next header starts isn't known.
|
||||
if (flags & DATA_DESCRIPTOR_FLAG) break
|
||||
pos = nameEnd + extraLength + compressed
|
||||
}
|
||||
return result.files.length > 0 || result.encrypted ? result : null
|
||||
}
|
||||
+21
-6
@@ -17,12 +17,21 @@ const SOURCE_LABELS = {
|
||||
subject: 'Subject',
|
||||
post: 'Post id',
|
||||
}
|
||||
/** Where a known password came from (release_passwords.source). */
|
||||
const PASSWORD_SOURCES = {
|
||||
nzb: "from the spot's NZB",
|
||||
'spot-title': "from the spot's title",
|
||||
'spot-text': "from the spot's text",
|
||||
}
|
||||
/** Newznab categories (lib/newznab/categories.ts). */
|
||||
const CATEGORIES = [
|
||||
[1000, 'Console', []],
|
||||
[2000, 'Movies', [[2030, 'SD'], [2040, 'HD'], [2045, 'UHD']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3040, 'Lossless']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD']]],
|
||||
[7000, 'Books', [[7020, 'Ebook']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3030, 'Audiobook'], [3040, 'Lossless']]],
|
||||
[4000, 'PC', [[4050, 'Games']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD'], [5070, 'Anime']]],
|
||||
[6000, 'XXX', [[6040, 'x264'], [6045, 'UHD'], [6060, 'ImgSet'], [6070, 'Other']]],
|
||||
[7000, 'Books', [[7020, 'Ebook'], [7030, 'Comics']]],
|
||||
[8000, 'Other', [[8010, 'Misc']]],
|
||||
]
|
||||
function categoryLabel(id) {
|
||||
@@ -220,9 +229,10 @@ function releaseRow(release) {
|
||||
if (release.complete) badges.append(badge(files, 'good', 'Complete'))
|
||||
else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
|
||||
else badges.append(badge(files))
|
||||
if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
|
||||
if (release.passworded && release.passwordKnown) badges.append(badge('Password known', 'warn', 'Its archive is encrypted; the password is in the NZB'))
|
||||
else if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
|
||||
if (!release.settled) badges.append(badge('Collecting', 'accent', 'Recently seen: more files may still arrive'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name)'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name, the inner name, a spot or the groups)'))
|
||||
if (release.source && release.source !== 'stem' && release.source !== 'subject') {
|
||||
badges.append(badge(SOURCE_LABELS[release.source] ?? release.source, 'accent', 'How this release was recognised'))
|
||||
}
|
||||
@@ -347,7 +357,12 @@ function renderDetail(release) {
|
||||
if (!release.settledAt) {
|
||||
notices.push(el('div', { class: 'notice notice-warn', text: 'Recently seen: more files of this release may still arrive. The NZB has what is indexed so far.' }))
|
||||
}
|
||||
if (release.passworded) {
|
||||
const passwords = release.passwords ?? []
|
||||
if (passwords.length) {
|
||||
notices.push(el('div', { class: 'notice notice-warn' },
|
||||
el('span', { text: `${passwords.length === 1 ? 'Password' : 'Passwords'} (the first is in the NZB): ` }),
|
||||
...passwords.map((p, i) => el('span', {}, i ? ', ' : '', el('code', { text: p.password }), ` (${PASSWORD_SOURCES[p.source] ?? p.source})`))))
|
||||
} else if (release.passworded) {
|
||||
notices.push(el('div', { class: 'notice notice-bad', text: 'Passworded: its archive has encrypted entries or headers. The password is not in the index.' }))
|
||||
}
|
||||
if (!release.readable && !release.innerName) {
|
||||
|
||||
@@ -43,10 +43,15 @@ export default await Env.create(new URL('../', import.meta.url), {
|
||||
NNTP_PASSWORD: Env.schema.string.optional(),
|
||||
NNTP_SECURE: Env.schema.boolean.optional(),
|
||||
NNTP_CONNECTIONS: Env.schema.number.optional(),
|
||||
NNTP_MIN_CONNECTIONS: Env.schema.number.optional(),
|
||||
NNTP_ACCOUNT_CONNECTIONS: Env.schema.number.optional(),
|
||||
NNTP_RESERVED_CONNECTIONS: Env.schema.number.optional(),
|
||||
NNTP_TIMEOUT_MS: Env.schema.number.optional(),
|
||||
NNTP_HEDGE_AFTER_SECONDS: Env.schema.number.optional(),
|
||||
NNTP_HEDGE_CONNECTIONS: Env.schema.number.optional(),
|
||||
NNTP_CONNECTION_MAX_MINUTES: Env.schema.number.optional(),
|
||||
NNTP_IDLE_CLOSE_SECONDS: Env.schema.number.optional(),
|
||||
CORPUS_FILTER: Env.schema.boolean.optional(),
|
||||
NNTP_STALL_KB_PER_MINUTE: Env.schema.number.optional(),
|
||||
|
||||
// Newznab API key (empty: no key needed)
|
||||
@@ -69,11 +74,16 @@ export default await Env.create(new URL('../', import.meta.url), {
|
||||
SPOT_STANDALONE_AFTER_HOURS: Env.schema.number.optional(),
|
||||
CORPUS_AHEAD: Env.schema.number.optional(),
|
||||
PROBE_PIPELINE: Env.schema.number.optional(),
|
||||
PENDING_SHARE: Env.schema.number.optional(),
|
||||
PENDING_MIN_PROBES: Env.schema.number.optional(),
|
||||
PENDING_MAX_NAMED: Env.schema.number.optional(),
|
||||
SPOOL_AHEAD_PER_LEASE: Env.schema.number.optional(),
|
||||
SPOOL_MIN_AHEAD: Env.schema.number.optional(),
|
||||
SPOOL_MAX_GB: Env.schema.number.optional(),
|
||||
PREFETCH_PER_RANGE: Env.schema.number.optional(),
|
||||
PROBE_CONNECTIONS: Env.schema.number.optional(),
|
||||
PROBE_MIN_CONNECTIONS: Env.schema.number.optional(),
|
||||
PROBE_MIN_FREE_GB: Env.schema.number.optional(),
|
||||
HTTP_ROUTES: Env.schema.enum.optional(['all', 'node', 'public'] as const),
|
||||
PREFETCH_PER_GROUP: Env.schema.number.optional(),
|
||||
})
|
||||
@@ -8,6 +8,7 @@ import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { authApiClient } from '@adonisjs/auth/plugins/api_client'
|
||||
import { sessionApiClient } from '@adonisjs/session/plugins/api_client'
|
||||
import type { Registry } from '../.adonisjs/client/registry/schema.d.ts'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
|
||||
/**
|
||||
* This file is imported by the "bin/test.ts" entrypoint file
|
||||
@@ -51,6 +52,8 @@ export const runnerHooks: Required<Pick<Config, 'setup' | 'teardown'>> = {
|
||||
*/
|
||||
export const configureSuite: Config['configureSuite'] = (suite) => {
|
||||
if (['browser', 'functional', 'e2e'].includes(suite.name)) {
|
||||
// Each test's transaction is rolled back: the poster and group-set ids cached meanwhile are gone with it.
|
||||
suite.onGroup((group) => group.each.setup(() => CatalogRefs.forget()))
|
||||
return suite.setup(() => testUtils.httpServer().start())
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Loaded 100 of 140 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user