The window worker: group leases, the stream serving the corpus, the snapshot through the API (design 1 and 5)

A worker thread leases a group and takes its ranges in article order through
one carried window; the coordinator keeps one cursor a group (group_leases,
migration 6500) and hands out the next range, the rules and nothing about
where the chunks are: the range stream serves a stored range's chunks from the
corpus file and a live one's from the spool. The window's snapshot (bincode:
the frontier, the open files, releases and albums, the upload numbers, the
batch-counter table) goes to the coordinator every EDGE_SNAPSHOT_MINUTES and
when the group is given back, and the next holder restores it; a takeover
after LEASE_TTL_SECONDS plans the ranges from the snapshot's article again.
The prefetcher runs ahead of held groups' cursors and skips chunks the corpus
has. LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY bound what is leased. edge-node
holds groups by default; --ranges keeps the old per-range path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

edge/src/bin/edge-node.rs

@@ -1,16 +1,20 @@
-//! The edge worker in Rust (REBUILD.md, part 2; commands/edge_node.ts): leases
-//! ranges from the coordinator, reads their XOVER chunks from its stream,
-//! indexes them (the same RangeIndexer as edge-bench, kept identical to
-//! the TypeScript by check.ts) and posts the batches. One range per thread.
+//! The window worker in Rust (REBUILD.md, "Voyager 3: window indexing", 1 and 5): each thread leases a *group*
+//! from the coordinator, takes its ranges in article order, reads each range's XOVER chunks from the coordinator's
+//! stream (the corpus or the spool, the worker can't tell), indexes them through one carried window and posts the
+//! batches, a batch at least at every range end; the window's snapshot goes to the coordinator every
+//! --snapshot-minutes and when the group is given back, and the next holder restores it. A group with no range
+//! left (caught up: the tail) is given back and leased again later.
 //!
-//!   COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
+//!   COORDINATOR_URL=… NODE_TOKEN=… edge-node [--threads N] [--batch-articles 500000] [--snapshot-minutes 30] [--groups N] [--max-mbit M]
 //!
-//! SIGTERM/SIGINT: each thread gives its range back after the current chunk
-//! (the next worker resumes at the last stored batch).
+//! SIGTERM/SIGINT: each thread finishes the chunk, posts its batch, snapshots and gives its group back.
+//! `--ranges`: the old range leases (POST lease), one range per thread without a carried window (kept for a
+//! coordinator without group leases).
 
 use edge::indexer::{Batch, RangeIndexer};
-use edge::lease::LeaseJson;
+use edge::lease::{GroupLeaseJson, LeaseJson, NextRangeJson, RangeSpanJson};
 use edge::wire;
+use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -163,6 +167,81 @@ impl Api {
         }
     }
 
+    // Group leases (REBUILD.md design 1 and 5).
+
+    fn lease_group(&self) -> Result<Option<GroupLeaseJson>, ApiError> {
+        let mut response = self.request("POST", "groups/lease", None, false, "application/json")?;
+        match response.status().as_u16() {
+            204 => Ok(None),
+            200 => {
+                let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("groups/lease: {e}")))?;
+                serde_json::from_str(&text).map(Some).map_err(|e| ApiError::Other(format!("groups/lease: {e}")))
+            }
+            status => Err(ApiError::Other(format!("groups/lease: HTTP {status}"))),
+        }
+    }
+
+    fn next_range(&self, group: i64) -> Result<Option<RangeSpanJson>, ApiError> {
+        let mut response = self.request("POST", &format!("groups/{group}/next"), None, false, "application/json")?;
+        match response.status().as_u16() {
+            204 => Ok(None),
+            200 => {
+                let text = response.body_mut().read_to_string().map_err(|e| ApiError::Other(format!("next: {e}")))?;
+                serde_json::from_str::<NextRangeJson>(&text).map(|n| Some(n.range)).map_err(|e| ApiError::Other(format!("next: {e}")))
+            }
+            status => Err(ApiError::Other(format!("next: HTTP {status}"))),
+        }
+    }
+
+    fn group_call(&self, group: i64, action: &str) -> Result<(), ApiError> {
+        let response = self.request("POST", &format!("groups/{group}/{action}"), None, false, "application/json")?;
+        match response.status().as_u16() {
+            200..=299 => Ok(()),
+            status => Err(ApiError::Other(format!("{action}: HTTP {status}"))),
+        }
+    }
+
+    /// The window's snapshot (bincode), gzipped, standing at `next`.
+    fn put_snapshot(&self, group: i64, next: i64, snapshot: &[u8]) -> Result<usize, ApiError> {
+        let mut gz = GzEncoder::new(Vec::with_capacity(snapshot.len() / 3), flate2::Compression::new(1));
+        gz.write_all(snapshot).unwrap();
+        let body = gz.finish().unwrap();
+        let url = format!("{}/api/v1/node/groups/{group}/snapshot?next={next}", self.base);
+        let auth = format!("Bearer {}", self.token);
+        for attempt in 1..=RETRIES {
+            let result = self.agent.put(&url).header("Authorization", &auth).header("Content-Type", "application/octet-stream").send(&body[..]);
+            match result {
+                Ok(response) => match response.status().as_u16() {
+                    200..=299 => return Ok(body.len()),
+                    409 => return Err(ApiError::LeaseLost("snapshot: the group is no longer this node's".into())),
+                    status if status < 500 => return Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
+                    status if attempt == RETRIES => return Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
+                    _ => {}
+                },
+                Err(error) if attempt == RETRIES => return Err(ApiError::Other(format!("snapshot: {error}"))),
+                Err(_) => {}
+            }
+            std::thread::sleep(Duration::from_millis(60_000.min(1000 * 2u64.pow(attempt))));
+        }
+        unreachable!()
+    }
+
+    /// The group's snapshot: its bytes (inflated) and the article it stands at; None without one.
+    fn get_snapshot(&self, group: i64) -> Result<Option<(i64, Vec<u8>)>, ApiError> {
+        let response = self.request("GET", &format!("groups/{group}/snapshot"), None, false, "application/octet-stream")?;
+        match response.status().as_u16() {
+            404 => Ok(None),
+            200 => {
+                let next: i64 = response.headers().get("x-snapshot-next").and_then(|v| v.to_str().ok()).and_then(|v| v.parse().ok()).ok_or_else(|| ApiError::Other("snapshot: no X-Snapshot-Next".into()))?;
+                let mut gz = GzDecoder::new(response.into_body().into_reader());
+                let mut bytes = Vec::new();
+                gz.read_to_end(&mut bytes).map_err(|e| ApiError::Other(format!("snapshot: {e}")))?;
+                Ok(Some((next, bytes)))
+            }
+            status => Err(ApiError::Other(format!("snapshot: HTTP {status}"))),
+        }
+    }
+
     fn corrupt(&self, range: i64, from: i64) -> Result<(), ApiError> {
         let body = format!("{{\"from\":{from}}}");
         let response = self.request("POST", &format!("ranges/{range}/corrupt"), Some(body.as_bytes()), false, "application/json")?;
@@ -275,6 +354,8 @@ fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Result<Vec<u8>, Str
 struct Config {
     batch_articles: i64,
     bytes_per_second: Option<f64>,
+    /// Between two snapshots of a held group's window.
+    snapshot_every: Duration,
     /// Ranges whose batch was too large (413), and how much smaller their batches are now (halved each time).
     smaller: Mutex<HashMap<i64, i64>>,
 }
@@ -452,6 +533,255 @@ fn run_range(api: &Arc<Api>, lease: LeaseJson, config: &Config, stopping: &Atomi
     }
 }
 
+/// What one range of a held group came to.
+enum RangeOutcome {
+    Done,
+    /// The group is given back (a stop, a lost lease, a failure): no more ranges.
+    Stop,
+}
+
+/// One range of a held group through the carried window: the chunks streamed, batches posted at the range's end at
+/// least, the range finished. The indexer keeps the window for the next range.
+#[allow(clippy::too_many_arguments)]
+fn run_group_range(api: &Arc<Api>, group_id: i64, group_name: &str, range: RangeSpanJson, indexer: &mut RangeIndexer, config: &Config, stopping: &AtomicBool, totals: &Arc<Mutex<Totals>>) -> RangeOutcome {
+    let started = Instant::now();
+    indexer.begin_range(range.next, range.end);
+    let post_error: Arc<Mutex<Option<ApiError>>> = Arc::new(Mutex::new(None));
+    let last_contact = Arc::new(AtomicI64::new(0));
+    let since = Instant::now();
+    let (post_tx, post_rx) = sync_channel::<Batch>(1);
+    let poster = {
+        let (api, totals, post_error, last_contact) = (api.clone(), totals.clone(), post_error.clone(), last_contact.clone());
+        std::thread::spawn(move || {
+            for batch in post_rx {
+                if post_error.lock().unwrap().is_some() {
+                    continue;
+                }
+                let at = Instant::now();
+                match api.batch(range.id, &batch) {
+                    Ok((releases, candidates, bytes)) => {
+                        let mut t = totals.lock().unwrap();
+                        t.post_ms += at.elapsed().as_millis();
+                        t.bytes += bytes;
+                        t.headers += batch.headers;
+                        t.releases += releases;
+                        t.candidates += candidates;
+                        last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
+                    }
+                    Err(e) => *post_error.lock().unwrap() = Some(e),
+                }
+            }
+        })
+    };
+    let frames = match api.stream(range.id) {
+        Ok(stream) => read_frames(stream, config.bytes_per_second),
+        Err(e) => {
+            log("error", &format!("{group_name} {}–{}: stream failed: {e}; the group goes back.", range.start, range.end));
+            drop(post_tx);
+            let _ = poster.join();
+            return RangeOutcome::Stop;
+        }
+    };
+    let (batch_articles, max_pending) = config.limits(range.id);
+    let mut batch_from = range.next;
+    let mut reached_end = false;
+    let mut corrupt: Option<(i64, i64, String)> = None;
+    let mut failure: Option<String> = None;
+    let headers_before = totals.lock().unwrap().headers;
+    for frame in frames.iter() {
+        if stopping.load(Ordering::Relaxed) || post_error.lock().unwrap().is_some() {
+            break;
+        }
+        if since.elapsed().as_millis() as i64 - last_contact.load(Ordering::Relaxed) > HEARTBEAT.as_millis() as i64 {
+            last_contact.store(since.elapsed().as_millis() as i64, Ordering::Relaxed);
+            let _ = api.group_call(group_id, "heartbeat");
+        }
+        match frame {
+            Frame::KeepAlive => continue,
+            Frame::End => break,
+            Frame::Failed(e) => {
+                failure = Some(e);
+                break;
+            }
+            Frame::Chunk { from, to, text } => {
+                let data = match text {
+                    Ok(data) => data,
+                    Err(e) => {
+                        corrupt = Some((from, to, e));
+                        break;
+                    }
+                };
+                indexer.add_text(&String::from_utf8_lossy(&data), to + 1);
+                reached_end = to >= range.end;
+                // The window carries on past the range's end: no finish here; a batch at the end at least.
+                if reached_end || to + 1 - batch_from >= batch_articles || indexer.pending() >= max_pending {
+                    if post_tx.send(indexer.take()).is_err() {
+                        break;
+                    }
+                    batch_from = to + 1;
+                }
+                if reached_end {
+                    break;
+                }
+            }
+        }
+    }
+    // Stopped mid-range: the batch so far goes out, so the snapshot stands at a stored resume point.
+    if !reached_end && corrupt.is_none() && failure.is_none() && post_error.lock().unwrap().is_none() && indexer.next() > batch_from {
+        let _ = post_tx.send(indexer.take());
+    }
+    drop(post_tx);
+    let _ = poster.join();
+    let post_error = post_error.lock().unwrap().take();
+    if let Some((from, to, e)) = corrupt {
+        log("warn", &format!("{group_name} {}–{}: chunk {from}–{to} doesn't inflate ({e}); reported, the group goes back", range.start, range.end));
+        let _ = api.corrupt(range.id, from);
+        return RangeOutcome::Stop;
+    }
+    match post_error {
+        Some(ApiError::LeaseLost(m)) => {
+            log("warn", &format!("{group_name} {}–{}: {m}", range.start, range.end));
+            return RangeOutcome::Stop;
+        }
+        Some(ApiError::TooLarge(m)) => {
+            let divisor = {
+                let mut smaller = config.smaller.lock().unwrap();
+                let d = smaller.entry(range.id).or_insert(1);
+                *d = (*d * 2).min(64);
+                *d
+            };
+            log("warn", &format!("{group_name} {}–{}: {m}; the group goes back, its batches 1/{divisor} the size from now on.", range.start, range.end));
+            return RangeOutcome::Stop;
+        }
+        Some(ApiError::Other(m)) => {
+            log("error", &format!("{group_name} {}–{}: {m}; the group goes back.", range.start, range.end));
+            return RangeOutcome::Stop;
+        }
+        None => {}
+    }
+    if let Some(e) = failure {
+        log("error", &format!("{group_name} {}–{}: {e}; the group goes back.", range.start, range.end));
+        return RangeOutcome::Stop;
+    }
+    if !reached_end {
+        return RangeOutcome::Stop;
+    }
+    match api.call(range.id, "finish") {
+        Ok(()) => {
+            let t = totals.lock().unwrap();
+            let seconds = started.elapsed().as_secs_f64();
+            let headers = t.headers - headers_before;
+            log(
+                "info",
+                &format!(
+                    "{group_name} {}–{}: {} headers in {:.0} s ({:.0}/s); {} releases and {} files open in the window.",
+                    range.start,
+                    range.end,
+                    headers,
+                    seconds,
+                    headers as f64 / seconds.max(0.001),
+                    indexer.open_releases(),
+                    indexer.open_files()
+                ),
+            );
+            RangeOutcome::Done
+        }
+        Err(e) => {
+            log("warn", &format!("{group_name} {}–{}: finish: {e}", range.start, range.end));
+            RangeOutcome::Stop
+        }
+    }
+}
+
+/// The window's snapshot to the coordinator, standing at the indexer's next article.
+fn snapshot(api: &Api, group_id: i64, group_name: &str, indexer: &RangeIndexer) {
+    match indexer.snapshot() {
+        Ok(bytes) => match api.put_snapshot(group_id, indexer.next(), &bytes) {
+            Ok(gz) => log("info", &format!("{group_name}: snapshot at {} ({:.1} MB, {:.1} MB gzipped).", indexer.next(), bytes.len() as f64 / 1e6, gz as f64 / 1e6)),
+            Err(e) => log("warn", &format!("{group_name}: snapshot failed: {e}")),
+        },
+        Err(e) => log("warn", &format!("{group_name}: no snapshot: {e}")),
+    }
+}
+
+/// One held group: its window restored from the snapshot (or new), its ranges one after another until it is
+/// caught up or the worker stops; then a snapshot, and the group given back.
+fn run_group(api: &Arc<Api>, lease: GroupLeaseJson, config: &Config, stopping: &AtomicBool) {
+    let group_id = lease.group.id;
+    let group_name = lease.group.name.clone();
+    let rules = lease.rules.rules();
+    let mut indexer = match lease.snapshot {
+        Some(s) => match api.get_snapshot(group_id) {
+            Ok(Some((next, bytes))) => match RangeIndexer::restore(&bytes, rules.clone(), 5) {
+                Ok(ix) => {
+                    log("info", &format!("{group_name}: window restored from the snapshot at {next} ({:.1} MB).", bytes.len() as f64 / 1e6));
+                    ix
+                }
+                Err(e) => {
+                    log("warn", &format!("{group_name}: the snapshot at {} can't be restored ({e}); the window starts empty.", s.next));
+                    RangeIndexer::for_group(&group_name, rules.clone(), 5)
+                }
+            },
+            Ok(None) => {
+                log("warn", &format!("{group_name}: the lease names a snapshot at {} but there is none; the window starts empty.", s.next));
+                RangeIndexer::for_group(&group_name, rules.clone(), 5)
+            }
+            Err(e) => {
+                log("warn", &format!("{group_name}: snapshot: {e}; the group goes back."));
+                let _ = api.group_call(group_id, "release");
+                return;
+            }
+        },
+        None => RangeIndexer::for_group(&group_name, rules.clone(), 5),
+    };
+    let totals = Arc::new(Mutex::new(Totals::default()));
+    let started = Instant::now();
+    let mut snapshotted = Instant::now();
+    let mut ranges = 0;
+    loop {
+        if stopping.load(Ordering::Relaxed) {
+            break;
+        }
+        let range = match api.next_range(group_id) {
+            Ok(Some(range)) => range,
+            Ok(None) => {
+                log("info", &format!("{group_name}: caught up after {ranges} ranges; the group goes back."));
+                break;
+            }
+            Err(e) => {
+                log("warn", &format!("{group_name}: next range: {e}; the group goes back."));
+                break;
+            }
+        };
+        if range.next != indexer.next() && ranges > 0 {
+            log("warn", &format!("{group_name}: the window stands at {} but the next range resumes at {}: the window carries on from there.", indexer.next(), range.next));
+        }
+        match run_group_range(api, group_id, &group_name, range, &mut indexer, config, stopping, &totals) {
+            RangeOutcome::Done => ranges += 1,
+            RangeOutcome::Stop => break,
+        }
+        if snapshotted.elapsed() >= config.snapshot_every {
+            snapshot(api, group_id, &group_name, &indexer);
+            snapshotted = Instant::now();
+        }
+    }
+    snapshot(api, group_id, &group_name, &indexer);
+    let _ = api.group_call(group_id, "release");
+    let t = totals.lock().unwrap();
+    log(
+        "info",
+        &format!(
+            "{group_name}: {ranges} ranges, {} headers in {:.0} s, {} release chunks, {} probe candidates; posted {:.1} MB in {:.1} s.",
+            t.headers,
+            started.elapsed().as_secs_f64(),
+            t.releases,
+            t.candidates,
+            t.bytes as f64 / 1e6,
+            t.post_ms as f64 / 1000.0
+        ),
+    );
+}
+
 fn main() {
     let args: Vec<String> = std::env::args().collect();
     let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
@@ -466,6 +796,7 @@ fn main() {
     let max_mbit: Option<f64> = flag("--max-mbit").or_else(|| std::env::var("EDGE_MAX_MBIT").ok()).map(|v| v.parse().unwrap()).filter(|&m: &f64| m > 0.0);
     let config = Arc::new(Config {
         batch_articles: flag("--batch-articles").map(|v| v.parse().unwrap()).unwrap_or(500_000),
+        snapshot_every: Duration::from_secs(60 * flag("--snapshot-minutes").or_else(|| std::env::var("EDGE_SNAPSHOT_MINUTES").ok()).map(|v| v.parse::<u64>().unwrap()).unwrap_or(30)),
         // The node's cap, shared by its threads.
         bytes_per_second: max_mbit.map(|m| m * 1e6 / 8.0 / threads as f64),
         smaller: Mutex::new(HashMap::new()),
@@ -475,21 +806,39 @@ fn main() {
     signal_hook::flag::register(signal_hook::consts::SIGINT, stopping.clone()).unwrap();
     let api = Arc::new(Api::new(&url, &token));
     log("info", &format!("Starting {threads} worker threads{}.", max_mbit.map(|m| format!(" (download cap {m} Mbit/s)")).unwrap_or_default()));
+    let groups: Option<usize> = flag("--groups").map(|v| v.parse().unwrap());
+    let range_leases = ranges.is_some();
     let workers: Vec<_> = (0..threads)
         .map(|_| {
             let (api, config, stopping) = (api.clone(), config.clone(), stopping.clone());
             std::thread::spawn(move || {
                 let mut done = 0;
-                while !stopping.load(Ordering::Relaxed) && ranges.is_none_or(|n| done < n) {
-                    match api.lease() {
+                if range_leases {
+                    while !stopping.load(Ordering::Relaxed) && ranges.is_none_or(|n| done < n) {
+                        match api.lease() {
+                            Ok(Some(lease)) => {
+                                run_range(&api, lease, &config, &stopping);
+                                done += 1;
+                            }
+                            Ok(None) => break,
+                            Err(e) => {
+                                log("warn", &format!("Lease failed: {e}"));
+                                idle(&stopping);
+                            }
+                        }
+                    }
+                    return;
+                }
+                while !stopping.load(Ordering::Relaxed) && groups.is_none_or(|n| done < n) {
+                    match api.lease_group() {
                         Ok(Some(lease)) => {
-                            run_range(&api, lease, &config, &stopping);
+                            run_group(&api, lease, &config, &stopping);
                             done += 1;
                         }
-                        Ok(None) if ranges.is_some() => break,
+                        Ok(None) if groups.is_some() => break,
                         Ok(None) => idle(&stopping),
                         Err(e) => {
-                            log("warn", &format!("Lease failed: {e}"));
+                            log("warn", &format!("Group lease failed: {e}"));
                             idle(&stopping);
                         }
                     }

edge/src/indexer.rs

@@ -7,6 +7,7 @@ use crate::codec::{encode_chunk, ChunkFile};
 use crate::js::{self, sha1_hex};
 use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
 use rustc_hash::{FxHashMap, FxHashSet};
+use serde::{Deserialize, Serialize};
 use std::rc::Rc;
 
 pub static mut PROF: [u64; 8] = [0; 8];
@@ -503,7 +504,7 @@ fn median(dates: &mut [i64]) -> i64 {
 /// Where the block is in posting time: the median Date of the last FRONTIER_ARTICLES plausible ones. Date headers
 /// alone aren't trusted: a date more than PLAUSIBLE_MS from the frontier is set aside, and only FRONTIER_ARTICLES
 /// such dates in a row, agreeing among themselves, move the frontier there.
-#[derive(Default)]
+#[derive(Default, Serialize, Deserialize)]
 pub struct Frontier {
     ring: Vec<i64>,
     at: usize,
@@ -592,6 +593,163 @@ struct OpenFile {
     latest: Option<i64>,
 }
 
+// The snapshot (REBUILD.md, "Voyager 3: window indexing", 5): the window's state between two batches, serialized
+// (bincode, gzip by the worker) and kept by the coordinator, restored by the group's next holder. Segments share
+// their file's FileMeta through an Rc; in the snapshot the files are a table and a segment points at its index.
+#[derive(Serialize, Deserialize)]
+struct SegSnap {
+    file: u32,
+    number: u32,
+    message_id: String,
+    bytes: i64,
+    article: i64,
+    posted_at: Option<i64>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenFileSnap {
+    parts: Vec<SegSnap>,
+    total: u32,
+    touched: u64,
+    earliest: Option<i64>,
+    latest: Option<i64>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenReleaseSnap {
+    upload: usize,
+    files: Vec<Vec<SegSnap>>,
+    first: Option<i64>,
+    last: Option<i64>,
+    touched: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenAlbumSnap {
+    upload: usize,
+    members: Vec<(String, OpenReleaseSnap)>,
+    first: Option<i64>,
+    last: Option<i64>,
+    touched: u64,
+}
+
+/// A file's details in the snapshot (FileMeta, its release identity's source as text: a &'static str in memory).
+#[derive(Serialize, Deserialize)]
+struct FileMetaSnap {
+    file_key: String,
+    name: String,
+    poster: String,
+    subject: String,
+    total_parts: u32,
+    groups: Vec<String>,
+    release_key: String,
+    release_name: String,
+    release_source: String,
+    release_file_total: Option<i64>,
+}
+
+/// The sources release_identity gives (lib/release/source.ts), as the static strings the code compares.
+fn source_static(source: &str) -> &'static str {
+    match source {
+        "subject" => "subject",
+        "post" => "post",
+        "stem" => "stem",
+        "album" => "album",
+        "weak-stem" => "weak-stem",
+        _ => "stem",
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Snapshot {
+    pub version: u32,
+    pub group: String,
+    /// The article the window stands at: the next one to index (a batch boundary).
+    pub next: i64,
+    frontier: Frontier,
+    files: Vec<FileMetaSnap>,
+    aggregator: Vec<(String, Vec<OpenFileSnap>)>,
+    aggregator_batch: u64,
+    open: Vec<(String, Vec<OpenReleaseSnap>)>,
+    albums: Vec<(String, Vec<OpenAlbumSnap>)>,
+    uploads: Vec<(String, usize)>,
+    batch_totals: Vec<(String, i64, u32, u64)>,
+}
+
+/// The snapshot's format version: raised when the state's shape changes (an older snapshot is refused: the group
+/// continues at its resume point, one seam).
+pub const SNAPSHOT_VERSION: u32 = 1;
+
+/// The files of a snapshot as they are taken: each FileMeta once, by its Rc.
+#[derive(Default)]
+struct FileTable {
+    index: FxHashMap<*const FileMeta, u32>,
+    files: Vec<Rc<FileMeta>>,
+}
+
+impl FileTable {
+    fn seg(&mut self, s: &Seg) -> SegSnap {
+        let ptr = Rc::as_ptr(&s.file);
+        let file = match self.index.get(&ptr) {
+            Some(&i) => i,
+            None => {
+                let i = self.files.len() as u32;
+                self.index.insert(ptr, i);
+                self.files.push(s.file.clone());
+                i
+            }
+        };
+        SegSnap { file, number: s.number, message_id: s.message_id.clone(), bytes: s.bytes, article: s.article, posted_at: s.posted_at }
+    }
+
+    fn segs(&mut self, segs: &[Seg]) -> Vec<SegSnap> {
+        segs.iter().map(|s| self.seg(s)).collect()
+    }
+
+    fn release(&mut self, u: &OpenRelease) -> OpenReleaseSnap {
+        OpenReleaseSnap { upload: u.upload, files: u.files.iter().map(|f| self.segs(f)).collect(), first: u.first, last: u.last, touched: u.touched }
+    }
+
+    /// The table's files as the snapshot stores them.
+    fn into_files(self) -> Vec<FileMetaSnap> {
+        self.files
+            .iter()
+            .map(|f| FileMetaSnap {
+                file_key: f.file_key.clone(),
+                name: f.name.clone(),
+                poster: f.poster.clone(),
+                subject: f.subject.clone(),
+                total_parts: f.total_parts,
+                groups: f.groups.clone(),
+                release_key: f.release.key.clone(),
+                release_name: f.release.name.clone(),
+                release_source: f.release.source.to_string(),
+                release_file_total: f.release.file_total,
+            })
+            .collect()
+    }
+}
+
+fn file_of(f: FileMetaSnap) -> FileMeta {
+    FileMeta {
+        file_key: f.file_key,
+        name: f.name,
+        poster: f.poster,
+        subject: f.subject,
+        total_parts: f.total_parts,
+        groups: f.groups,
+        release: Release { key: f.release_key, name: f.release_name, source: source_static(&f.release_source), file_total: f.release_file_total },
+    }
+}
+
+fn seg_of(s: SegSnap, files: &[Rc<FileMeta>]) -> Seg {
+    Seg { file: files[s.file as usize].clone(), number: s.number, message_id: s.message_id, bytes: s.bytes, article: s.article, posted_at: s.posted_at }
+}
+
+fn release_of(u: OpenReleaseSnap, files: &[Rc<FileMeta>]) -> OpenRelease {
+    OpenRelease { upload: u.upload, files: u.files.into_iter().map(|f| f.into_iter().map(|s| seg_of(s, files)).collect()).collect(), first: u.first, last: u.last, touched: u.touched }
+}
+
 impl OpenFile {
     fn distance(&self, t: Option<i64>) -> i64 {
         match (t, self.earliest, self.latest) {
@@ -873,6 +1031,107 @@ impl RangeIndexer {
         }
     }
 
+    /// A worker's indexer for a group it holds (REBUILD.md design 1): ranges come one after another (begin_range),
+    /// the window carried between them; the batch starts at the first range's resume point.
+    pub fn for_group(group: &str, rules: Rules, idle: u64) -> Self {
+        Self::new(group, 0, 0, 0, rules, idle)
+    }
+
+    /// The next range of the group: the batch starts at its resume point (after a gap too), the range's end is known.
+    pub fn begin_range(&mut self, next: i64, end: i64) {
+        self.end = end;
+        if self.batch.headers == 0 && self.batch.releases.is_empty() && self.batch.candidates.is_empty() {
+            self.batch.from = next;
+            self.batch.next = next;
+        }
+    }
+
+    /// The article the window stands at: the next one to index (right after a take, a batch boundary).
+    pub fn next(&self) -> i64 {
+        self.batch.next
+    }
+
+    /// The window's state between two batches (right after a take; the batch must be empty), serialized (bincode).
+    pub fn snapshot(&self) -> Result<Vec<u8>, String> {
+        if self.batch.headers != 0 || !self.batch.releases.is_empty() || !self.batch.candidates.is_empty() {
+            return Err("a snapshot is taken right after a batch, not with one open".to_string());
+        }
+        let mut table = FileTable::default();
+        let aggregator: Vec<(String, Vec<OpenFileSnap>)> = self
+            .aggregator
+            .open
+            .iter()
+            .map(|(key, copies)| {
+                (key.clone(), copies.iter().map(|c| OpenFileSnap { parts: table.segs(&c.parts), total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }).collect())
+            })
+            .collect();
+        let open: Vec<(String, Vec<OpenReleaseSnap>)> = self.open.iter().map(|(key, uploads)| (key.clone(), uploads.iter().map(|u| table.release(u)).collect())).collect();
+        let albums: Vec<(String, Vec<OpenAlbumSnap>)> = self
+            .albums
+            .iter()
+            .map(|(key, list)| {
+                (
+                    key.clone(),
+                    list.iter()
+                        .map(|a| OpenAlbumSnap { upload: a.upload, members: a.members.iter().map(|(k, u)| (k.clone(), table.release(u))).collect(), first: a.first, last: a.last, touched: a.touched })
+                        .collect(),
+                )
+            })
+            .collect();
+        let snapshot = Snapshot {
+            version: SNAPSHOT_VERSION,
+            group: self.group.clone(),
+            next: self.batch.next,
+            frontier: Frontier { ring: self.frontier.ring.clone(), at: self.frontier.at, value: self.frontier.value, start: self.frontier.start, aside: self.frontier.aside.clone(), pending: self.frontier.pending },
+            files: table.into_files(),
+            aggregator,
+            aggregator_batch: self.aggregator.batch,
+            open,
+            albums,
+            uploads: self.uploads.iter().map(|(k, &n)| (k.clone(), n)).collect(),
+            batch_totals: self.batch_totals.iter().map(|((poster, total), (n, touched))| (poster.clone(), *total, *n, *touched)).collect(),
+        };
+        bincode::serialize(&snapshot).map_err(|e| e.to_string())
+    }
+
+    /// A worker's indexer for a group from its snapshot (the rules fresh from the lease).
+    pub fn restore(bytes: &[u8], rules: Rules, idle: u64) -> Result<Self, String> {
+        let snapshot: Snapshot = bincode::deserialize(bytes).map_err(|e| format!("snapshot: {e}"))?;
+        if snapshot.version != SNAPSHOT_VERSION {
+            return Err(format!("snapshot version {} (this build writes {SNAPSHOT_VERSION})", snapshot.version));
+        }
+        let files: Vec<Rc<FileMeta>> = snapshot.files.into_iter().map(|f| Rc::new(file_of(f))).collect();
+        let mut ix = Self::for_group(&snapshot.group, rules, idle);
+        ix.frontier = snapshot.frontier;
+        ix.aggregator.batch = snapshot.aggregator_batch;
+        for (key, copies) in snapshot.aggregator {
+            ix.aggregator.open.insert(
+                key,
+                copies
+                    .into_iter()
+                    .map(|c| {
+                        let parts: Vec<Seg> = c.parts.into_iter().map(|s| seg_of(s, &files)).collect();
+                        let seen = parts.iter().map(|s| s.number).collect();
+                        OpenFile { parts, seen, total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }
+                    })
+                    .collect(),
+            );
+        }
+        for (key, uploads) in snapshot.open {
+            ix.open.insert(key, uploads.into_iter().map(|u| release_of(u, &files)).collect());
+        }
+        for (key, list) in snapshot.albums {
+            ix.albums.insert(
+                key,
+                list.into_iter().map(|a| OpenAlbum { upload: a.upload, members: a.members.into_iter().map(|(k, u)| (k, release_of(u, &files))).collect(), first: a.first, last: a.last, touched: a.touched }).collect(),
+            );
+        }
+        ix.uploads = snapshot.uploads.into_iter().collect();
+        ix.batch_totals = snapshot.batch_totals.into_iter().map(|(poster, total, n, touched)| ((poster, total), (n, touched))).collect();
+        ix.batch = Batch { from: snapshot.next, next: snapshot.next, ..Default::default() };
+        Ok(ix)
+    }
+
     pub fn add_text(&mut self, text: &str, next: i64) {
         let mut segments: Vec<Seg> = Vec::new();
         let mut crossposts: Vec<String> = Vec::new();
@@ -1488,6 +1747,80 @@ mod tests {
         assert_eq!(real.file_total, Some(120));
     }
 
+    #[test]
+    fn a_snapshot_restores_the_window_between_two_ranges() {
+        let rules = || {
+            let mut priorities = FxHashMap::default();
+            priorities.insert(GROUP.to_string(), 1);
+            Rules::defaults(priorities)
+        };
+        let day = 24 * 60;
+        // Range 1: a release whose last file comes in range 2, a day later; an album's first member; fillers.
+        let mut first = vec![
+            line(10, r#"Two.Ranges [1/2] - "Two.Ranges.part1.rar" yEnc (1/1)"#, 0),
+            line(11, r#"[1/1] "Scan_001.pdf" yEnc (1/1)"#, 1),
+        ];
+        first.extend(fillers(100, 100, 2, 1));
+        // The second file within the upload gap of the first (3 h), the album's second member within its window.
+        let mut second = vec![
+            line(1000, r#"Two.Ranges [2/2] - "Two.Ranges.part2.rar" yEnc (1/1)"#, 100),
+            line(1001, r#"[1/1] "Scan_002.pdf" yEnc (1/1)"#, 101),
+        ];
+        second.extend(fillers(2000, 300, 3 * day, 1));
+        let summary = |batches: Vec<Batch>| {
+            let mut got: Vec<(String, &str, usize, bool)> = batches.iter().flat_map(|b| b.releases.iter()).filter(|r| !r.name.starts_with("Filler")).map(|r| (r.name.clone(), r.source, r.files, r.finished)).collect();
+            got.sort();
+            got
+        };
+        // In one go.
+        let mut whole = RangeIndexer::for_group(GROUP, rules(), 5);
+        whole.begin_range(1, 999);
+        whole.add_text(&first.join("\n"), 1000);
+        let w1 = whole.take();
+        whole.begin_range(1000, 1_000_000);
+        whole.add_text(&second.join("\n"), 1_000_001);
+        let w2 = whole.take();
+        // With a snapshot between the ranges, restored into a fresh indexer.
+        let mut before = RangeIndexer::for_group(GROUP, rules(), 5);
+        before.begin_range(1, 999);
+        before.add_text(&first.join("\n"), 1000);
+        let s1 = before.take();
+        assert_eq!(before.next(), 1000);
+        let bytes = before.snapshot().unwrap();
+        assert!(bytes.len() > 1000);
+        let mut after = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
+        assert_eq!(after.next(), 1000);
+        assert_eq!(after.open_releases(), before.open_releases());
+        assert_eq!(after.open_files(), before.open_files());
+        after.begin_range(1000, 1_000_000);
+        after.add_text(&second.join("\n"), 1_000_001);
+        let s2 = after.take();
+        assert_eq!(summary(vec![s1, s2]), summary(vec![w1, w2]));
+        let got = summary(vec![after.take()]);
+        assert!(got.is_empty());
+        // The two-range release is one, finished, with both files; the album joined its second member.
+        let mut a2 = RangeIndexer::for_group(GROUP, rules(), 5);
+        a2.begin_range(1, 999);
+        a2.add_text(&first.join("\n"), 1000);
+        let _ = a2.take();
+        let restored = RangeIndexer::restore(&a2.snapshot().unwrap(), rules(), 5).unwrap();
+        let _ = restored;
+        let s2b = {
+            let mut r = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
+            r.begin_range(1000, 1_000_000);
+            r.add_text(&second.join("\n"), 1_000_001);
+            r.take()
+        };
+        let two = s2b.releases.iter().find(|r| r.name == "Two Ranges").expect("the two-range release");
+        assert!(two.finished && two.files == 2);
+        assert!(s2b.releases.iter().any(|r| r.source == "album" && r.files == 2));
+        // A snapshot with a batch open is refused; one of another version too.
+        let mut open = RangeIndexer::for_group(GROUP, rules(), 5);
+        open.begin_range(1, 999);
+        open.add_text(&first.join("\n"), 1000);
+        assert!(open.snapshot().is_err());
+    }
+
     #[test]
     fn the_frontier_ignores_implausible_dates_and_follows_a_pause() {
         let hour = 3_600_000i64;

edge/src/lease.rs

@@ -11,6 +11,40 @@ pub struct LeaseJson {
     pub rules: RulesJson,
 }
 
+/// A group lease (lib/edge/lease.ts GroupLease): the group a worker thread holds, the rules, the window's snapshot if any.
+#[derive(Deserialize)]
+pub struct GroupLeaseJson {
+    pub group: GroupJson,
+    pub rules: RulesJson,
+    pub snapshot: Option<SnapshotJson>,
+}
+
+#[derive(Deserialize)]
+pub struct GroupJson {
+    pub id: i64,
+    pub name: String,
+    pub priority: i64,
+}
+
+#[derive(Deserialize)]
+pub struct SnapshotJson {
+    pub next: i64,
+}
+
+/// The group's next range (lib/edge/lease.ts NextRange).
+#[derive(Deserialize)]
+pub struct NextRangeJson {
+    pub range: RangeSpanJson,
+}
+
+#[derive(Deserialize, Clone, Copy)]
+pub struct RangeSpanJson {
+    pub id: i64,
+    pub start: i64,
+    pub end: i64,
+    pub next: i64,
+}
+
 #[derive(Deserialize)]
 pub struct RangeJson {
     pub id: i64,