The window worker: group leases, the stream serving the corpus, the snapshot through the API (design 1 and 5)

A worker thread leases a group and takes its ranges in article order through
one carried window; the coordinator keeps one cursor a group (group_leases,
migration 6500) and hands out the next range, the rules and nothing about
where the chunks are: the range stream serves a stored range's chunks from the
corpus file and a live one's from the spool. The window's snapshot (bincode:
the frontier, the open files, releases and albums, the upload numbers, the
batch-counter table) goes to the coordinator every EDGE_SNAPSHOT_MINUTES and
when the group is given back, and the next holder restores it; a takeover
after LEASE_TTL_SECONDS plans the ranges from the snapshot's article again.
The prefetcher runs ahead of held groups' cursors and skips chunks the corpus
has. LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY bound what is leased. edge-node
holds groups by default; --ranges keeps the old per-range path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

edge/src/indexer.rs

@@ -7,6 +7,7 @@ use crate::codec::{encode_chunk, ChunkFile};
 use crate::js::{self, sha1_hex};
 use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
 use rustc_hash::{FxHashMap, FxHashSet};
+use serde::{Deserialize, Serialize};
 use std::rc::Rc;
 
 pub static mut PROF: [u64; 8] = [0; 8];
@@ -503,7 +504,7 @@ fn median(dates: &mut [i64]) -> i64 {
 /// Where the block is in posting time: the median Date of the last FRONTIER_ARTICLES plausible ones. Date headers
 /// alone aren't trusted: a date more than PLAUSIBLE_MS from the frontier is set aside, and only FRONTIER_ARTICLES
 /// such dates in a row, agreeing among themselves, move the frontier there.
-#[derive(Default)]
+#[derive(Default, Serialize, Deserialize)]
 pub struct Frontier {
     ring: Vec<i64>,
     at: usize,
@@ -592,6 +593,163 @@ struct OpenFile {
     latest: Option<i64>,
 }
 
+// The snapshot (REBUILD.md, "Voyager 3: window indexing", 5): the window's state between two batches, serialized
+// (bincode, gzip by the worker) and kept by the coordinator, restored by the group's next holder. Segments share
+// their file's FileMeta through an Rc; in the snapshot the files are a table and a segment points at its index.
+#[derive(Serialize, Deserialize)]
+struct SegSnap {
+    file: u32,
+    number: u32,
+    message_id: String,
+    bytes: i64,
+    article: i64,
+    posted_at: Option<i64>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenFileSnap {
+    parts: Vec<SegSnap>,
+    total: u32,
+    touched: u64,
+    earliest: Option<i64>,
+    latest: Option<i64>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenReleaseSnap {
+    upload: usize,
+    files: Vec<Vec<SegSnap>>,
+    first: Option<i64>,
+    last: Option<i64>,
+    touched: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct OpenAlbumSnap {
+    upload: usize,
+    members: Vec<(String, OpenReleaseSnap)>,
+    first: Option<i64>,
+    last: Option<i64>,
+    touched: u64,
+}
+
+/// A file's details in the snapshot (FileMeta, its release identity's source as text: a &'static str in memory).
+#[derive(Serialize, Deserialize)]
+struct FileMetaSnap {
+    file_key: String,
+    name: String,
+    poster: String,
+    subject: String,
+    total_parts: u32,
+    groups: Vec<String>,
+    release_key: String,
+    release_name: String,
+    release_source: String,
+    release_file_total: Option<i64>,
+}
+
+/// The sources release_identity gives (lib/release/source.ts), as the static strings the code compares.
+fn source_static(source: &str) -> &'static str {
+    match source {
+        "subject" => "subject",
+        "post" => "post",
+        "stem" => "stem",
+        "album" => "album",
+        "weak-stem" => "weak-stem",
+        _ => "stem",
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Snapshot {
+    pub version: u32,
+    pub group: String,
+    /// The article the window stands at: the next one to index (a batch boundary).
+    pub next: i64,
+    frontier: Frontier,
+    files: Vec<FileMetaSnap>,
+    aggregator: Vec<(String, Vec<OpenFileSnap>)>,
+    aggregator_batch: u64,
+    open: Vec<(String, Vec<OpenReleaseSnap>)>,
+    albums: Vec<(String, Vec<OpenAlbumSnap>)>,
+    uploads: Vec<(String, usize)>,
+    batch_totals: Vec<(String, i64, u32, u64)>,
+}
+
+/// The snapshot's format version: raised when the state's shape changes (an older snapshot is refused: the group
+/// continues at its resume point, one seam).
+pub const SNAPSHOT_VERSION: u32 = 1;
+
+/// The files of a snapshot as they are taken: each FileMeta once, by its Rc.
+#[derive(Default)]
+struct FileTable {
+    index: FxHashMap<*const FileMeta, u32>,
+    files: Vec<Rc<FileMeta>>,
+}
+
+impl FileTable {
+    fn seg(&mut self, s: &Seg) -> SegSnap {
+        let ptr = Rc::as_ptr(&s.file);
+        let file = match self.index.get(&ptr) {
+            Some(&i) => i,
+            None => {
+                let i = self.files.len() as u32;
+                self.index.insert(ptr, i);
+                self.files.push(s.file.clone());
+                i
+            }
+        };
+        SegSnap { file, number: s.number, message_id: s.message_id.clone(), bytes: s.bytes, article: s.article, posted_at: s.posted_at }
+    }
+
+    fn segs(&mut self, segs: &[Seg]) -> Vec<SegSnap> {
+        segs.iter().map(|s| self.seg(s)).collect()
+    }
+
+    fn release(&mut self, u: &OpenRelease) -> OpenReleaseSnap {
+        OpenReleaseSnap { upload: u.upload, files: u.files.iter().map(|f| self.segs(f)).collect(), first: u.first, last: u.last, touched: u.touched }
+    }
+
+    /// The table's files as the snapshot stores them.
+    fn into_files(self) -> Vec<FileMetaSnap> {
+        self.files
+            .iter()
+            .map(|f| FileMetaSnap {
+                file_key: f.file_key.clone(),
+                name: f.name.clone(),
+                poster: f.poster.clone(),
+                subject: f.subject.clone(),
+                total_parts: f.total_parts,
+                groups: f.groups.clone(),
+                release_key: f.release.key.clone(),
+                release_name: f.release.name.clone(),
+                release_source: f.release.source.to_string(),
+                release_file_total: f.release.file_total,
+            })
+            .collect()
+    }
+}
+
+fn file_of(f: FileMetaSnap) -> FileMeta {
+    FileMeta {
+        file_key: f.file_key,
+        name: f.name,
+        poster: f.poster,
+        subject: f.subject,
+        total_parts: f.total_parts,
+        groups: f.groups,
+        release: Release { key: f.release_key, name: f.release_name, source: source_static(&f.release_source), file_total: f.release_file_total },
+    }
+}
+
+fn seg_of(s: SegSnap, files: &[Rc<FileMeta>]) -> Seg {
+    Seg { file: files[s.file as usize].clone(), number: s.number, message_id: s.message_id, bytes: s.bytes, article: s.article, posted_at: s.posted_at }
+}
+
+fn release_of(u: OpenReleaseSnap, files: &[Rc<FileMeta>]) -> OpenRelease {
+    OpenRelease { upload: u.upload, files: u.files.into_iter().map(|f| f.into_iter().map(|s| seg_of(s, files)).collect()).collect(), first: u.first, last: u.last, touched: u.touched }
+}
+
 impl OpenFile {
     fn distance(&self, t: Option<i64>) -> i64 {
         match (t, self.earliest, self.latest) {
@@ -873,6 +1031,107 @@ impl RangeIndexer {
         }
     }
 
+    /// A worker's indexer for a group it holds (REBUILD.md design 1): ranges come one after another (begin_range),
+    /// the window carried between them; the batch starts at the first range's resume point.
+    pub fn for_group(group: &str, rules: Rules, idle: u64) -> Self {
+        Self::new(group, 0, 0, 0, rules, idle)
+    }
+
+    /// The next range of the group: the batch starts at its resume point (after a gap too), the range's end is known.
+    pub fn begin_range(&mut self, next: i64, end: i64) {
+        self.end = end;
+        if self.batch.headers == 0 && self.batch.releases.is_empty() && self.batch.candidates.is_empty() {
+            self.batch.from = next;
+            self.batch.next = next;
+        }
+    }
+
+    /// The article the window stands at: the next one to index (right after a take, a batch boundary).
+    pub fn next(&self) -> i64 {
+        self.batch.next
+    }
+
+    /// The window's state between two batches (right after a take; the batch must be empty), serialized (bincode).
+    pub fn snapshot(&self) -> Result<Vec<u8>, String> {
+        if self.batch.headers != 0 || !self.batch.releases.is_empty() || !self.batch.candidates.is_empty() {
+            return Err("a snapshot is taken right after a batch, not with one open".to_string());
+        }
+        let mut table = FileTable::default();
+        let aggregator: Vec<(String, Vec<OpenFileSnap>)> = self
+            .aggregator
+            .open
+            .iter()
+            .map(|(key, copies)| {
+                (key.clone(), copies.iter().map(|c| OpenFileSnap { parts: table.segs(&c.parts), total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }).collect())
+            })
+            .collect();
+        let open: Vec<(String, Vec<OpenReleaseSnap>)> = self.open.iter().map(|(key, uploads)| (key.clone(), uploads.iter().map(|u| table.release(u)).collect())).collect();
+        let albums: Vec<(String, Vec<OpenAlbumSnap>)> = self
+            .albums
+            .iter()
+            .map(|(key, list)| {
+                (
+                    key.clone(),
+                    list.iter()
+                        .map(|a| OpenAlbumSnap { upload: a.upload, members: a.members.iter().map(|(k, u)| (k.clone(), table.release(u))).collect(), first: a.first, last: a.last, touched: a.touched })
+                        .collect(),
+                )
+            })
+            .collect();
+        let snapshot = Snapshot {
+            version: SNAPSHOT_VERSION,
+            group: self.group.clone(),
+            next: self.batch.next,
+            frontier: Frontier { ring: self.frontier.ring.clone(), at: self.frontier.at, value: self.frontier.value, start: self.frontier.start, aside: self.frontier.aside.clone(), pending: self.frontier.pending },
+            files: table.into_files(),
+            aggregator,
+            aggregator_batch: self.aggregator.batch,
+            open,
+            albums,
+            uploads: self.uploads.iter().map(|(k, &n)| (k.clone(), n)).collect(),
+            batch_totals: self.batch_totals.iter().map(|((poster, total), (n, touched))| (poster.clone(), *total, *n, *touched)).collect(),
+        };
+        bincode::serialize(&snapshot).map_err(|e| e.to_string())
+    }
+
+    /// A worker's indexer for a group from its snapshot (the rules fresh from the lease).
+    pub fn restore(bytes: &[u8], rules: Rules, idle: u64) -> Result<Self, String> {
+        let snapshot: Snapshot = bincode::deserialize(bytes).map_err(|e| format!("snapshot: {e}"))?;
+        if snapshot.version != SNAPSHOT_VERSION {
+            return Err(format!("snapshot version {} (this build writes {SNAPSHOT_VERSION})", snapshot.version));
+        }
+        let files: Vec<Rc<FileMeta>> = snapshot.files.into_iter().map(|f| Rc::new(file_of(f))).collect();
+        let mut ix = Self::for_group(&snapshot.group, rules, idle);
+        ix.frontier = snapshot.frontier;
+        ix.aggregator.batch = snapshot.aggregator_batch;
+        for (key, copies) in snapshot.aggregator {
+            ix.aggregator.open.insert(
+                key,
+                copies
+                    .into_iter()
+                    .map(|c| {
+                        let parts: Vec<Seg> = c.parts.into_iter().map(|s| seg_of(s, &files)).collect();
+                        let seen = parts.iter().map(|s| s.number).collect();
+                        OpenFile { parts, seen, total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }
+                    })
+                    .collect(),
+            );
+        }
+        for (key, uploads) in snapshot.open {
+            ix.open.insert(key, uploads.into_iter().map(|u| release_of(u, &files)).collect());
+        }
+        for (key, list) in snapshot.albums {
+            ix.albums.insert(
+                key,
+                list.into_iter().map(|a| OpenAlbum { upload: a.upload, members: a.members.into_iter().map(|(k, u)| (k, release_of(u, &files))).collect(), first: a.first, last: a.last, touched: a.touched }).collect(),
+            );
+        }
+        ix.uploads = snapshot.uploads.into_iter().collect();
+        ix.batch_totals = snapshot.batch_totals.into_iter().map(|(poster, total, n, touched)| ((poster, total), (n, touched))).collect();
+        ix.batch = Batch { from: snapshot.next, next: snapshot.next, ..Default::default() };
+        Ok(ix)
+    }
+
     pub fn add_text(&mut self, text: &str, next: i64) {
         let mut segments: Vec<Seg> = Vec::new();
         let mut crossposts: Vec<String> = Vec::new();
@@ -1488,6 +1747,80 @@ mod tests {
         assert_eq!(real.file_total, Some(120));
     }
 
+    #[test]
+    fn a_snapshot_restores_the_window_between_two_ranges() {
+        let rules = || {
+            let mut priorities = FxHashMap::default();
+            priorities.insert(GROUP.to_string(), 1);
+            Rules::defaults(priorities)
+        };
+        let day = 24 * 60;
+        // Range 1: a release whose last file comes in range 2, a day later; an album's first member; fillers.
+        let mut first = vec![
+            line(10, r#"Two.Ranges [1/2] - "Two.Ranges.part1.rar" yEnc (1/1)"#, 0),
+            line(11, r#"[1/1] "Scan_001.pdf" yEnc (1/1)"#, 1),
+        ];
+        first.extend(fillers(100, 100, 2, 1));
+        // The second file within the upload gap of the first (3 h), the album's second member within its window.
+        let mut second = vec![
+            line(1000, r#"Two.Ranges [2/2] - "Two.Ranges.part2.rar" yEnc (1/1)"#, 100),
+            line(1001, r#"[1/1] "Scan_002.pdf" yEnc (1/1)"#, 101),
+        ];
+        second.extend(fillers(2000, 300, 3 * day, 1));
+        let summary = |batches: Vec<Batch>| {
+            let mut got: Vec<(String, &str, usize, bool)> = batches.iter().flat_map(|b| b.releases.iter()).filter(|r| !r.name.starts_with("Filler")).map(|r| (r.name.clone(), r.source, r.files, r.finished)).collect();
+            got.sort();
+            got
+        };
+        // In one go.
+        let mut whole = RangeIndexer::for_group(GROUP, rules(), 5);
+        whole.begin_range(1, 999);
+        whole.add_text(&first.join("\n"), 1000);
+        let w1 = whole.take();
+        whole.begin_range(1000, 1_000_000);
+        whole.add_text(&second.join("\n"), 1_000_001);
+        let w2 = whole.take();
+        // With a snapshot between the ranges, restored into a fresh indexer.
+        let mut before = RangeIndexer::for_group(GROUP, rules(), 5);
+        before.begin_range(1, 999);
+        before.add_text(&first.join("\n"), 1000);
+        let s1 = before.take();
+        assert_eq!(before.next(), 1000);
+        let bytes = before.snapshot().unwrap();
+        assert!(bytes.len() > 1000);
+        let mut after = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
+        assert_eq!(after.next(), 1000);
+        assert_eq!(after.open_releases(), before.open_releases());
+        assert_eq!(after.open_files(), before.open_files());
+        after.begin_range(1000, 1_000_000);
+        after.add_text(&second.join("\n"), 1_000_001);
+        let s2 = after.take();
+        assert_eq!(summary(vec![s1, s2]), summary(vec![w1, w2]));
+        let got = summary(vec![after.take()]);
+        assert!(got.is_empty());
+        // The two-range release is one, finished, with both files; the album joined its second member.
+        let mut a2 = RangeIndexer::for_group(GROUP, rules(), 5);
+        a2.begin_range(1, 999);
+        a2.add_text(&first.join("\n"), 1000);
+        let _ = a2.take();
+        let restored = RangeIndexer::restore(&a2.snapshot().unwrap(), rules(), 5).unwrap();
+        let _ = restored;
+        let s2b = {
+            let mut r = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
+            r.begin_range(1000, 1_000_000);
+            r.add_text(&second.join("\n"), 1_000_001);
+            r.take()
+        };
+        let two = s2b.releases.iter().find(|r| r.name == "Two Ranges").expect("the two-range release");
+        assert!(two.finished && two.files == 2);
+        assert!(s2b.releases.iter().any(|r| r.source == "album" && r.files == 2));
+        // A snapshot with a batch open is refused; one of another version too.
+        let mut open = RangeIndexer::for_group(GROUP, rules(), 5);
+        open.begin_range(1, 999);
+        open.add_text(&first.join("\n"), 1000);
+        assert!(open.snapshot().is_err());
+    }
+
     #[test]
     fn the_frontier_ignores_implausible_dates_and_follows_a_pause() {
         let hour = 3_600_000i64;