Pending ladder by class, twins by content, dropped classes at the edge

The prober names 6% of the unnamed releases it finishes and the pending
queue (44M) grows eleven times faster than it probes. A sample of 44,530
finished pending rows shows the headers predict a name: size bucket, name
shape, completeness, tool and group separate 0% classes from 95% ones,
while the learner keyed on tool and group alone (LOG.md (130)).

- lib/release/classes.ts, names::pending_class, SQL pending_class():
  a release's class (size × shape × partial, 1..70), the three in step.
- Migration 6800: pending_releases.klass and target_bytes (the first
  target's article; candidates carry bytes now), pending_classes with
  each class's climbs, names, waiting and verdict, the claim's index.
- The ladder claims the classes with the best record first (a prior for
  the unexplored, at most a quarter of a claim from one), the cheapest
  target first within one, then the oldest of the rest; records classes
  like signatures and drops them at the same thresholds; discards their
  waiting releases. Claimed rows sorted in code: an UPDATE returns none.
- The rule set's droppedClasses: the edge keeps unnamed releases of a
  dropped class out of the pending store (counted as garbage), the
  explored sample excepted.
- hash_twins.ts: a probed file whose first 16 KiB and size match a named
  catalog release's names a pending release the probe didn't, and sets
  the inner name of a catalog release without one.
- pending:classes classes the rows from before and seeds the record from
  the finished ones; rules:classes publishes the dropped classes;
  releases:readable also marks the rows' candidates unreadable.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

edge/src/indexer.rs

@@ -72,6 +72,8 @@ pub struct Candidate {
     pub kind: &'static str,
     pub message_id: String,
     pub file: String,
+    /// The target article's bytes: the cheapest probe first within a class (LOG.md (130)).
+    pub bytes: i64,
 }
 
 #[derive(Default)]
@@ -1451,7 +1453,16 @@ impl RangeIndexer {
                 }
                 if !chunk.readable {
                     // Unnamed: no chunk (its segments are read again once a probe names it), the tool for its signature.
-                    chunk.tool = Some(names::upload_tool(&list[0][0].message_id));
+                    let tool = names::upload_tool(&list[0][0].message_id);
+                    // A class the ladder found never names anything stays out of the pending store, its explored
+                    // sample excepted (so the rate keeps being measured); counted as garbage (LOG.md (130)).
+                    let klass = names::pending_class(&chunk.name, chunk.bytes.max(0) as u64, chunk.partial);
+                    if self.rules.dropped_classes.contains(&names::class_key(tool, &self.group, klass)) && !names::explored(key, self.rules.sample_percent) {
+                        let segments: usize = list.iter().map(|f| f.len()).sum();
+                        self.count("garbage", format!("{tool} {} #{klass}", self.group), segments as u64, list.len() as u64);
+                        return;
+                    }
+                    chunk.tool = Some(tool);
                     chunk.payload = encode_chunk(&[]);
                 }
                 if chunk.readable {
@@ -1574,7 +1585,7 @@ fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
 fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate> {
     let mut out = Vec::new();
     let mut tail: Option<(i64, &Vec<Seg>)> = None;
-    let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
+    let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone(), bytes: s.bytes };
     for f in files {
         let name = &f[0].file.name;
         let volume = names::seven_zip_volume(name);
@@ -1595,7 +1606,7 @@ fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate>
         }
     }
     if let Some((_, f)) = tail {
-        out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
+        out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone(), bytes: f.last().unwrap().bytes });
     }
     if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
         out.push(c("lone", &files[0][0]));

edge/src/lease.rs

@@ -62,6 +62,8 @@ pub struct RulesJson {
     #[serde(default)]
     pub dropped_signatures: Vec<String>,
     #[serde(default)]
+    pub dropped_classes: Vec<String>,
+    #[serde(default)]
     pub dropped_tools: Option<Vec<String>>,
     #[serde(default)]
     pub per_article_max_parts: Option<u32>,
@@ -83,6 +85,7 @@ impl RulesJson {
         let mut rules = Rules::defaults(self.priorities.into_iter().collect());
         rules.version = self.version;
         rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
+        rules.dropped_classes = self.dropped_classes.into_iter().collect();
         if let Some(tools) = self.dropped_tools {
             rules.dropped_tools = tools.into_iter().collect();
         }

edge/src/names.rs

@@ -1228,6 +1228,39 @@ pub fn is_unreadable_name(name: &str) -> bool {
     random > 0 && random > words
 }
 
+/// The class of an unnamed release (lib/release/classes.ts pendingClass; LOG.md (130)): size bucket × name shape ×
+/// partial, 1..70. The lease's dropped classes ("tool\tgroup\tklass") are skipped at the edge.
+pub fn pending_class(name: &str, bytes: u64, partial: bool) -> u8 {
+    let size = [1_000_000u64, 10_000_000, 100_000_000, 1_000_000_000].iter().filter(|&&b| bytes >= b).count() as u8;
+    let s = match name.rfind('.') {
+        Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
+        _ => name,
+    };
+    let b = s.as_bytes();
+    let all = |f: fn(&u8) -> bool| !b.is_empty() && b.iter().all(f);
+    let shape: u8 = if b.len() >= 16 && all(u8::is_ascii_hexdigit) {
+        1
+    } else if b.len() >= 6 && all(u8::is_ascii_digit) {
+        2
+    } else if b.len() >= 20 && all(|c| c.is_ascii_lowercase() || (b'2'..=b'7').contains(c)) && b.iter().any(|c| (b'2'..=b'7').contains(c)) {
+        3
+    } else if all(u8::is_ascii_alphabetic) {
+        4
+    } else if all(u8::is_ascii_alphanumeric) {
+        5
+    } else if all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'-')) {
+        6
+    } else {
+        7
+    };
+    size * 14 + (shape - 1) * 2 + u8::from(partial) + 1
+}
+
+/// A class as droppedClasses lists it.
+pub fn class_key(tool: &str, group: &str, klass: u8) -> String {
+    format!("{tool}\t{group}\t{klass}")
+}
+
 /// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
 /// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
 /// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
@@ -1559,6 +1592,8 @@ pub struct Rules {
     pub priorities: rustc_hash::FxHashMap<String, u32>,
     /// "tool\tgroup"
     pub dropped_signatures: rustc_hash::FxHashSet<String>,
+    /// "tool\tgroup\tklass" (pending_class): unnamed releases of these classes don't go to the pending store.
+    pub dropped_classes: rustc_hash::FxHashSet<String>,
     pub dropped_tools: rustc_hash::FxHashSet<String>,
     pub per_article_max_parts: u32,
     pub sample_percent: f64,
@@ -1577,6 +1612,7 @@ impl Rules {
             version: 1,
             priorities,
             dropped_signatures: Default::default(),
+            dropped_classes: Default::default(),
             dropped_tools: ["ngPost", "hex32", "yuun", "JBinUp", "random"].iter().map(|t| t.to_string()).collect(),
             per_article_max_parts: 3,
             sample_percent: 1.0,
@@ -1785,6 +1821,20 @@ mod tests {
         assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
     }
 
+    #[test]
+    fn pending_classes() {
+        use super::pending_class;
+        // Size bucket × shape × partial: a 5 MB hex name whole, the same partial, a 500 MB scramble with separators.
+        assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, false), 15);
+        assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, true), 16);
+        assert_eq!(pending_class("Zx5AjNaK-mh7CFg6Hf8uu", 500_000_000, false), 3 * 14 + 5 * 2 + 1);
+        assert_eq!(pending_class("1650777167", 100, false), 3);
+        assert_eq!(pending_class("mrrtmzrzmi4dgmrzmyytqmlbztczbx", 2_000_000_000, false), 4 * 14 + 2 * 2 + 1);
+        assert_eq!(pending_class("PsNdnYiugfZ.rar", 20_000_000, false), 2 * 14 + 3 * 2 + 1);
+        assert_eq!(pending_class("n5d1ueIf34X", 20_000_000, true), 2 * 14 + 4 * 2 + 2);
+        assert_eq!(pending_class("sgkra vU7u2yuw9LDLCjDu04pOez", 999, false), 13);
+    }
+
     #[test]
     fn unreadable_names() {
         use super::is_unreadable_name;

edge/src/wire.rs

@@ -44,6 +44,7 @@ struct WireCandidate<'a> {
     kind: &'a str,
     message_id: &'a str,
     file: &'a str,
+    bytes: i64,
 }
 
 #[derive(Serialize)]
@@ -97,7 +98,7 @@ fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRel
 }
 
 fn candidate(c: &Candidate) -> WireCandidate<'_> {
-    WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
+    WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file, bytes: c.bytes }
 }
 
 /// toWire(batch) as JSON bytes.