Subjects without a quoted file name: the name from the rest of the subject
Test and deploy / test (push) Successful in 51s
Test and deploy / edge (push) Successful in 1m48s
Test and deploy / deploy (push) Successful in 1m25s

Most "obfuscated" subjects of older and smaller groups name their file
without quotes ("DVDFab 8.2.0.5.rar (006/332)", fta's "q3B9…part023.rar
(104/137)", "New_Super_Mario_Bros._[Keygen].exe (1/2)") or with one
quote ("[keira_s04_0037_l.jpg\" yEnc] (2/2)"): 100% of ath 2013's and
fta's, 94% of models' (LOG.md 2026-10-11 (5)). With a segment counter and
no quoted name, unquotedName takes the name before a lone quote (after a
"[", else after the last space), else the text when it ends in a file
name (after "] " or " - "), else its first word that is one; a file
counter stuck in a bracket is dropped. The release identity takes such a
name out of the subject as it would quotes (else every image of a set
was a release: 9,130 instead of 267 on the models fixture).

Both implementations; a fourth fixture (fta, 25k lines) and tests; check
identical on all four. Truly random subjects (ngPost hex = message-id,
random tokens) stay obfuscated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-11 20:52:55 +02:00
1 parent 18b9ffdca3
commit 90f30f445d
13 files changed
+174 -25

No files matched your search

Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
{"group":"alt.binaries.fta","start":210034490,"end":210059489}
+96 -10
View File
@@ -89,8 +89,8 @@ struct Counter {
total: i64,
}
/// The last "(k/N)" after `after`, scanning back from the end (findCounter).
fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
/// The last "(k/N)" after `after` (anywhere when None: JS -1), scanning back from the end (findCounter).
fn find_counter(subject: &[u8], after: Option<usize>) -> Option<Counter> {
let mut open = subject.len();
loop {
// lastIndexOf('(', open - 1): at most open - 1.
@@ -98,7 +98,7 @@ fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
return None;
}
open = subject[..open].iter().rposition(|&c| c == b'(')?;
if open <= after {
if after.is_some_and(|a| open <= a) {
return None;
}
let mut i = open + 1;
@@ -188,10 +188,8 @@ impl Scanner {
return None;
}
let sb = subject.as_bytes();
let name_end = subject.rfind('"')?;
if name_end == 0 {
return None;
}
// After the last quote; without one, anywhere (unquoted_name finds the file name).
let name_end = subject.rfind('"');
let counter = find_counter(sb, name_end)?;
self.key.clear();
self.key.push_str(from);
@@ -223,9 +221,16 @@ impl Scanner {
}
}
fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
let name_start = subject[..name_end].rfind('"')?;
let name = &subject[name_start + 1..name_end];
fn describe(subject: &str, name_end: Option<usize>, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
let quoted = match name_end {
Some(end) if end > 0 => subject[..end].rfind('"').map(|start| subject[start + 1..end].to_string()),
_ => None,
};
let name = match quoted {
Some(name) => name,
None => unquoted_name(&subject[..counter.index])?,
};
let name = name.as_str();
if name.is_empty() || counter.total < 1 {
return None;
}
@@ -244,6 +249,64 @@ fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref:
})
}
/// EXTENSION: /[^.\s]\.[A-Za-z0-9]{2,4}$/
fn has_extension(s: &str) -> bool {
let Some(dot) = s.rfind('.') else { return false };
let ext = &s.as_bytes()[dot + 1..];
if !(2..=4).contains(&ext.len()) || !ext.iter().all(|c| c.is_ascii_alphanumeric()) {
return false;
}
s[..dot].chars().last().is_some_and(|c| c != '.' && !js::is_space(c))
}
/// unquotedName (lib/headers/subject.ts): the file name of a subject without a quoted one, from the text before its counter.
pub fn unquoted_name(before_counter: &str) -> Option<String> {
let mut text = before_counter.trim_end_matches(js::is_space);
if text.len() >= 4 && text.is_char_boundary(text.len() - 4) && text[text.len() - 4..].eq_ignore_ascii_case("yenc") {
text = text[..text.len() - 4].trim_end_matches(js::is_space);
}
if let Some(quote) = text.find('"') {
if text[quote + 1..].contains('"') {
return None;
}
let before = &text[..quote];
if let Some(bracket) = before.rfind('[') {
let inside = js::trim(&before[bracket + 1..]);
// replace(/^\d+\/\d+ +/, '')
let b = inside.as_bytes();
let mut i = 0;
while i < b.len() && b[i].is_ascii_digit() {
i += 1;
}
let mut stripped = inside;
if i > 0 && b.get(i) == Some(&b'/') {
let mut j = i + 1;
while j < b.len() && b[j].is_ascii_digit() {
j += 1;
}
if j > i + 1 && b.get(j) == Some(&b' ') {
while j < b.len() && b[j] == b' ' {
j += 1;
}
stripped = &inside[j..];
}
}
if has_extension(stripped) {
return Some(stripped.to_string());
}
}
let word = &before[before.rfind(' ').map_or(0, |i| i + 1)..];
return has_extension(word).then(|| word.to_string());
}
if has_extension(text) {
let bracket = text.rfind("] ").map_or(0, |i| i + 2);
let dash = text.rfind(" - ").map_or(0, |i| i + 3);
let name = js::trim(&text[bracket.max(dash)..]);
return (!name.is_empty() && has_extension(name)).then(|| name.to_string());
}
text.split(' ').find(|w| has_extension(w)).map(str::to_string)
}
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
let b = text.as_bytes();
@@ -794,4 +857,27 @@ mod tests {
assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["a1", "a2", "a3"]);
assert!(agg.open.is_empty());
}
#[cfg(test)]
mod unquoted_tests {
use super::unquoted_name;
#[test]
fn file_names_of_subjects_without_a_quoted_one() {
let cases = [
("DVDFab 8.2.0.5.rar ", Some("DVDFab 8.2.0.5.rar")),
("q3B9OrpKiCxUnzgtUGnp.part023.rar ", Some("q3B9OrpKiCxUnzgtUGnp.part023.rar")),
("New_Super_Mario_Bros._[Keygen].exe ", Some("New_Super_Mario_Bros._[Keygen].exe")),
("Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE ", Some("Emily_18_-_Bed.part01.rar")),
("[33/117] 674a37548b09.dat ", Some("674a37548b09.dat")),
("pal! [Mivina high(1).wmv\" yEnc] ", Some("Mivina high(1).wmv")),
("[03/21 Geroellheimer - S03E16.part01.rar\" yEnc ", Some("Geroellheimer - S03E16.part01.rar")),
("[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc ", None),
("Re: Will somebody answer the question? ", None),
];
for (text, want) in cases {
assert_eq!(unquoted_name(text).as_deref(), want, "{text}");
}
}
}
}
+2 -1
View File
@@ -514,7 +514,8 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
let post_total = file_counter_total(subject);
let file_total = counter_total(subject).and_then(|t| t.parse::<i64>().ok());
let stem = file_stem(name);
let remainder = subject_remainder(subject);
// A name that isn't quoted (unquoted_name) is taken out as the quotes would have been.
let remainder = if subject.contains(&format!("\"{name}\"")) { subject_remainder(subject) } else { subject_remainder(&subject.replacen(name, " ", 1).replace('"', " ")) };
// A document or comic is a release of its own.
if test(&DOCUMENT, name) {
let shown = replace(&DOCUMENT, &stem, "", false);
+6 -6
View File
@@ -1,4 +1,4 @@
import { fileKey } from '../headers/index.js'
import { fileKey, unquotedName } from '../headers/index.js'
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
/** A file's details, shared by all of its segments (one object per file, not per article). */
@@ -112,8 +112,8 @@ export class SegmentScanner {
/** A segment from an article's fields, or null when its subject is obfuscated. */
segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null {
if (!subject || !messageId) return null
// After the last quote; without one, anywhere (unquotedName finds the file name).
const nameEnd = subject.lastIndexOf('"')
if (nameEnd <= 0) return null
const counter = findCounter(subject, nameEnd)
if (!counter) return null
const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length)
@@ -137,10 +137,10 @@ export class SegmentScanner {
/** A file's details from its first article seen (as parseSubject and segmentFromHeader). */
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
if (nameStart === -1) return null
const name = own(subject.slice(nameStart + 1, nameEnd))
if (!name || counter.total < 1) return null
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
const found = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index))
if (!found || counter.total < 1) return null
const name = own(found)
const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
const poster = own(from)
return {
+1 -1
View File
@@ -1,3 +1,3 @@
export { parseSubject, type ParsedSubject } from './subject.js'
export { parseSubject, unquotedName, type ParsedSubject } from './subject.js'
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js'
+44 -5
View File
@@ -16,6 +16,46 @@ export interface ParsedSubject {
}
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
/** A file name's extension: a dot and 2 to 4 letters or digits at the end. */
const EXTENSION = /[^.\s]\.[A-Za-z0-9]{2,4}$/
/**
* The file name of a subject without a quoted one, from the text before its
* segment counter (LOG.md 2026-10-11 (5): most "obfuscated" subjects of older
* groups): with one quote only, the name before it (after a "[", else after
* the last space: `[keira_s04_0037_l.jpg" yEnc] (2/2)`); without quotes, the
* text when it ends in a file name (after a "] " or " - ": `DVDFab
* 8.2.0.5.rar (006/332)`, `[33/117] 674a37548b09.dat (1/1)`), else its
* first word that is one (`Emily_18_-_Bed.part01.rar Emily 18 bed video
* (01/24)`). Null when no file name is found: the subject stays obfuscated.
* Words are split on spaces only (both implementations alike).
*/
export function unquotedName(beforeCounter: string): string | null {
let text = beforeCounter.trimEnd()
if (/yenc$/i.test(text)) text = text.slice(0, -4).trimEnd()
const quote = text.indexOf('"')
if (quote !== -1) {
if (text.indexOf('"', quote + 1) !== -1) return null
const before = text.slice(0, quote)
const bracket = before.lastIndexOf('[')
if (bracket !== -1) {
// A file counter that lost its closing bracket ("[03/21 Name.part01.rar") isn't part of the name.
const inBracket = before.slice(bracket + 1).trim().replace(/^\d+\/\d+ +/, '')
if (EXTENSION.test(inBracket)) return inBracket
}
const word = before.slice(before.lastIndexOf(' ') + 1)
return EXTENSION.test(word) ? word : null
}
if (EXTENSION.test(text)) {
const bracket = text.lastIndexOf('] ')
const dash = text.lastIndexOf(' - ')
const cut = Math.max(bracket === -1 ? 0 : bracket + 2, dash === -1 ? 0 : dash + 3)
const name = text.slice(cut).trim()
return name && EXTENSION.test(name) ? name : null
}
for (const word of text.split(' ')) if (EXTENSION.test(word)) return word
return null
}
const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
/**
@@ -28,16 +68,15 @@ const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
export function parseSubject(subject: string): ParsedSubject | null {
const nameEnd = subject.lastIndexOf('"')
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
if (nameStart === -1) return null
const name = subject.slice(nameStart + 1, nameEnd)
if (!name) return null
// The segment counter is the last "(k/N)" after the filename; a file size may follow it.
// The segment counter is the last "(k/N)" after the filename (anywhere without a quote); a file size may follow it.
let counter: RegExpExecArray | undefined
for (const match of subject.matchAll(SEGMENT_COUNTER)) {
if (match.index > nameEnd) counter = match
}
if (!counter) return null
const name = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index))
if (!name) return null
const segment = Number.parseInt(counter[1], 10)
const totalSegments = Number.parseInt(counter[2], 10)
@@ -47,7 +86,7 @@ export function parseSubject(subject: string): ParsedSubject | null {
.replace(/\s+/g, ' ')
.trim()
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart !== -1 ? nameStart : counter.index))
return {
name,
segment,
+4 -2
View File
@@ -197,8 +197,10 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
}
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
const remainder = subjectRemainder(input.subject)
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". A name that isn't
// quoted (unquotedName: "DVDFab 8.2.0.5.rar (006/332)", "[keira_s04_0037_l.jpg" yEnc] (2/2)") is taken out
// as the quotes would have been: otherwise every file of a set would name a release of its own.
const remainder = subjectRemainder(input.subject.includes(`"${input.name}"`) ? input.subject : input.subject.replace(input.name, ' ').replace(/"/g, ' '))
// A document or comic is a release of its own (collections post hundreds under one subject).
if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
const image = IMAGE.test(input.name)
+20
View File
@@ -41,6 +41,26 @@ test.group('subject / parseSubject', () => {
assert.isNull(parseSubject('"name.rar" yEnc (0/5)'))
assert.isNull(parseSubject('"name.rar" yEnc (6/5)'))
assert.isNull(parseSubject('"" yEnc (1/5)'))
// A counter but no file name: still obfuscated.
assert.isNull(parseSubject('[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc (1/12432)'))
assert.isNull(parseSubject('Re: Will somebody answer the question? (1/2)'))
})
test('finds the file name of subjects without a quoted one (unquotedName)', ({ assert }) => {
const cases: [string, string, number, number, number?][] = [
['DVDFab 8.2.0.5.rar (006/332)', 'DVDFab 8.2.0.5.rar', 6, 332],
['q3B9OrpKiCxUnzgtUGnp.part023.rar (104/137)', 'q3B9OrpKiCxUnzgtUGnp.part023.rar', 104, 137],
['New_Super_Mario_Bros._[Keygen].exe (1/2)', 'New_Super_Mario_Bros._[Keygen].exe', 1, 2],
['Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE (01/24)', 'Emily_18_-_Bed.part01.rar', 1, 24],
['[33/117] 674a37548b09.dat (1/1)', '674a37548b09.dat', 1, 1, 33],
["pal! [Mivina high(1).wmv\" yEnc] (323/349)", 'Mivina high(1).wmv', 323, 349],
['[03/21 Geroellheimer - S03E16.part01.rar" yEnc (1/50)', 'Geroellheimer - S03E16.part01.rar', 1, 50],
]
for (const [subject, name, segment, total, fileIndex] of cases) {
const parsed = parseSubject(subject)
assert.include(parsed, { name, segment, totalSegments: total }, subject)
if (fileIndex) assert.equal(parsed!.fileIndex, fileIndex)
}
})
})