Subjects without a quoted file name: the name from the rest of the subject
Most "obfuscated" subjects of older and smaller groups name their file
without quotes ("DVDFab 8.2.0.5.rar (006/332)", fta's "q3B9…part023.rar
(104/137)", "New_Super_Mario_Bros._[Keygen].exe (1/2)") or with one
quote ("[keira_s04_0037_l.jpg\" yEnc] (2/2)"): 100% of ath 2013's and
fta's, 94% of models' (LOG.md 2026-10-11 (5)). With a segment counter and
no quoted name, unquotedName takes the name before a lone quote (after a
"[", else after the last space), else the text when it ends in a file
name (after "] " or " - "), else its first word that is one; a file
counter stuck in a bracket is dropped. The release identity takes such a
name out of the subject as it would quotes (else every image of a set
was a release: 9,130 instead of 267 on the models fixture).
Both implementations; a fourth fixture (fta, 25k lines) and tests; check
identical on all four. Truly random subjects (ngPost hex = message-id,
random tokens) stay obfuscated.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
18b9ffdca3
commit
90f30f445d
13 files changed
+174
-25
No files matched your search
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"group":"alt.binaries.fta","start":210034490,"end":210059489}
|
||||
+96
-10
@@ -89,8 +89,8 @@ struct Counter {
|
||||
total: i64,
|
||||
}
|
||||
|
||||
/// The last "(k/N)" after `after`, scanning back from the end (findCounter).
|
||||
fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
|
||||
/// The last "(k/N)" after `after` (anywhere when None: JS -1), scanning back from the end (findCounter).
|
||||
fn find_counter(subject: &[u8], after: Option<usize>) -> Option<Counter> {
|
||||
let mut open = subject.len();
|
||||
loop {
|
||||
// lastIndexOf('(', open - 1): at most open - 1.
|
||||
@@ -98,7 +98,7 @@ fn find_counter(subject: &[u8], after: usize) -> Option<Counter> {
|
||||
return None;
|
||||
}
|
||||
open = subject[..open].iter().rposition(|&c| c == b'(')?;
|
||||
if open <= after {
|
||||
if after.is_some_and(|a| open <= a) {
|
||||
return None;
|
||||
}
|
||||
let mut i = open + 1;
|
||||
@@ -188,10 +188,8 @@ impl Scanner {
|
||||
return None;
|
||||
}
|
||||
let sb = subject.as_bytes();
|
||||
let name_end = subject.rfind('"')?;
|
||||
if name_end == 0 {
|
||||
return None;
|
||||
}
|
||||
// After the last quote; without one, anywhere (unquoted_name finds the file name).
|
||||
let name_end = subject.rfind('"');
|
||||
let counter = find_counter(sb, name_end)?;
|
||||
self.key.clear();
|
||||
self.key.push_str(from);
|
||||
@@ -223,9 +221,16 @@ impl Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
||||
let name_start = subject[..name_end].rfind('"')?;
|
||||
let name = &subject[name_start + 1..name_end];
|
||||
fn describe(subject: &str, name_end: Option<usize>, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
||||
let quoted = match name_end {
|
||||
Some(end) if end > 0 => subject[..end].rfind('"').map(|start| subject[start + 1..end].to_string()),
|
||||
_ => None,
|
||||
};
|
||||
let name = match quoted {
|
||||
Some(name) => name,
|
||||
None => unquoted_name(&subject[..counter.index])?,
|
||||
};
|
||||
let name = name.as_str();
|
||||
if name.is_empty() || counter.total < 1 {
|
||||
return None;
|
||||
}
|
||||
@@ -244,6 +249,64 @@ fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref:
|
||||
})
|
||||
}
|
||||
|
||||
/// EXTENSION: /[^.\s]\.[A-Za-z0-9]{2,4}$/
|
||||
fn has_extension(s: &str) -> bool {
|
||||
let Some(dot) = s.rfind('.') else { return false };
|
||||
let ext = &s.as_bytes()[dot + 1..];
|
||||
if !(2..=4).contains(&ext.len()) || !ext.iter().all(|c| c.is_ascii_alphanumeric()) {
|
||||
return false;
|
||||
}
|
||||
s[..dot].chars().last().is_some_and(|c| c != '.' && !js::is_space(c))
|
||||
}
|
||||
|
||||
/// unquotedName (lib/headers/subject.ts): the file name of a subject without a quoted one, from the text before its counter.
|
||||
pub fn unquoted_name(before_counter: &str) -> Option<String> {
|
||||
let mut text = before_counter.trim_end_matches(js::is_space);
|
||||
if text.len() >= 4 && text.is_char_boundary(text.len() - 4) && text[text.len() - 4..].eq_ignore_ascii_case("yenc") {
|
||||
text = text[..text.len() - 4].trim_end_matches(js::is_space);
|
||||
}
|
||||
if let Some(quote) = text.find('"') {
|
||||
if text[quote + 1..].contains('"') {
|
||||
return None;
|
||||
}
|
||||
let before = &text[..quote];
|
||||
if let Some(bracket) = before.rfind('[') {
|
||||
let inside = js::trim(&before[bracket + 1..]);
|
||||
// replace(/^\d+\/\d+ +/, '')
|
||||
let b = inside.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < b.len() && b[i].is_ascii_digit() {
|
||||
i += 1;
|
||||
}
|
||||
let mut stripped = inside;
|
||||
if i > 0 && b.get(i) == Some(&b'/') {
|
||||
let mut j = i + 1;
|
||||
while j < b.len() && b[j].is_ascii_digit() {
|
||||
j += 1;
|
||||
}
|
||||
if j > i + 1 && b.get(j) == Some(&b' ') {
|
||||
while j < b.len() && b[j] == b' ' {
|
||||
j += 1;
|
||||
}
|
||||
stripped = &inside[j..];
|
||||
}
|
||||
}
|
||||
if has_extension(stripped) {
|
||||
return Some(stripped.to_string());
|
||||
}
|
||||
}
|
||||
let word = &before[before.rfind(' ').map_or(0, |i| i + 1)..];
|
||||
return has_extension(word).then(|| word.to_string());
|
||||
}
|
||||
if has_extension(text) {
|
||||
let bracket = text.rfind("] ").map_or(0, |i| i + 2);
|
||||
let dash = text.rfind(" - ").map_or(0, |i| i + 3);
|
||||
let name = js::trim(&text[bracket.max(dash)..]);
|
||||
return (!name.is_empty() && has_extension(name)).then(|| name.to_string());
|
||||
}
|
||||
text.split(' ').find(|w| has_extension(w)).map(str::to_string)
|
||||
}
|
||||
|
||||
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
|
||||
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
|
||||
let b = text.as_bytes();
|
||||
@@ -794,4 +857,27 @@ mod tests {
|
||||
assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["a1", "a2", "a3"]);
|
||||
assert!(agg.open.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod unquoted_tests {
|
||||
use super::unquoted_name;
|
||||
|
||||
#[test]
|
||||
fn file_names_of_subjects_without_a_quoted_one() {
|
||||
let cases = [
|
||||
("DVDFab 8.2.0.5.rar ", Some("DVDFab 8.2.0.5.rar")),
|
||||
("q3B9OrpKiCxUnzgtUGnp.part023.rar ", Some("q3B9OrpKiCxUnzgtUGnp.part023.rar")),
|
||||
("New_Super_Mario_Bros._[Keygen].exe ", Some("New_Super_Mario_Bros._[Keygen].exe")),
|
||||
("Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE ", Some("Emily_18_-_Bed.part01.rar")),
|
||||
("[33/117] 674a37548b09.dat ", Some("674a37548b09.dat")),
|
||||
("pal! [Mivina high(1).wmv\" yEnc] ", Some("Mivina high(1).wmv")),
|
||||
("[03/21 Geroellheimer - S03E16.part01.rar\" yEnc ", Some("Geroellheimer - S03E16.part01.rar")),
|
||||
("[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc ", None),
|
||||
("Re: Will somebody answer the question? ", None),
|
||||
];
|
||||
for (text, want) in cases {
|
||||
assert_eq!(unquoted_name(text).as_deref(), want, "{text}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -514,7 +514,8 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||
let post_total = file_counter_total(subject);
|
||||
let file_total = counter_total(subject).and_then(|t| t.parse::<i64>().ok());
|
||||
let stem = file_stem(name);
|
||||
let remainder = subject_remainder(subject);
|
||||
// A name that isn't quoted (unquoted_name) is taken out as the quotes would have been.
|
||||
let remainder = if subject.contains(&format!("\"{name}\"")) { subject_remainder(subject) } else { subject_remainder(&subject.replacen(name, " ", 1).replace('"', " ")) };
|
||||
// A document or comic is a release of its own.
|
||||
if test(&DOCUMENT, name) {
|
||||
let shown = replace(&DOCUMENT, &stem, "", false);
|
||||
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
import { fileKey } from '../headers/index.js'
|
||||
import { fileKey, unquotedName } from '../headers/index.js'
|
||||
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
||||
|
||||
/** A file's details, shared by all of its segments (one object per file, not per article). */
|
||||
@@ -112,8 +112,8 @@ export class SegmentScanner {
|
||||
/** A segment from an article's fields, or null when its subject is obfuscated. */
|
||||
segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null {
|
||||
if (!subject || !messageId) return null
|
||||
// After the last quote; without one, anywhere (unquotedName finds the file name).
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
if (nameEnd <= 0) return null
|
||||
const counter = findCounter(subject, nameEnd)
|
||||
if (!counter) return null
|
||||
const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length)
|
||||
@@ -137,10 +137,10 @@ export class SegmentScanner {
|
||||
|
||||
/** A file's details from its first article seen (as parseSubject and segmentFromHeader). */
|
||||
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
|
||||
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
|
||||
if (nameStart === -1) return null
|
||||
const name = own(subject.slice(nameStart + 1, nameEnd))
|
||||
if (!name || counter.total < 1) return null
|
||||
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
|
||||
const found = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index))
|
||||
if (!found || counter.total < 1) return null
|
||||
const name = own(found)
|
||||
const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
|
||||
const poster = own(from)
|
||||
return {
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
export { parseSubject, type ParsedSubject } from './subject.js'
|
||||
export { parseSubject, unquotedName, type ParsedSubject } from './subject.js'
|
||||
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
||||
export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js'
|
||||
+44
-5
@@ -16,6 +16,46 @@ export interface ParsedSubject {
|
||||
}
|
||||
|
||||
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
/** A file name's extension: a dot and 2 to 4 letters or digits at the end. */
|
||||
const EXTENSION = /[^.\s]\.[A-Za-z0-9]{2,4}$/
|
||||
|
||||
/**
|
||||
* The file name of a subject without a quoted one, from the text before its
|
||||
* segment counter (LOG.md 2026-10-11 (5): most "obfuscated" subjects of older
|
||||
* groups): with one quote only, the name before it (after a "[", else after
|
||||
* the last space: `[keira_s04_0037_l.jpg" yEnc] (2/2)`); without quotes, the
|
||||
* text when it ends in a file name (after a "] " or " - ": `DVDFab
|
||||
* 8.2.0.5.rar (006/332)`, `[33/117] 674a37548b09.dat (1/1)`), else its
|
||||
* first word that is one (`Emily_18_-_Bed.part01.rar Emily 18 bed video
|
||||
* (01/24)`). Null when no file name is found: the subject stays obfuscated.
|
||||
* Words are split on spaces only (both implementations alike).
|
||||
*/
|
||||
export function unquotedName(beforeCounter: string): string | null {
|
||||
let text = beforeCounter.trimEnd()
|
||||
if (/yenc$/i.test(text)) text = text.slice(0, -4).trimEnd()
|
||||
const quote = text.indexOf('"')
|
||||
if (quote !== -1) {
|
||||
if (text.indexOf('"', quote + 1) !== -1) return null
|
||||
const before = text.slice(0, quote)
|
||||
const bracket = before.lastIndexOf('[')
|
||||
if (bracket !== -1) {
|
||||
// A file counter that lost its closing bracket ("[03/21 Name.part01.rar") isn't part of the name.
|
||||
const inBracket = before.slice(bracket + 1).trim().replace(/^\d+\/\d+ +/, '')
|
||||
if (EXTENSION.test(inBracket)) return inBracket
|
||||
}
|
||||
const word = before.slice(before.lastIndexOf(' ') + 1)
|
||||
return EXTENSION.test(word) ? word : null
|
||||
}
|
||||
if (EXTENSION.test(text)) {
|
||||
const bracket = text.lastIndexOf('] ')
|
||||
const dash = text.lastIndexOf(' - ')
|
||||
const cut = Math.max(bracket === -1 ? 0 : bracket + 2, dash === -1 ? 0 : dash + 3)
|
||||
const name = text.slice(cut).trim()
|
||||
return name && EXTENSION.test(name) ? name : null
|
||||
}
|
||||
for (const word of text.split(' ')) if (EXTENSION.test(word)) return word
|
||||
return null
|
||||
}
|
||||
const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
|
||||
|
||||
/**
|
||||
@@ -28,16 +68,15 @@ const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
|
||||
export function parseSubject(subject: string): ParsedSubject | null {
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name) return null
|
||||
|
||||
// The segment counter is the last "(k/N)" after the filename; a file size may follow it.
|
||||
// The segment counter is the last "(k/N)" after the filename (anywhere without a quote); a file size may follow it.
|
||||
let counter: RegExpExecArray | undefined
|
||||
for (const match of subject.matchAll(SEGMENT_COUNTER)) {
|
||||
if (match.index > nameEnd) counter = match
|
||||
}
|
||||
if (!counter) return null
|
||||
const name = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index))
|
||||
if (!name) return null
|
||||
|
||||
const segment = Number.parseInt(counter[1], 10)
|
||||
const totalSegments = Number.parseInt(counter[2], 10)
|
||||
@@ -47,7 +86,7 @@ export function parseSubject(subject: string): ParsedSubject | null {
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
|
||||
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
|
||||
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart !== -1 ? nameStart : counter.index))
|
||||
return {
|
||||
name,
|
||||
segment,
|
||||
|
||||
@@ -197,8 +197,10 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
|
||||
}
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". A name that isn't
|
||||
// quoted (unquotedName: "DVDFab 8.2.0.5.rar (006/332)", "[keira_s04_0037_l.jpg" yEnc] (2/2)") is taken out
|
||||
// as the quotes would have been: otherwise every file of a set would name a release of its own.
|
||||
const remainder = subjectRemainder(input.subject.includes(`"${input.name}"`) ? input.subject : input.subject.replace(input.name, ' ').replace(/"/g, ' '))
|
||||
// A document or comic is a release of its own (collections post hundreds under one subject).
|
||||
if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
|
||||
const image = IMAGE.test(input.name)
|
||||
|
||||
@@ -41,6 +41,26 @@ test.group('subject / parseSubject', () => {
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (0/5)'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (6/5)'))
|
||||
assert.isNull(parseSubject('"" yEnc (1/5)'))
|
||||
// A counter but no file name: still obfuscated.
|
||||
assert.isNull(parseSubject('[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc (1/12432)'))
|
||||
assert.isNull(parseSubject('Re: Will somebody answer the question? (1/2)'))
|
||||
})
|
||||
|
||||
test('finds the file name of subjects without a quoted one (unquotedName)', ({ assert }) => {
|
||||
const cases: [string, string, number, number, number?][] = [
|
||||
['DVDFab 8.2.0.5.rar (006/332)', 'DVDFab 8.2.0.5.rar', 6, 332],
|
||||
['q3B9OrpKiCxUnzgtUGnp.part023.rar (104/137)', 'q3B9OrpKiCxUnzgtUGnp.part023.rar', 104, 137],
|
||||
['New_Super_Mario_Bros._[Keygen].exe (1/2)', 'New_Super_Mario_Bros._[Keygen].exe', 1, 2],
|
||||
['Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE (01/24)', 'Emily_18_-_Bed.part01.rar', 1, 24],
|
||||
['[33/117] 674a37548b09.dat (1/1)', '674a37548b09.dat', 1, 1, 33],
|
||||
["pal! [Mivina high(1).wmv\" yEnc] (323/349)", 'Mivina high(1).wmv', 323, 349],
|
||||
['[03/21 Geroellheimer - S03E16.part01.rar" yEnc (1/50)', 'Geroellheimer - S03E16.part01.rar', 1, 50],
|
||||
]
|
||||
for (const [subject, name, segment, total, fileIndex] of cases) {
|
||||
const parsed = parseSubject(subject)
|
||||
assert.include(parsed, { name, segment, totalSegments: total }, subject)
|
||||
if (fileIndex) assert.equal(parsed!.fileIndex, fileIndex)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in new issue
Block a user