diff --git a/edge/fixtures/fta/0000-210034490-210039489.gz b/edge/fixtures/fta/0000-210034490-210039489.gz new file mode 100644 index 0000000..526de44 Binary files /dev/null and b/edge/fixtures/fta/0000-210034490-210039489.gz differ diff --git a/edge/fixtures/fta/0001-210039490-210044489.gz b/edge/fixtures/fta/0001-210039490-210044489.gz new file mode 100644 index 0000000..6eed818 Binary files /dev/null and b/edge/fixtures/fta/0001-210039490-210044489.gz differ diff --git a/edge/fixtures/fta/0002-210044490-210049489.gz b/edge/fixtures/fta/0002-210044490-210049489.gz new file mode 100644 index 0000000..d83295a Binary files /dev/null and b/edge/fixtures/fta/0002-210044490-210049489.gz differ diff --git a/edge/fixtures/fta/0003-210049490-210054489.gz b/edge/fixtures/fta/0003-210049490-210054489.gz new file mode 100644 index 0000000..8621a33 Binary files /dev/null and b/edge/fixtures/fta/0003-210049490-210054489.gz differ diff --git a/edge/fixtures/fta/0004-210054490-210059489.gz b/edge/fixtures/fta/0004-210054490-210059489.gz new file mode 100644 index 0000000..1288cc7 Binary files /dev/null and b/edge/fixtures/fta/0004-210054490-210059489.gz differ diff --git a/edge/fixtures/fta/range.json b/edge/fixtures/fta/range.json new file mode 100644 index 0000000..389a9c2 --- /dev/null +++ b/edge/fixtures/fta/range.json @@ -0,0 +1 @@ +{"group":"alt.binaries.fta","start":210034490,"end":210059489} \ No newline at end of file diff --git a/edge/src/indexer.rs b/edge/src/indexer.rs index 40fd5a7..b035d4a 100644 --- a/edge/src/indexer.rs +++ b/edge/src/indexer.rs @@ -89,8 +89,8 @@ struct Counter { total: i64, } -/// The last "(k/N)" after `after`, scanning back from the end (findCounter). -fn find_counter(subject: &[u8], after: usize) -> Option { +/// The last "(k/N)" after `after` (anywhere when None: JS -1), scanning back from the end (findCounter). +fn find_counter(subject: &[u8], after: Option) -> Option { let mut open = subject.len(); loop { // lastIndexOf('(', open - 1): at most open - 1. @@ -98,7 +98,7 @@ fn find_counter(subject: &[u8], after: usize) -> Option { return None; } open = subject[..open].iter().rposition(|&c| c == b'(')?; - if open <= after { + if after.is_some_and(|a| open <= a) { return None; } let mut i = open + 1; @@ -188,10 +188,8 @@ impl Scanner { return None; } let sb = subject.as_bytes(); - let name_end = subject.rfind('"')?; - if name_end == 0 { - return None; - } + // After the last quote; without one, anywhere (unquoted_name finds the file name). + let name_end = subject.rfind('"'); let counter = find_counter(sb, name_end)?; self.key.clear(); self.key.push_str(from); @@ -223,9 +221,16 @@ impl Scanner { } } -fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: &str, group: &str) -> Option { - let name_start = subject[..name_end].rfind('"')?; - let name = &subject[name_start + 1..name_end]; +fn describe(subject: &str, name_end: Option, counter: &Counter, from: &str, xref: &str, group: &str) -> Option { + let quoted = match name_end { + Some(end) if end > 0 => subject[..end].rfind('"').map(|start| subject[start + 1..end].to_string()), + _ => None, + }; + let name = match quoted { + Some(name) => name, + None => unquoted_name(&subject[..counter.index])?, + }; + let name = name.as_str(); if name.is_empty() || counter.total < 1 { return None; } @@ -244,6 +249,64 @@ fn describe(subject: &str, name_end: usize, counter: &Counter, from: &str, xref: }) } +/// EXTENSION: /[^.\s]\.[A-Za-z0-9]{2,4}$/ +fn has_extension(s: &str) -> bool { + let Some(dot) = s.rfind('.') else { return false }; + let ext = &s.as_bytes()[dot + 1..]; + if !(2..=4).contains(&ext.len()) || !ext.iter().all(|c| c.is_ascii_alphanumeric()) { + return false; + } + s[..dot].chars().last().is_some_and(|c| c != '.' && !js::is_space(c)) +} + +/// unquotedName (lib/headers/subject.ts): the file name of a subject without a quoted one, from the text before its counter. +pub fn unquoted_name(before_counter: &str) -> Option { + let mut text = before_counter.trim_end_matches(js::is_space); + if text.len() >= 4 && text.is_char_boundary(text.len() - 4) && text[text.len() - 4..].eq_ignore_ascii_case("yenc") { + text = text[..text.len() - 4].trim_end_matches(js::is_space); + } + if let Some(quote) = text.find('"') { + if text[quote + 1..].contains('"') { + return None; + } + let before = &text[..quote]; + if let Some(bracket) = before.rfind('[') { + let inside = js::trim(&before[bracket + 1..]); + // replace(/^\d+\/\d+ +/, '') + let b = inside.as_bytes(); + let mut i = 0; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let mut stripped = inside; + if i > 0 && b.get(i) == Some(&b'/') { + let mut j = i + 1; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if j > i + 1 && b.get(j) == Some(&b' ') { + while j < b.len() && b[j] == b' ' { + j += 1; + } + stripped = &inside[j..]; + } + } + if has_extension(stripped) { + return Some(stripped.to_string()); + } + } + let word = &before[before.rfind(' ').map_or(0, |i| i + 1)..]; + return has_extension(word).then(|| word.to_string()); + } + if has_extension(text) { + let bracket = text.rfind("] ").map_or(0, |i| i + 2); + let dash = text.rfind(" - ").map_or(0, |i| i + 3); + let name = js::trim(&text[bracket.max(dash)..]); + return (!name.is_empty() && has_extension(name)).then(|| name.to_string()); + } + text.split(' ').find(|w| has_extension(w)).map(str::to_string) +} + /// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText). fn scan_overview_text(text: &str, mut line: impl FnMut(Option, &str, &str, &str, &str, &str, &str)) { let b = text.as_bytes(); @@ -794,4 +857,27 @@ mod tests { assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::>(), vec!["a1", "a2", "a3"]); assert!(agg.open.is_empty()); } + +#[cfg(test)] +mod unquoted_tests { + use super::unquoted_name; + + #[test] + fn file_names_of_subjects_without_a_quoted_one() { + let cases = [ + ("DVDFab 8.2.0.5.rar ", Some("DVDFab 8.2.0.5.rar")), + ("q3B9OrpKiCxUnzgtUGnp.part023.rar ", Some("q3B9OrpKiCxUnzgtUGnp.part023.rar")), + ("New_Super_Mario_Bros._[Keygen].exe ", Some("New_Super_Mario_Bros._[Keygen].exe")), + ("Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE ", Some("Emily_18_-_Bed.part01.rar")), + ("[33/117] 674a37548b09.dat ", Some("674a37548b09.dat")), + ("pal! [Mivina high(1).wmv\" yEnc] ", Some("Mivina high(1).wmv")), + ("[03/21 Geroellheimer - S03E16.part01.rar\" yEnc ", Some("Geroellheimer - S03E16.part01.rar")), + ("[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc ", None), + ("Re: Will somebody answer the question? ", None), + ]; + for (text, want) in cases { + assert_eq!(unquoted_name(text).as_deref(), want, "{text}"); + } + } +} } diff --git a/edge/src/names.rs b/edge/src/names.rs index db00429..f00d9d7 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -514,7 +514,8 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let post_total = file_counter_total(subject); let file_total = counter_total(subject).and_then(|t| t.parse::().ok()); let stem = file_stem(name); - let remainder = subject_remainder(subject); + // A name that isn't quoted (unquoted_name) is taken out as the quotes would have been. + let remainder = if subject.contains(&format!("\"{name}\"")) { subject_remainder(subject) } else { subject_remainder(&subject.replacen(name, " ", 1).replace('"', " ")) }; // A document or comic is a release of its own. if test(&DOCUMENT, name) { let shown = replace(&DOCUMENT, &stem, "", false); diff --git a/lib/edge/scan.ts b/lib/edge/scan.ts index 132c5ca..df4b96c 100644 --- a/lib/edge/scan.ts +++ b/lib/edge/scan.ts @@ -1,4 +1,4 @@ -import { fileKey } from '../headers/index.js' +import { fileKey, unquotedName } from '../headers/index.js' import { releaseIdentity, type ReleaseIdentity } from '../release/index.js' /** A file's details, shared by all of its segments (one object per file, not per article). */ @@ -112,8 +112,8 @@ export class SegmentScanner { /** A segment from an article's fields, or null when its subject is obfuscated. */ segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null { if (!subject || !messageId) return null + // After the last quote; without one, anywhere (unquotedName finds the file name). const nameEnd = subject.lastIndexOf('"') - if (nameEnd <= 0) return null const counter = findCounter(subject, nameEnd) if (!counter) return null const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length) @@ -137,10 +137,10 @@ export class SegmentScanner { /** A file's details from its first article seen (as parseSubject and segmentFromHeader). */ private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null { - const nameStart = subject.lastIndexOf('"', nameEnd - 1) - if (nameStart === -1) return null - const name = own(subject.slice(nameStart + 1, nameEnd)) - if (!name || counter.total < 1) return null + const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1 + const found = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index)) + if (!found || counter.total < 1) return null + const name = own(found) const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()) const poster = own(from) return { diff --git a/lib/headers/index.ts b/lib/headers/index.ts index 24639a6..1ce3a54 100644 --- a/lib/headers/index.ts +++ b/lib/headers/index.ts @@ -1,3 +1,3 @@ -export { parseSubject, type ParsedSubject } from './subject.js' +export { parseSubject, unquotedName, type ParsedSubject } from './subject.js' export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js' export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js' diff --git a/lib/headers/subject.ts b/lib/headers/subject.ts index 8304476..756fb4c 100644 --- a/lib/headers/subject.ts +++ b/lib/headers/subject.ts @@ -16,6 +16,46 @@ export interface ParsedSubject { } const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g +/** A file name's extension: a dot and 2 to 4 letters or digits at the end. */ +const EXTENSION = /[^.\s]\.[A-Za-z0-9]{2,4}$/ + +/** + * The file name of a subject without a quoted one, from the text before its + * segment counter (LOG.md 2026-10-11 (5): most "obfuscated" subjects of older + * groups): with one quote only, the name before it (after a "[", else after + * the last space: `[keira_s04_0037_l.jpg" yEnc] (2/2)`); without quotes, the + * text when it ends in a file name (after a "] " or " - ": `DVDFab + * 8.2.0.5.rar (006/332)`, `[33/117] 674a37548b09.dat (1/1)`), else its + * first word that is one (`Emily_18_-_Bed.part01.rar Emily 18 bed video + * (01/24)`). Null when no file name is found: the subject stays obfuscated. + * Words are split on spaces only (both implementations alike). + */ +export function unquotedName(beforeCounter: string): string | null { + let text = beforeCounter.trimEnd() + if (/yenc$/i.test(text)) text = text.slice(0, -4).trimEnd() + const quote = text.indexOf('"') + if (quote !== -1) { + if (text.indexOf('"', quote + 1) !== -1) return null + const before = text.slice(0, quote) + const bracket = before.lastIndexOf('[') + if (bracket !== -1) { + // A file counter that lost its closing bracket ("[03/21 Name.part01.rar") isn't part of the name. + const inBracket = before.slice(bracket + 1).trim().replace(/^\d+\/\d+ +/, '') + if (EXTENSION.test(inBracket)) return inBracket + } + const word = before.slice(before.lastIndexOf(' ') + 1) + return EXTENSION.test(word) ? word : null + } + if (EXTENSION.test(text)) { + const bracket = text.lastIndexOf('] ') + const dash = text.lastIndexOf(' - ') + const cut = Math.max(bracket === -1 ? 0 : bracket + 2, dash === -1 ? 0 : dash + 3) + const name = text.slice(cut).trim() + return name && EXTENSION.test(name) ? name : null + } + for (const word of text.split(' ')) if (EXTENSION.test(word)) return word + return null +} const FILE_COUNTER = /\[(\d+)\/(\d+)\]/ /** @@ -28,16 +68,15 @@ const FILE_COUNTER = /\[(\d+)\/(\d+)\]/ export function parseSubject(subject: string): ParsedSubject | null { const nameEnd = subject.lastIndexOf('"') const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1 - if (nameStart === -1) return null - const name = subject.slice(nameStart + 1, nameEnd) - if (!name) return null - // The segment counter is the last "(k/N)" after the filename; a file size may follow it. + // The segment counter is the last "(k/N)" after the filename (anywhere without a quote); a file size may follow it. let counter: RegExpExecArray | undefined for (const match of subject.matchAll(SEGMENT_COUNTER)) { if (match.index > nameEnd) counter = match } if (!counter) return null + const name = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : unquotedName(subject.slice(0, counter.index)) + if (!name) return null const segment = Number.parseInt(counter[1], 10) const totalSegments = Number.parseInt(counter[2], 10) @@ -47,7 +86,7 @@ export function parseSubject(subject: string): ParsedSubject | null { .replace(/\s+/g, ' ') .trim() - const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart)) + const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart !== -1 ? nameStart : counter.index)) return { name, segment, diff --git a/lib/release/identity.ts b/lib/release/identity.ts index b066694..ff4cb1d 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -197,8 +197,10 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { return { key: hash('weak', stem), name: stem, source: 'weak-stem' } } - // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". - const remainder = subjectRemainder(input.subject) + // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". A name that isn't + // quoted (unquotedName: "DVDFab 8.2.0.5.rar (006/332)", "[keira_s04_0037_l.jpg" yEnc] (2/2)") is taken out + // as the quotes would have been: otherwise every file of a set would name a release of its own. + const remainder = subjectRemainder(input.subject.includes(`"${input.name}"`) ? input.subject : input.subject.replace(input.name, ' ').replace(/"/g, ' ')) // A document or comic is a release of its own (collections post hundreds under one subject). if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' } const image = IMAGE.test(input.name) diff --git a/tests/unit/headers/subject.spec.ts b/tests/unit/headers/subject.spec.ts index 35b9b45..5880c7a 100644 --- a/tests/unit/headers/subject.spec.ts +++ b/tests/unit/headers/subject.spec.ts @@ -41,6 +41,26 @@ test.group('subject / parseSubject', () => { assert.isNull(parseSubject('"name.rar" yEnc (0/5)')) assert.isNull(parseSubject('"name.rar" yEnc (6/5)')) assert.isNull(parseSubject('"" yEnc (1/5)')) + // A counter but no file name: still obfuscated. + assert.isNull(parseSubject('[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc (1/12432)')) + assert.isNull(parseSubject('Re: Will somebody answer the question? (1/2)')) + }) + + test('finds the file name of subjects without a quoted one (unquotedName)', ({ assert }) => { + const cases: [string, string, number, number, number?][] = [ + ['DVDFab 8.2.0.5.rar (006/332)', 'DVDFab 8.2.0.5.rar', 6, 332], + ['q3B9OrpKiCxUnzgtUGnp.part023.rar (104/137)', 'q3B9OrpKiCxUnzgtUGnp.part023.rar', 104, 137], + ['New_Super_Mario_Bros._[Keygen].exe (1/2)', 'New_Super_Mario_Bros._[Keygen].exe', 1, 2], + ['Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE (01/24)', 'Emily_18_-_Bed.part01.rar', 1, 24], + ['[33/117] 674a37548b09.dat (1/1)', '674a37548b09.dat', 1, 1, 33], + ["pal! [Mivina high(1).wmv\" yEnc] (323/349)", 'Mivina high(1).wmv', 323, 349], + ['[03/21 Geroellheimer - S03E16.part01.rar" yEnc (1/50)', 'Geroellheimer - S03E16.part01.rar', 1, 50], + ] + for (const [subject, name, segment, total, fileIndex] of cases) { + const parsed = parseSubject(subject) + assert.include(parsed, { name, segment, totalSegments: total }, subject) + if (fileIndex) assert.equal(parsed!.fileIndex, fileIndex) + } }) })