Tiny-archive rule; the ladder's discard by index; the namer in chunks
The tiny-archive rule (dropTinyArchives, both indexers): a settled single archive of one article and 2-100 kB under a release-like name, spaces allowed, outside the exempt groups and without an e-book, subtitle, crack, cover or sidecar word: the 2015-17 exe-stub flood in miniature, 1.66M catalog rows (LOG.md (131), (133)). PendingLadder.discard scanned every waiting row against the dropped signatures and classes, 9+ minutes a sweep inside the claim, stalling every lane: now driven from the dropped classes and signatures into the indexes. The namer feeds the indexer the span's overview text chunk by chunk (one string of it topped V8's limit) and gives up spans over 3M articles. Migration 7000: a partial index on the taken probe candidates for the prober's start. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
4d46db291e
commit
aa4022295a
10 files changed
+267
-72
No files matched your search
+43
-1
@@ -47,6 +47,12 @@ export interface Rules {
|
||||
* the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)).
|
||||
*/
|
||||
dropNonsensePieces: boolean
|
||||
/**
|
||||
* Drop one-article archives of TINY_MIN_BYTES to NONSENSE_MIN_BYTES under a release-like name (spaces allowed),
|
||||
* outside the exempt groups, without an e-book, subtitle, crack or sidecar word: the 2015-17 exe-stub flood in
|
||||
* miniature (a RAR holding "<title>.exe" at 2-60 kB; 1.65M catalog rows; LOG.md (131), (133)).
|
||||
*/
|
||||
dropTinyArchives: boolean
|
||||
}
|
||||
|
||||
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||
@@ -58,6 +64,7 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
|
||||
dropStubArchives: true,
|
||||
dropNonsensePieces: true,
|
||||
dropTinyArchives: true,
|
||||
droppedClasses: [],
|
||||
}
|
||||
|
||||
@@ -69,8 +76,12 @@ export const STUB_WIDE_GROUPS = 4
|
||||
/** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */
|
||||
export const NONSENSE_MIN_BYTES = 100_000
|
||||
export const NONSENSE_MAX_BYTES = 1_000_000
|
||||
/** The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. */
|
||||
export const TINY_MIN_BYTES = 2_000
|
||||
|
||||
export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece'; key: string; sample: boolean }
|
||||
export type Verdict =
|
||||
| { rule: null }
|
||||
| { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive'; key: string; sample: boolean }
|
||||
|
||||
/** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */
|
||||
export interface JudgedFile {
|
||||
@@ -148,6 +159,36 @@ export function isNonsensePiece(f: JudgedFile): boolean {
|
||||
)
|
||||
}
|
||||
|
||||
/** RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. */
|
||||
const RELEASE_LIKE_SPACED = new RegExp(RELEASE_LIKE.source.replace('[._-]', '[._ -]'), 'i')
|
||||
|
||||
/**
|
||||
* Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
|
||||
* exe is the content), covers and sidecars.
|
||||
*/
|
||||
const TINY_CONTENT =
|
||||
/\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b/i
|
||||
|
||||
/**
|
||||
* A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
|
||||
* unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
|
||||
* subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
|
||||
*/
|
||||
export function isTinyArchive(f: JudgedFile): boolean {
|
||||
return (
|
||||
f.totalParts === 1 &&
|
||||
f.segments === 1 &&
|
||||
f.bytes >= TINY_MIN_BYTES &&
|
||||
f.bytes < NONSENSE_MIN_BYTES &&
|
||||
!f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g)) &&
|
||||
RELEASE_LIKE_SPACED.test(fileStem(f.name)) &&
|
||||
!EBOOK.test(f.name) &&
|
||||
!TINY_CONTENT.test(f.name) &&
|
||||
!isScrambledFile(f.name) &&
|
||||
!isRandomFile(f.name)
|
||||
)
|
||||
}
|
||||
|
||||
export function isRandomFile(name: string) {
|
||||
return looksRandom(fileStem(name))
|
||||
}
|
||||
@@ -221,6 +262,7 @@ export class RuleSet {
|
||||
if (legs !== '') return { rule: 'stub-archive', key: legs, sample }
|
||||
}
|
||||
if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample }
|
||||
if (this.rules.dropTinyArchives && isTinyArchive(archive)) return { rule: 'tiny-archive', key: uploadTool(archive.firstMessageId), sample }
|
||||
}
|
||||
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
|
||||
const first = files[0]
|
||||
|
||||
Reference in new issue
Block a user