Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -25,6 +25,7 @@ import SevenZipService from '#services/SevenZipService'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/**
|
||||
* MD5 of the first 16 KiB of a file, as par2 records it, from the body of
|
||||
@@ -49,6 +50,8 @@ export interface ContentResult {
|
||||
nicovideo?: string
|
||||
/** The real name from the yEnc header, for a file posted under a random subject (D36). */
|
||||
yencName?: string
|
||||
/** Why the file's release was deleted as garbage after this probe (D42). */
|
||||
garbage?: string
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,6 +132,8 @@ class ContentService {
|
||||
async prepare(fileId: number): Promise<InspectTask | null> {
|
||||
const file = await db.from('files').where('id', fileId).first()
|
||||
if (!file || !this.isCandidate(file.filename)) return null
|
||||
// Stored before its signature was found to be garbage (D42): not worth an article.
|
||||
if (await GarbageFilter.skipProbe(file)) return null
|
||||
if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
if (!volumes || volumes[0].archive) return null
|
||||
@@ -148,8 +153,17 @@ class ContentService {
|
||||
}
|
||||
}
|
||||
|
||||
/** The database half after a probe: names, groupings, listings and flags from what it found. */
|
||||
/**
|
||||
* The database half after a probe: names, groupings, listings and flags from
|
||||
* what it found. Then the release is deleted if it turned out to be garbage (D42).
|
||||
*/
|
||||
async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
const result = await this.applyProbe(task, probe)
|
||||
result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
|
||||
return result
|
||||
}
|
||||
|
||||
private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
const result: ContentResult = { named: 0, contentType: probe.contentType }
|
||||
const file = await db.from('files').where('id', task.fileId).first()
|
||||
if (!file) return result
|
||||
@@ -277,6 +291,7 @@ export function describeContent(filename: string, content: ContentResult | null)
|
||||
if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
|
||||
if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
|
||||
if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
|
||||
if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage}, no name or password).`)
|
||||
if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
|
||||
if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
|
||||
if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
|
||||
|
||||
Reference in new issue
Block a user