Drop garbage uploads early (D42)
Test and deploy / test (push) Successful in 1m0s
Test and deploy / deploy (push) Successful in 2m33s

Random-named uploads that never get a real name or password were ~90% of
what seeding stores. Two filters:

- before storing (seeding, ingest): random-named uploads of a dropped
  signature (posting tool from the Message-ID, first group) or of a
  release dropped after its probe aren't stored; 2% of a dropped
  signature's releases are still kept and probed. Stored files of dropped
  signatures aren't probed.
- after a probe: a random-named release that is still unnamed, without
  password, names from files or spot, with all its probes done, and
  encrypted, a container or missing its first article, is deleted and
  tombstoned in dropped_releases.

Signatures are learned by gc:scheduler from a sample of probed releases
(signature_probes); garbage:learn --seed fills it from existing files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-08 11:21:01 +02:00
1 parent 35d0b2da65
commit 211820067a
13 files changed
+683 -6

No files matched your search

+16 -1
View File
@@ -25,6 +25,7 @@ import SevenZipService from '#services/SevenZipService'
import SfvService from '#services/SfvService'
import ArchiveWalkService from '#services/ArchiveWalkService'
import { segmentsOf } from '#services/FileStore'
import GarbageFilter from '#services/GarbageFilter'
/**
* MD5 of the first 16 KiB of a file, as par2 records it, from the body of
@@ -49,6 +50,8 @@ export interface ContentResult {
nicovideo?: string
/** The real name from the yEnc header, for a file posted under a random subject (D36). */
yencName?: string
/** Why the file's release was deleted as garbage after this probe (D42). */
garbage?: string
}
/**
@@ -129,6 +132,8 @@ class ContentService {
async prepare(fileId: number): Promise<InspectTask | null> {
const file = await db.from('files').where('id', fileId).first()
if (!file || !this.isCandidate(file.filename)) return null
// Stored before its signature was found to be garbage (D42): not worth an article.
if (await GarbageFilter.skipProbe(file)) return null
if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
const volumes = await SevenZipService.volumes(file)
if (!volumes || volumes[0].archive) return null
@@ -148,8 +153,17 @@ class ContentService {
}
}
/** The database half after a probe: names, groupings, listings and flags from what it found. */
/**
* The database half after a probe: names, groupings, listings and flags from
* what it found. Then the release is deleted if it turned out to be garbage (D42).
*/
async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
const result = await this.applyProbe(task, probe)
result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
return result
}
private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
const result: ContentResult = { named: 0, contentType: probe.contentType }
const file = await db.from('files').where('id', task.fileId).first()
if (!file) return result
@@ -277,6 +291,7 @@ export function describeContent(filename: string, content: ContentResult | null)
if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage}, no name or password).`)
if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)