Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -4,6 +4,7 @@ import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import type { Knex } from 'knex'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NameService from '#services/NameService'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
import type { IngestSegment } from '#lib/ingest/segment'
|
||||
|
||||
// Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
|
||||
@@ -110,7 +111,14 @@ function validDate(date?: Date) {
|
||||
* no-op. See CONCEPT.md (D12).
|
||||
*/
|
||||
class IngestService {
|
||||
async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
|
||||
async ingest(batch: IngestSegment[]): Promise<CompletedBinary[]> {
|
||||
// Garbage isn't stored at all: no release, binary or parts (CONCEPT.md D42).
|
||||
const { kept: segments } = await GarbageFilter.keep(batch, (s) => ({
|
||||
name: s.name,
|
||||
messageId: s.messageId,
|
||||
groups: s.groups,
|
||||
releaseKey: s.release?.key ?? s.fileKey,
|
||||
}))
|
||||
if (segments.length === 0) return []
|
||||
|
||||
// Short statements, no enclosing transaction: every step is idempotent, and
|
||||
|
||||
Reference in new issue
Block a user