Drop garbage uploads early (D42)
Random-named uploads that never get a real name or password were ~90% of what seeding stores. Two filters: - before storing (seeding, ingest): random-named uploads of a dropped signature (posting tool from the Message-ID, first group) or of a release dropped after its probe aren't stored; 2% of a dropped signature's releases are still kept and probed. Stored files of dropped signatures aren't probed. - after a probe: a random-named release that is still unnamed, without password, names from files or spot, with all its probes done, and encrypted, a container or missing its first article, is deleted and tombstoned in dropped_releases. Signatures are learned by gc:scheduler from a sample of probed releases (signature_probes); garbage:learn --seed fills it from existing files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d0b2da65
commit
211820067a
13 files changed
+683
-6
No files matched your search
@@ -5,6 +5,7 @@ import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/** Articles from a range's edge within which a file may continue in the neighbouring range. */
|
||||
const EDGE_ARTICLES = 50_000
|
||||
@@ -15,6 +16,8 @@ export interface SeedStoreResult {
|
||||
existing: number
|
||||
releases: number
|
||||
inspections: number
|
||||
/** Random-named files of garbage signatures or dropped releases, not stored (D42). */
|
||||
dropped: number
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -26,9 +29,17 @@ export interface SeedStoreResult {
|
||||
*/
|
||||
class SeedService {
|
||||
async store(batch: SeedFile[]): Promise<SeedStoreResult> {
|
||||
const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 }
|
||||
const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0, dropped: 0 }
|
||||
// A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
|
||||
const files = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
|
||||
const unique = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
|
||||
// Garbage isn't stored at all: no release, file or binary (CONCEPT.md D42).
|
||||
const { kept: files, dropped } = await GarbageFilter.keep(unique, (file) => ({
|
||||
name: file.name,
|
||||
messageId: file.segments[0]?.[1],
|
||||
groups: file.groups,
|
||||
releaseKey: file.release?.key ?? file.fileKey,
|
||||
}))
|
||||
result.dropped = dropped
|
||||
if (files.length === 0) return result
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
|
||||
Reference in new issue
Block a user