Seeding node: gzip overviews, batched SFV lookups, fragments beside fetching
- lib/nntp/xover_client: a minimal one-connection client for seed:range with XFEATURE COMPRESS GZIP (a zlib stream after "224 … [COMPRESS=GZIP]", then a "." line; 420 for an empty range). About 6x less header traffic. Overviews come out with the field names nntp-js uses. - SfvService.claimAll: one sfv_entries lookup per stored batch. - Fragments go through the regular ingest path beside fetching (at most two batches in flight); a failure fails the run at the end. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
68eb9a44fd
commit
97bdf8e982
5 files changed
+329
-26
No files matched your search
@@ -89,15 +89,14 @@ class SeedService {
|
||||
|
||||
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
|
||||
const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
|
||||
const candidates: { name: string; data: { fileId: number }; opts: { jobId: string } }[] = []
|
||||
for (const file of inserted) {
|
||||
const added = inserted.flatMap((file) => {
|
||||
const row = rowById.get(file.id)
|
||||
if (!row) continue
|
||||
await SfvService.claim({ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id })
|
||||
if (ContentService.isCandidate(row.filename)) {
|
||||
candidates.push({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } })
|
||||
}
|
||||
}
|
||||
return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
|
||||
})
|
||||
await SfvService.claimAll(added)
|
||||
const candidates = added
|
||||
.filter((file) => ContentService.isCandidate(file.name))
|
||||
.map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
|
||||
if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
|
||||
result.inspections = candidates.length
|
||||
return result
|
||||
|
||||
@@ -52,6 +52,21 @@ class SfvService {
|
||||
}
|
||||
|
||||
/** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
|
||||
/**
|
||||
* claim() for a batch of new files: one lookup for all their names, the
|
||||
* per-file work only for the few an SFV lists (seeding stores thousands
|
||||
* of files per batch; CONCEPT.md D37).
|
||||
*/
|
||||
async claimAll(files: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }[]): Promise<number> {
|
||||
if (files.length === 0) return 0
|
||||
const listed = new Set(
|
||||
(await db.from('sfv_entries').whereIn('name', [...new Set(files.map((f) => f.name))]).select('name')).map((e) => e.name as string)
|
||||
)
|
||||
let moved = 0
|
||||
for (const file of files) if (listed.has(file.name) && (await this.claim(file)) !== null) moved++
|
||||
return moved
|
||||
}
|
||||
|
||||
async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
|
||||
const entries = await db.from('sfv_entries').where('name', file.name)
|
||||
const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
|
||||
|
||||
Reference in new issue
Block a user