Seeding node: gzip overviews, batched SFV lookups, fragments beside fetching
Test and deploy / test (push) Successful in 1m0s
Test and deploy / deploy (push) Successful in 1m43s

- lib/nntp/xover_client: a minimal one-connection client for seed:range
  with XFEATURE COMPRESS GZIP (a zlib stream after "224 … [COMPRESS=GZIP]",
  then a "." line; 420 for an empty range). About 6x less header traffic.
  Overviews come out with the field names nntp-js uses.
- SfvService.claimAll: one sfv_entries lookup per stored batch.
- Fragments go through the regular ingest path beside fetching (at most
  two batches in flight); a failure fails the run at the end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-05 11:11:41 +02:00
1 parent 68eb9a44fd
commit 97bdf8e982
5 files changed
+329 -26

No files matched your search

+7 -8
View File
@@ -89,15 +89,14 @@ class SeedService {
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
const candidates: { name: string; data: { fileId: number }; opts: { jobId: string } }[] = []
for (const file of inserted) {
const added = inserted.flatMap((file) => {
const row = rowById.get(file.id)
if (!row) continue
await SfvService.claim({ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id })
if (ContentService.isCandidate(row.filename)) {
candidates.push({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } })
}
}
return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
})
await SfvService.claimAll(added)
const candidates = added
.filter((file) => ContentService.isCandidate(file.name))
.map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
result.inspections = candidates.length
return result
+15
View File
@@ -52,6 +52,21 @@ class SfvService {
}
/** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
/**
* claim() for a batch of new files: one lookup for all their names, the
* per-file work only for the few an SFV lists (seeding stores thousands
* of files per batch; CONCEPT.md D37).
*/
async claimAll(files: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }[]): Promise<number> {
if (files.length === 0) return 0
const listed = new Set(
(await db.from('sfv_entries').whereIn('name', [...new Set(files.map((f) => f.name))]).select('name')).map((e) => e.name as string)
)
let moved = 0
for (const file of files) if (listed.has(file.name) && (await this.claim(file)) !== null) moved++
return moved
}
async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
const entries = await db.from('sfv_entries').where('name', file.name)
const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))