diff --git a/app/services/SeedService.ts b/app/services/SeedService.ts index 2733c67..546e9fe 100644 --- a/app/services/SeedService.ts +++ b/app/services/SeedService.ts @@ -58,9 +58,16 @@ class SeedService { result.files = inserted.length result.existing = rows.length - inserted.length + // File ids for the binaries below (files queued for inspection only): new ones from the insert, + // existing ones looked up. Looking up every key of the batch probed all files partitions again, + // 15% of Postgres' active time (LOG.md 2026-10-06 (12)). const fileIds = new Map() - for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) { - fileIds.set(row.file_key, Number(row.id)) + for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id) + const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey) + for (let i = 0; i < missing.length; i += 5000) { + for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) { + fileIds.set(row.file_key, Number(row.id)) + } } // A completed binary, as worker:file would have left behind, only for files queued for content // inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others @@ -97,9 +104,9 @@ class SeedService { } // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection. - const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row])) + const rowByKey = new Map(rows.map((row) => [row.file_key, row])) const added = inserted.flatMap((file) => { - const row = rowById.get(file.id) + const row = file.file_key ? rowByKey.get(file.file_key) : undefined return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : [] }) await SfvService.claimAll(added) diff --git a/commands/GcScheduler.ts b/commands/GcScheduler.ts index e6d741f..0dca953 100644 --- a/commands/GcScheduler.ts +++ b/commands/GcScheduler.ts @@ -34,6 +34,8 @@ export default class GcScheduler extends BaseCommand { /** When release completeness was last recomputed (it runs less often than the sweep). */ private completenessAt = 0 + /** When split uploads were last merged. */ + private mergedAt = 0 public async run() { this.logger.info( @@ -115,9 +117,13 @@ export default class GcScheduler extends BaseCommand { ) } - // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep. - const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun }) - if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`) + // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep. It reads + // every release and its files: hourly, not every sweep (LOG.md 2026-10-06 (12)). + if (this.once || Date.now() - this.mergedAt >= gcConfig.mergeIntervalMs) { + this.mergedAt = Date.now() + const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun }) + if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`) + } await this.rebuildIndexes() } diff --git a/config/gc.ts b/config/gc.ts index eb17ca3..98d1530 100644 --- a/config/gc.ts +++ b/config/gc.ts @@ -10,6 +10,9 @@ export default { /** How often gc:scheduler recomputes the completeness of releases that changed (LOG.md 2026-10-06 (6)). */ releaseCompleteIntervalMs: env.get('RELEASE_COMPLETE_INTERVAL_MS', 30 * 60 * 1000), + /** How often gc:scheduler merges uploads split per poster (D35); it reads every release. */ + mergeIntervalMs: env.get('GC_MERGE_INTERVAL_MS', 60 * 60 * 1000), + /** Time per sweep for copying old file keys into file_keys until it's complete (LOG.md 2026-10-06 (7)). */ fileKeysFillMs: env.get('FILE_KEYS_FILL_SECONDS', 90) * 1000, diff --git a/start/env.ts b/start/env.ts index 9b45a99..7969341 100644 --- a/start/env.ts +++ b/start/env.ts @@ -106,6 +106,7 @@ export default await Env.create(new URL('../', import.meta.url), { RELEASE_COMPLETE_INTERVAL_MS: Env.schema.number.optional(), FILE_KEYS_FILL_SECONDS: Env.schema.number.optional(), GC_AWAIT_INSPECTION_SECONDS: Env.schema.number.optional(), + GC_MERGE_INTERVAL_MS: Env.schema.number.optional(), ARCHIVE_WALK_MAX_SEGMENTS: Env.schema.number.optional(), DEFERRED_SETTLE_MS: Env.schema.number.optional(), DEFERRED_EXPAND_MAX_PARTS: Env.schema.number.optional(),