Content type and archive listing move from files to file_details
Test and deploy / test (push) Successful in 1m26s
Test and deploy / deploy (push) Successful in 1m54s

Every probe rewrote its file's row on the HDD (content_type, archive):
a new row version on a random page and an entry in each of the five
indexes. They now live in file_details on the NVMe; files rows are
inserted once. files:move-details copies the existing values after
the deploy; a later migration drops the old columns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-08 19:36:36 +02:00
1 parent 1520a8dc35
commit fad1716d3a
12 files changed
+138 -22

No files matched your search

+37
View File
@@ -37,10 +37,35 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
return stored ? unpackSegments(stored) : []
}
/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
export interface FileDetails {
content_type?: string | null
archive?: unknown
}
const DETAIL_COLUMNS = ['content_type', 'archive'] as const
/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
export const insertLimits = { rows: 2000 }
class FileStore {
/** Sets some of a file's details, leaving the others as they are. */
async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
if (columns.length === 0) return
const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
await client.rawQuery(
`INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
[fileId, ...(values as any[])]
)
}
/** A file's details (empty when it has none). */
async details(fileId: number): Promise<FileDetails> {
return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
}
/**
* Inserts files, skipping those whose `file_key` exists. Returns the rows
* that were inserted (id, filename, file_key). `files` is partitioned by id
@@ -88,12 +113,24 @@ class FileStore {
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
})
if (rows.length === 0) return []
// Details given with a new file (e.g. a known content type) go to file_details.
const details = rows.map((row: Record<string, unknown>) => {
const d: FileDetails = {}
for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
for (const c of DETAIL_COLUMNS) delete row[c]
return d
})
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
const inserted: { id: number; filename: string; file_key: string | null }[] = []
for (let i = 0; i < rows.length; i += insertLimits.rows) {
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
}
const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
for (const row of inserted) {
const d = detailsByKey.get(row.file_key ?? row.filename)
if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
}
return inserted
}
}