Content type and archive listing move from files to file_details
Every probe rewrote its file's row on the HDD (content_type, archive): a new row version on a random page and an entry in each of the five indexes. They now live in file_details on the NVMe; files rows are inserted once. files:move-details copies the existing values after the deploy; a later migration drops the old columns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1520a8dc35
commit
fad1716d3a
12 files changed
+138
-22
No files matched your search
@@ -37,10 +37,35 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
|
||||
export interface FileDetails {
|
||||
content_type?: string | null
|
||||
archive?: unknown
|
||||
}
|
||||
|
||||
const DETAIL_COLUMNS = ['content_type', 'archive'] as const
|
||||
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
|
||||
export const insertLimits = { rows: 2000 }
|
||||
|
||||
class FileStore {
|
||||
/** Sets some of a file's details, leaving the others as they are. */
|
||||
async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
|
||||
const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
|
||||
if (columns.length === 0) return
|
||||
const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
|
||||
await client.rawQuery(
|
||||
`INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
|
||||
ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
|
||||
[fileId, ...(values as any[])]
|
||||
)
|
||||
}
|
||||
|
||||
/** A file's details (empty when it has none). */
|
||||
async details(fileId: number): Promise<FileDetails> {
|
||||
return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts files, skipping those whose `file_key` exists. Returns the rows
|
||||
* that were inserted (id, filename, file_key). `files` is partitioned by id
|
||||
@@ -88,12 +113,24 @@ class FileStore {
|
||||
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
|
||||
})
|
||||
if (rows.length === 0) return []
|
||||
// Details given with a new file (e.g. a known content type) go to file_details.
|
||||
const details = rows.map((row: Record<string, unknown>) => {
|
||||
const d: FileDetails = {}
|
||||
for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
|
||||
for (const c of DETAIL_COLUMNS) delete row[c]
|
||||
return d
|
||||
})
|
||||
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
|
||||
const inserted: { id: number; filename: string; file_key: string | null }[] = []
|
||||
for (let i = 0; i < rows.length; i += insertLimits.rows) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
|
||||
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
|
||||
}
|
||||
const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
|
||||
for (const row of inserted) {
|
||||
const d = detailsByKey.get(row.file_key ?? row.filename)
|
||||
if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
|
||||
}
|
||||
return inserted
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user