Partition files by age (id)
New rows went into free space scattered through the 9.6 GB heap on the HDD (left behind by inspection's updates): one random HDD read per insert, and seed batches took a median 13 s to store. The existing table becomes the first partition as it is (a CHECK scan, no rewrite; indexes, its foreign key and the triggers are taken over by the partitioned parent); new partitions of 4M ids go in the same tablespace and stay in memory while they fill. - Every partition has a unique file_key index; FileStore.insert looks keys up in all partitions first (a key can't be unique across partitions). - FilePartitions.ensure() (gc:scheduler) keeps two partitions ahead. - Stats count files from the partitions; seed:node waits ~20 minutes for the API before giving a range up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
41859636d3
commit
f1773744c0
8 files changed
+310
-8
No files matched your search
@@ -209,6 +209,117 @@ export class FileSchema extends BaseModel {
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class FilesP0Schema extends BaseModel {
|
||||
static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
|
||||
$columns = FilesP0Schema.$columns
|
||||
@column()
|
||||
declare archive: any | null
|
||||
@column()
|
||||
declare contentType: string | null
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime | null
|
||||
@column()
|
||||
declare date: bigint | number
|
||||
@column()
|
||||
declare fileKey: string | null
|
||||
@column()
|
||||
declare filename: string
|
||||
@column()
|
||||
declare firstMessageId: string | null
|
||||
@column()
|
||||
declare groups: any | null
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
@column()
|
||||
declare parts: number
|
||||
@column()
|
||||
declare poster: string
|
||||
@column()
|
||||
declare realName: string | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare segments: Buffer
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class FilesP1000000Schema extends BaseModel {
|
||||
static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
|
||||
$columns = FilesP1000000Schema.$columns
|
||||
@column()
|
||||
declare archive: any | null
|
||||
@column()
|
||||
declare contentType: string | null
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime | null
|
||||
@column()
|
||||
declare date: bigint | number
|
||||
@column()
|
||||
declare fileKey: string | null
|
||||
@column()
|
||||
declare filename: string
|
||||
@column()
|
||||
declare firstMessageId: string | null
|
||||
@column()
|
||||
declare groups: any | null
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
@column()
|
||||
declare parts: number
|
||||
@column()
|
||||
declare poster: string
|
||||
@column()
|
||||
declare realName: string | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare segments: Buffer
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class FilesP5000000Schema extends BaseModel {
|
||||
static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
|
||||
$columns = FilesP5000000Schema.$columns
|
||||
@column()
|
||||
declare archive: any | null
|
||||
@column()
|
||||
declare contentType: string | null
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime | null
|
||||
@column()
|
||||
declare date: bigint | number
|
||||
@column()
|
||||
declare fileKey: string | null
|
||||
@column()
|
||||
declare filename: string
|
||||
@column()
|
||||
declare firstMessageId: string | null
|
||||
@column()
|
||||
declare groups: any | null
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
@column()
|
||||
declare parts: number
|
||||
@column()
|
||||
declare poster: string
|
||||
@column()
|
||||
declare realName: string | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare segments: Buffer
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class GroupSchema extends BaseModel {
|
||||
static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
|
||||
$columns = GroupSchema.$columns
|
||||
|
||||
Reference in new issue
Block a user