Keep binaries until inspected; seeding writes binaries only for inspection
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc861ce8ae
commit
b2cc75fdd0
11 files changed
+149
-35
No files matched your search
@@ -0,0 +1,20 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A completed binary whose file is queued for content inspection keeps its
|
||||
* row until inspection has stored the file's fingerprint on it (LOG.md
|
||||
* 2026-10-06 (11)): par2 naming (D16) matches par2 listings against binary
|
||||
* fingerprints and attaches sets to binaries. With the inspection queue
|
||||
* days behind, the GC deleted them first (after 6 hours unfingerprinted,
|
||||
* 48 hours stale). TRUE while waiting; cleared by inspection.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||||
await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS awaiting_inspection boolean`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS awaiting_inspection`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Indexes for the GC's deletes of finished binaries, without the ones
|
||||
* waiting for inspection (LOG.md 2026-10-06 (11)): those can be millions
|
||||
* when inspection is behind, and the deletes must not read past them every
|
||||
* sweep. Built CONCURRENTLY: binaries takes writes all the time.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_done_last_seen_idx ON binaries (last_seen_at)
|
||||
WHERE file_id IS NOT NULL AND awaiting_inspection IS NOT TRUE`
|
||||
)
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_awaiting_inspection_idx ON binaries (last_seen_at)
|
||||
WHERE awaiting_inspection IS TRUE`
|
||||
)
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_open_last_seen_idx ON binaries (last_seen_at)
|
||||
WHERE completed_at IS NULL OR conflicted_at IS NOT NULL`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_open_last_seen_idx`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_awaiting_inspection_idx`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_done_last_seen_idx`)
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -33,8 +33,10 @@ export class AuthAccessTokenSchema extends BaseModel {
|
||||
}
|
||||
|
||||
export class BinarySchema extends BaseModel {
|
||||
static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
|
||||
static $columns = ['awaitingInspection', 'completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
|
||||
$columns = BinarySchema.$columns
|
||||
@column()
|
||||
declare awaitingInspection: boolean | null
|
||||
@column.dateTime()
|
||||
declare completedAt: DateTime | null
|
||||
@column.dateTime()
|
||||
|
||||
Reference in new issue
Block a user