daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Content type and archive listing move from files to file_details
Test and deploy / test (push) Successful in 1m26s
Details
Test and deploy / deploy (push) Successful in 1m54s
Details

Browse Source 
Every probe rewrote its file's row on the HDD (content_type, archive):
a new row version on a random page and an entry in each of the five
indexes. They now live in file_details on the NVMe; files rows are
inserted once. files:move-details copies the existing values after
the deploy; a later migration drops the old columns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-08 19:36:36 +02:00
1 parent 1520a8dc35
commit fad1716d3a
12 files changed
+138 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

commands/ContentBackfill.ts
+7 -7
View File
@@ -48,20 +48,20 @@ export default class ContentBackfill extends BaseCommand {
try {
while (!this.limit || queued < this.limit) {
// Files never inspected: no fingerprint on their binary and no archive listing.
const query = db.from('files as f').where('f.id', '>', lastId)
const query = db.from('files as f').leftJoin('file_details as d', 'd.file_id', 'f.id').where('f.id', '>', lastId)
// Files from NZBs (spots) have no binary.
if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
else query.join('binaries as b', 'b.file_id', 'f.id')
if (this.nicovideo) {
query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
query.whereNull('d.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
} else if (this.sfv) {
query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
query.whereNull('d.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
} else if (this.archives) {
query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
query.whereNull('d.content_type').whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
} else if (this.sevenZip) {
query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
} else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
query.whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
} else if (this.archiveNzbs) query.whereRaw(`d.archive::text ~* '\\.nzb"'`)
else if (!this.all) query.whereNull('b.md5_16k').whereNull('d.archive')
const files = await query
.orderBy('f.id')
.limit(BATCH)