Rebuild: fresh AdonisJS 7 API app on Postgres
The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
699395c51f
commit
96f0ade10e
264 files changed
+777
-24132
No files matched your search
@@ -1,21 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'groups'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id')
|
||||
table.string('name').notNullable().unique()
|
||||
table.boolean('active').notNullable().defaultTo(true)
|
||||
table.bigInteger('last_indexed_id').defaultTo(0)
|
||||
|
||||
table.timestamp('created_at', { useTz: true })
|
||||
table.timestamp('updated_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'files'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('filename').notNullable()
|
||||
table.string('poster').notNullable()
|
||||
table.bigInteger('date').notNullable()
|
||||
table.integer('parts').notNullable()
|
||||
table.jsonb('message_ids').notNullable()
|
||||
table.jsonb('groups')
|
||||
|
||||
table.timestamp('created_at', { useTz: true })
|
||||
table.timestamp('updated_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Files that are still being collected from XOVER. One row per uploaded file,
|
||||
* identified by file_key (see lib/subject/file_key.ts).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'binaries'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('file_key', 40).notNullable().unique()
|
||||
table.text('poster').notNullable()
|
||||
table.text('subject').notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.integer('total_parts').notNullable()
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable().index()
|
||||
table.timestamp('completed_at', { useTz: true })
|
||||
// Set when two different articles claim the same part number (see IngestService).
|
||||
table.timestamp('conflicted_at', { useTz: true })
|
||||
table.integer('file_id').unsigned().references('files.id').onDelete('SET NULL')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Segments (articles) of a binary. The primary key makes re-ingesting the
|
||||
* same range a no-op.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'parts'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE')
|
||||
table.integer('number').notNullable()
|
||||
table.text('message_id').notNullable()
|
||||
table.integer('bytes').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.primary(['binary_id', 'number'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'files'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.string('file_key', 40).unique()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.dropColumn('file_key')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release is one download: the files of one post (archive volumes, par2
|
||||
* set, nfo). See lib/release/release_key.ts and CONCEPT.md (D16).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'releases'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('release_key', 40).notNullable().unique()
|
||||
table.text('name').notNullable()
|
||||
// Null for obfuscated posts, whose poster is random per article.
|
||||
table.text('poster')
|
||||
// Number of files announced by the [i/M] subject counter, if any.
|
||||
table.integer('file_total')
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable()
|
||||
})
|
||||
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('release_id'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('release_id'))
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Par2-based grouping of obfuscated posts (CONCEPT.md D16): the par2 file
|
||||
* list of a release, and the exact file size of body-derived binaries to
|
||||
* match against it.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
// Exact file size from =ybegin; only known for body-derived binaries.
|
||||
table.bigInteger('size').index()
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// How the release was recognised: subject, stem, weak-stem or par2.
|
||||
table.string('source', 16)
|
||||
})
|
||||
this.schema.createTable('release_files', (table) => {
|
||||
table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
|
||||
table.string('par2_file_id', 32).notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.bigInteger('size').notNullable().index()
|
||||
table.string('md5_16k', 32).notNullable()
|
||||
table.primary(['release_id', 'par2_file_id'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('release_files')
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('source'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('size'))
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Article ranges handed to worker:fetch, so a range only counts as indexed
|
||||
* once it was fetched successfully (CONCEPT.md D18).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('groups', (table) => {
|
||||
// How far the scheduler has queued; last_indexed_id becomes the
|
||||
// watermark below which every article was fetched.
|
||||
table.bigInteger('last_queued_id').notNullable().defaultTo(0)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('UPDATE groups SET last_queued_id = last_indexed_id')
|
||||
})
|
||||
|
||||
this.schema.createTable('fetch_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.integer('group_id').notNullable().references('groups.id').onDelete('CASCADE')
|
||||
table.bigInteger('start_id').notNullable()
|
||||
table.bigInteger('end_id').notNullable()
|
||||
// queued | done | failed
|
||||
table.string('status', 8).notNullable().defaultTo('queued')
|
||||
table.integer('requeues').notNullable().defaultTo(0)
|
||||
table.text('error')
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('updated_at', { useTz: true }).notNullable()
|
||||
table.index(['group_id', 'status'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('fetch_ranges')
|
||||
this.schema.alterTable('groups', (table) => table.dropColumn('last_queued_id'))
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Archive listings from the first RAR volume (CONCEPT.md D19). */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
// RarInfo of a first volume: format, flags and entries.
|
||||
table.jsonb('archive')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// Password protected: encrypted file data or encrypted headers.
|
||||
table.boolean('encrypted')
|
||||
// Entries of the release's first archive volume.
|
||||
table.jsonb('contents')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('contents')
|
||||
table.dropColumn('encrypted')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('archive'))
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Splitting equal-sized obfuscated volumes (CONCEPT.md D20): the extra
|
||||
* articles of conflicted binaries are kept, and the split-off volumes point
|
||||
* to the conflicted binary they came from.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('part_conflicts', (table) => {
|
||||
table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE')
|
||||
table.integer('number').notNullable()
|
||||
table.text('message_id').notNullable()
|
||||
table.integer('bytes').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.primary(['binary_id', 'number', 'message_id'])
|
||||
})
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.bigInteger('parent_binary_id').references('binaries.id').onDelete('SET NULL').index()
|
||||
// Last time the splitter processed this conflicted binary.
|
||||
table.timestamp('split_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.dropColumn('split_at')
|
||||
table.dropColumn('parent_binary_id')
|
||||
})
|
||||
this.schema.dropTable('part_conflicts')
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Full-file MD5 from par2, to verify assembled (e.g. split) volumes. */
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'release_files'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.string('md5', 32)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => table.dropColumn('md5'))
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Binaries that were flagged conflicted before part_conflicts existed lost
|
||||
* their extra articles, so splitting them only reshuffles the first article
|
||||
* per part number, a mix of different volumes. Mark them as never to be
|
||||
* split, and remove the volumes and files the first splitter run made from
|
||||
* them (see LOG.md, 2026-09-28).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
WITH boundary AS (
|
||||
SELECT migration_time FROM adonis_schema WHERE name LIKE '%1768620764790%'
|
||||
), legacy AS (
|
||||
SELECT id FROM binaries
|
||||
WHERE conflicted_at IS NOT NULL AND parent_binary_id IS NULL
|
||||
AND conflicted_at < (SELECT migration_time FROM boundary)
|
||||
), instances AS (
|
||||
SELECT id, file_id FROM binaries WHERE parent_binary_id IN (SELECT id FROM legacy)
|
||||
), deleted_files AS (
|
||||
DELETE FROM files WHERE id IN (SELECT file_id FROM instances WHERE file_id IS NOT NULL)
|
||||
), deleted_instances AS (
|
||||
DELETE FROM binaries WHERE id IN (SELECT id FROM instances)
|
||||
)
|
||||
-- 'infinity': last_seen_at > split_at is never true, so SplitService skips them.
|
||||
UPDATE binaries SET split_at = 'infinity' WHERE id IN (SELECT id FROM legacy)
|
||||
`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
// The removed volumes were unverifiable; nothing to restore.
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* yEnc CRCs of body-derived articles, so split volumes can be verified
|
||||
* without downloading them (CONCEPT.md D20): the combined part CRCs of a
|
||||
* volume must equal the whole-file crc32 in its last part.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
table.string('pcrc32', 8)
|
||||
// Decoded length of the part (=ypart end - begin + 1).
|
||||
table.integer('part_size')
|
||||
// Whole-file CRC32, when the poster included it (usually in the last part).
|
||||
table.string('file_crc32', 8)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
table.dropColumn('pcrc32')
|
||||
table.dropColumn('part_size')
|
||||
table.dropColumn('file_crc32')
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Names and groupings from file contents (CONCEPT.md D21): a fingerprint
|
||||
* (exact size + MD5 of the first 16 KiB, as par2 records it) and the real
|
||||
* filename recovered from par2.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.string('md5_16k', 32).index()
|
||||
// Filename recovered from file contents (par2); `name` stays as posted.
|
||||
table.text('real_name')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.text('real_name')
|
||||
})
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
// Only for part 1: MD5 of the first 16 KiB of the file.
|
||||
table.string('md5_16k', 32)
|
||||
})
|
||||
}
|
||||
this.schema.alterTable('release_files', (table) => {
|
||||
table.index(['md5_16k'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('release_files', (table) => table.dropIndex(['md5_16k']))
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => table.dropColumn('md5_16k'))
|
||||
}
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('real_name'))
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.dropColumn('real_name')
|
||||
table.dropColumn('md5_16k')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Web interface (CONCEPT.md D22): trigram indexes for substring search, and
|
||||
* SQL versions of the "readable name" rule used for filtering.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('CREATE EXTENSION IF NOT EXISTS pg_trgm')
|
||||
// Per-file suffixes stripped, as in lib/release fileStem.
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(name,
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
// Not an obfuscated token (as in ArchiveService.looksRandom).
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '\\d'
|
||||
AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]')
|
||||
OR name_stem(name) ~* '^[0-9a-f]{16,}$')
|
||||
$$`)
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_posted_at_idx ON releases (posted_at DESC NULLS LAST)')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_posted_at_idx')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_real_name_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_filename_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_name_trgm')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS is_readable_name(text)')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS name_stem(text)')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Per-release totals for browsing and sorting (CONCEPT.md D22). A trigger on
|
||||
* `files` keeps `releases.file_count` and `releases.size` current, whichever
|
||||
* code path assigns files to releases.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.bigInteger('size')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.integer('file_count').notNullable().defaultTo(0).index()
|
||||
table.bigInteger('size').notNullable().defaultTo(0).index()
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
UPDATE files f SET size = s.size
|
||||
FROM (SELECT id, (SELECT coalesce(sum((v->>'size')::bigint), 0) FROM jsonb_each(message_ids) e(k, v)) AS size FROM files) s
|
||||
WHERE s.id = f.id`)
|
||||
await db.rawQuery(`
|
||||
UPDATE releases r SET file_count = t.n, size = t.size
|
||||
FROM (SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS size FROM files
|
||||
WHERE release_id IS NOT NULL GROUP BY release_id) t
|
||||
WHERE t.release_id = r.id`)
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION files_release_totals() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN
|
||||
UPDATE releases SET file_count = file_count - 1, size = size - coalesce(OLD.size, 0) WHERE id = OLD.release_id;
|
||||
END IF;
|
||||
IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN
|
||||
UPDATE releases SET file_count = file_count + 1, size = size + coalesce(NEW.size, 0) WHERE id = NEW.release_id;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER files_release_totals
|
||||
AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS files_release_totals ON files')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS files_release_totals()')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('size')
|
||||
table.dropColumn('file_count')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('size'))
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Stored completeness for the "Complete" filter (CONCEPT.md D22); computing
|
||||
* it per request took ~30 s on 34k releases. Triggers keep it current when
|
||||
* files, par2 lists, sizes or announced totals change.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS binaries_file_id_idx ON binaries (file_id)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops)')
|
||||
await db.rawQuery('ALTER TABLE releases ADD COLUMN complete boolean NOT NULL DEFAULT false')
|
||||
await db.rawQuery('CREATE INDEX releases_complete_idx ON releases (complete)')
|
||||
|
||||
// All files announced by [i/M] are there, or every par2-listed file
|
||||
// (>= 64 KiB) is present by exact size or real name.
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean STABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = r.id AND rf.size >= 65536
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name)))
|
||||
ELSE false END
|
||||
FROM releases r WHERE r.id = rid), false)
|
||||
$$`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases SET complete = release_complete(rid) WHERE id = rid AND complete IS DISTINCT FROM release_complete(rid)
|
||||
$$`)
|
||||
|
||||
// files: after the totals trigger has updated file_count (trigger names sort alphabetically).
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN PERFORM refresh_release_complete(OLD.release_id); END IF;
|
||||
IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN PERFORM refresh_release_complete(NEW.release_id); END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER files_zz_release_complete
|
||||
AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(NEW.release_id);
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete
|
||||
AFTER INSERT ON release_files FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
|
||||
|
||||
// An exact size from content inspection can make a par2-listed file "present".
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION binaries_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.file_id IS NOT NULL THEN
|
||||
PERFORM refresh_release_complete(f.release_id) FROM files f WHERE f.id = NEW.file_id AND f.release_id IS NOT NULL;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER binaries_release_complete
|
||||
AFTER UPDATE OF size ON binaries FOR EACH ROW
|
||||
WHEN (OLD.size IS DISTINCT FROM NEW.size) EXECUTE FUNCTION binaries_release_complete()`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION releases_file_total_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(NEW.id);
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER releases_file_total_complete
|
||||
AFTER UPDATE OF file_total ON releases FOR EACH ROW
|
||||
WHEN (OLD.file_total IS DISTINCT FROM NEW.file_total) EXECUTE FUNCTION releases_file_total_complete()`)
|
||||
|
||||
await db.rawQuery('UPDATE releases SET complete = release_complete(id) WHERE file_count > 0')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS releases_file_total_complete ON releases')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS binaries_release_complete ON binaries')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS release_files_release_complete ON release_files')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS files_zz_release_complete ON files')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS releases_file_total_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS binaries_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_files_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS files_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS refresh_release_complete(bigint)')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_complete(bigint)')
|
||||
await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS complete')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_contents_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS binaries_file_id_idx')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* par2 set id per listed file (traceability; see LOG.md 2026-09-28 for the
|
||||
* cross-post merges it helps diagnose), and a stricter "random name" rule:
|
||||
* long letter-only tokens with many capitals count as random too
|
||||
* (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('release_files', (table) => {
|
||||
table.string('set_id', 32)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]'
|
||||
AND (name_stem(name) ~ '\\d' OR length(regexp_replace(name_stem(name), '[^A-Z]', '', 'g')) >= 6))
|
||||
OR name_stem(name) ~* '^[0-9a-f]{16,}$')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('release_files', (table) => table.dropColumn('set_id'))
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Newznab category per release (CONCEPT.md D23), derived from the name and
|
||||
* the archive contents. A stored generated column, so it follows renames
|
||||
* and newly listed contents without any application code.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text, contents jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- TV: S01E02, 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[._ -])(S\\d{1,2}E\\d{1,3}|\\d{1,2}x\\d{2})([._ -]|$)' OR name ~* '[._ -]S\\d{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)\\d{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\\.?26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: lossless or mp3 by contents, otherwise scene music names (-WEB-2026-GROUP).
|
||||
WHEN coalesce(contents::text, '') ~* '\\.flac"' OR name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN coalesce(contents::text, '') ~* '\\.mp3"' OR name ~* '(^|[._ -])mp3([._ -]|$)'
|
||||
OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\\d{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN coalesce(contents::text, '') ~* '\\.(epub|mobi|azw3)"' OR name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020
|
||||
WHEN coalesce(contents::text, '') ~* '\\.(exe|msi|dmg)"' THEN 4000
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
await db.rawQuery(
|
||||
'ALTER TABLE releases ADD COLUMN category integer GENERATED ALWAYS AS (release_category(name, contents)) STORED'
|
||||
)
|
||||
await db.rawQuery('CREATE INDEX releases_category_idx ON releases (category)')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS category')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_category(text, jsonb)')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A second article with the same part CRC is a repost of the same data, not
|
||||
* a different file (CONCEPT.md D20). Drop such recorded conflicts and unflag
|
||||
* binaries whose only conflicts were reposts. Binaries flagged before CRCs
|
||||
* were stored (split_at = 'infinity') cannot be checked and stay flagged.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
DELETE FROM part_conflicts c USING parts p
|
||||
WHERE p.binary_id = c.binary_id AND p.number = c.number
|
||||
AND p.pcrc32 IS NOT NULL AND c.pcrc32 IS NOT NULL AND p.pcrc32 = c.pcrc32`)
|
||||
// Split volumes made from reposts are copies of their parent's file.
|
||||
await db.rawQuery(`
|
||||
DELETE FROM binaries v USING binaries b
|
||||
WHERE v.parent_binary_id = b.id AND v.completed_at IS NULL
|
||||
AND b.conflicted_at IS NOT NULL AND b.split_at IS DISTINCT FROM 'infinity'
|
||||
AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`)
|
||||
await db.rawQuery(`
|
||||
UPDATE binaries b SET conflicted_at = NULL
|
||||
WHERE b.conflicted_at IS NOT NULL AND b.parent_binary_id IS NULL
|
||||
AND b.split_at IS DISTINCT FROM 'infinity'
|
||||
AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Body downloads per group (CONCEPT.md D24). Identifying obfuscated articles
|
||||
* costs one body download each; busy groups carry far more of them than the
|
||||
* line can fetch, so they are indexed header-only (normal subjects) unless
|
||||
* enabled.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'groups'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.boolean('fetch_bodies').notNullable().defaultTo(false)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`UPDATE groups SET fetch_bodies = true WHERE name = 'alt.binaries.test'`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => table.dropColumn('fetch_bodies'))
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* is_readable_name(): also judge the stem without an unknown short
|
||||
* extension, so `326678d1d07ef12a.bin` counts as random (CONCEPT.md D22).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
|
||||
AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
|
||||
OR stem ~* '^[0-9a-f]{16,}$')
|
||||
FROM s
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* File type by magic bytes (lib/content), and releases that are only an
|
||||
* encrypted container (CONCEPT.md D25).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.string('content_type', 16).index()
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// e.g. 'nzbe': an encrypted NZB, useless without its key.
|
||||
table.string('container', 16)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('container'))
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('content_type'))
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* NZBs posted to Usenet are read into releases (CONCEPT.md D26): the archive
|
||||
* password they may carry, and a lookup of files by their first segment.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// From <meta type="password"> of a posted NZB; passed on in our NZBs.
|
||||
table.text('password')
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_first_segment_idx')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('password'))
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** name_stem(): also strip subtitle extensions, as lib/release fileStem does. */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(name,
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* name_stem(): strip a 3-digit split suffix only after a file extension
|
||||
* (x.mkv.001), not codec names like H.264 / H.265, matching lib/release.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(
|
||||
regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'),
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** name_stem(): same extension list as lib/release fileStem (adds bin, m3u, cue, log, bz2). */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(
|
||||
regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'),
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|srr|srs|txt|md5|sha1|tar|zst|gz|xz|bz2|mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Files of uploaders whose bodies are not fetched right away (ngPost, see
|
||||
* CONCEPT.md D28): one row per file, from the headers alone. The row keeps
|
||||
* the article range, so the file can be found again, and is the queue for
|
||||
* worker:deferred, which fetches one body per file.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('deferred_files', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('file_key', 40).notNullable().unique()
|
||||
// File name from the subject (random for ngPost).
|
||||
table.text('name').notNullable()
|
||||
table.text('subject').notNullable()
|
||||
table.text('poster').notNullable()
|
||||
// Article numbers are counted in the group where the file was first seen.
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('first_article').notNullable()
|
||||
table.text('first_message_id').notNullable()
|
||||
table.bigInteger('last_article').notNullable()
|
||||
table.integer('article_count').notNullable()
|
||||
table.bigInteger('bytes').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable()
|
||||
// pending → sampled (one body read) or expanded (sent to the body queue) or failed.
|
||||
table.string('status', 16).notNullable().defaultTo('pending')
|
||||
table.timestamp('processed_at', { useTz: true })
|
||||
// From the sampled body.
|
||||
table.text('yenc_name')
|
||||
table.bigInteger('size')
|
||||
table.integer('total_parts')
|
||||
table.integer('sampled_part')
|
||||
table.string('content_type', 16)
|
||||
table.boolean('encrypted')
|
||||
table.text('error')
|
||||
table.index(['status', 'last_seen_at'])
|
||||
table.index(['first_seen_at'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('deferred_files')
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Deferred articles whose headers don't say which file they belong to
|
||||
* (ngPost with random subjects and a new poster every few articles, see
|
||||
* CONCEPT.md D28). Only the article range per fetch batch is kept;
|
||||
* `deferred:expand-ranges` finds the articles again and queues their bodies.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('deferred_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('first_article').notNullable()
|
||||
table.bigInteger('last_article').notNullable()
|
||||
table.integer('article_count').notNullable()
|
||||
table.bigInteger('bytes').notNullable()
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('expanded_at', { useTz: true })
|
||||
table.index(['expanded_at', 'id'])
|
||||
table.index(['created_at'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('deferred_ranges')
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Spotnet spots (CONCEPT.md D29): signed posts in free.pt that point to a
|
||||
* compressed NZB. Filled from headers by worker:fetch, then worker:spots
|
||||
* verifies each spot, reads its NZB and imports it as a release.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('spots', (table) => {
|
||||
table.text('message_id').primary()
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.text('title').notNullable()
|
||||
table.text('tag')
|
||||
table.text('poster').notNullable()
|
||||
table.integer('category')
|
||||
table.jsonb('subcats')
|
||||
table.bigInteger('size')
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.integer('key_id')
|
||||
// Spotweb's spotter id (CRC of the poster's key), to check personal disposes.
|
||||
table.text('spotter_id')
|
||||
// pending → imported, or unverified / no-nzb / moderation / removed / failed.
|
||||
table.string('status', 16).notNullable().defaultTo('pending')
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL')
|
||||
table.text('website')
|
||||
table.text('error')
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('processed_at', { useTz: true })
|
||||
table.index(['status', 'created_at'])
|
||||
table.index(['release_id'])
|
||||
table.index(['status', 'release_id'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('spots')
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Smaller indexes (LOG.md 2026-09-28 (17)). 99.5% of releases have no files
|
||||
* yet (most never will), and search only looks at releases with files; most
|
||||
* binaries have no size, fingerprint, file or parent. Indexing only the rows
|
||||
* that queries can match saves GBs and work on every insert. Built
|
||||
* concurrently, so ingest keeps running.
|
||||
*/
|
||||
const INDEXES: [table: string, name: string, definition: string][] = [
|
||||
['releases', 'releases_name_trgm', `USING gin (name gin_trgm_ops) WHERE file_count > 0`],
|
||||
['releases', 'releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops) WHERE file_count > 0`],
|
||||
['releases', 'releases_posted_at_idx', `(posted_at DESC NULLS LAST) WHERE file_count > 0`],
|
||||
['releases', 'releases_size_index', `(size) WHERE file_count > 0`],
|
||||
['releases', 'releases_file_count_index', `(file_count) WHERE file_count > 0`],
|
||||
['releases', 'releases_complete_idx', `(complete) WHERE file_count > 0`],
|
||||
['releases', 'releases_category_idx', `(category) WHERE file_count > 0`],
|
||||
['binaries', 'binaries_md5_16k_index', `(md5_16k) WHERE md5_16k IS NOT NULL`],
|
||||
['binaries', 'binaries_size_index', `(size) WHERE size IS NOT NULL`],
|
||||
['binaries', 'binaries_file_id_idx', `(file_id) WHERE file_id IS NOT NULL`],
|
||||
['binaries', 'binaries_parent_binary_id_index', `(parent_binary_id) WHERE parent_binary_id IS NOT NULL`],
|
||||
]
|
||||
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
for (const [table, name, definition] of INDEXES) {
|
||||
// Build the partial index under a temporary name, then swap it in.
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_partial`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_partial ON ${table} ${definition}`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`ALTER INDEX ${name}_partial RENAME TO ${name}`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const [table, name, definition] of INDEXES) {
|
||||
const full = definition.replace(/ WHERE .*$/, '')
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name} ON ${table} ${full}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Spot listings (CONCEPT.md D29): old spots become a release without files
|
||||
* (source `spot`), whose NZB is fetched when first downloaded. Search shows
|
||||
* releases with files or spot listings, so the partial release indexes
|
||||
* cover both. Spots are verified from their overview when fetched.
|
||||
*/
|
||||
const VISIBLE = `(file_count > 0 OR source = 'spot')`
|
||||
const INDEXES: [name: string, definition: string][] = [
|
||||
['releases_name_trgm', `USING gin (name gin_trgm_ops)`],
|
||||
['releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops)`],
|
||||
['releases_posted_at_idx', `(posted_at DESC NULLS LAST)`],
|
||||
['releases_size_index', `(size)`],
|
||||
['releases_file_count_index', `(file_count)`],
|
||||
['releases_complete_idx', `(complete)`],
|
||||
['releases_category_idx', `(category)`],
|
||||
]
|
||||
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
|
||||
// worker:spots claims recent spots first, then the history in posting order.
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
|
||||
await this.swap(VISIBLE)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.swap('file_count > 0')
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_article_idx`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_posted_idx`)
|
||||
await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
|
||||
}
|
||||
|
||||
private async swap(predicate: string) {
|
||||
for (const [name, definition] of INDEXES) {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_next`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_next ON releases ${definition} WHERE ${predicate}`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`ALTER INDEX ${name}_next RENAME TO ${name}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Spot details (CONCEPT.md D29): the description and image from the spot's
|
||||
* full header. Stored whenever the header is read (recent spots, moderation,
|
||||
* downloads); for other old spots on first view. `details_at` marks the read.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('spots', (table) => {
|
||||
table.text('description')
|
||||
table.jsonb('image')
|
||||
table.timestamp('details_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('spots', (table) => {
|
||||
table.dropColumn('description')
|
||||
table.dropColumn('image')
|
||||
table.dropColumn('details_at')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Where a release's archive password came from: `nzb` (the NZB's
|
||||
* `<meta type="password">`) or `spot` (the spot's title or description).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('releases', (table) => table.string('password_source', 8))
|
||||
this.schema.raw(`UPDATE releases SET password_source = 'nzb' WHERE password IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('password_source'))
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
import { packSegments, unpackSegments } from '#lib/nzb/segments'
|
||||
|
||||
const BATCH = 5000
|
||||
|
||||
/**
|
||||
* Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
|
||||
* ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
|
||||
* plus `first_message_id` for the lookups the old expression index served.
|
||||
* Run with the workers stopped; reclaim the space with VACUUM FULL files.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
|
||||
let last = 0
|
||||
let done = 0
|
||||
while (true) {
|
||||
const { rows } = await this.db.rawQuery(
|
||||
`SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
|
||||
[last, BATCH]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
const ids: number[] = []
|
||||
const packed: Buffer[] = []
|
||||
const firsts: (string | null)[] = []
|
||||
for (const row of rows) {
|
||||
const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
|
||||
.map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
|
||||
.sort((a, b) => a.number - b.number)
|
||||
ids.push(Number(row.id))
|
||||
packed.push(packSegments(segments))
|
||||
firsts.push(segments[0]?.id ?? null)
|
||||
}
|
||||
await this.db.rawQuery(
|
||||
`UPDATE files f SET segments = v.segments, first_message_id = v.first
|
||||
FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
|
||||
[ids, packed, firsts] as any
|
||||
)
|
||||
last = ids[ids.length - 1]
|
||||
done += rows.length
|
||||
if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
|
||||
}
|
||||
await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
|
||||
await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
|
||||
const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
|
||||
for (const row of rows) {
|
||||
const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
|
||||
await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
|
||||
}
|
||||
await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
|
||||
await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
|
||||
await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
|
||||
await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Header backfill (CONCEPT.md D32): a group walks backwards from
|
||||
* `backfill_next_id` down to `backfill_until_id` in low-priority,
|
||||
* header-only fetch ranges (`fetch_ranges.backfill`).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('groups', (table) => {
|
||||
table.bigInteger('backfill_until_id')
|
||||
table.bigInteger('backfill_next_id')
|
||||
})
|
||||
this.schema.alterTable('fetch_ranges', (table) => {
|
||||
table.boolean('backfill').notNullable().defaultTo(false)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('groups', (table) => {
|
||||
table.dropColumn('backfill_until_id')
|
||||
table.dropColumn('backfill_next_id')
|
||||
})
|
||||
this.schema.alterTable('fetch_ranges', (table) => table.dropColumn('backfill'))
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
const VISIBLE = `(file_count > 0 OR source = 'spot')`
|
||||
const BATCH = 20_000
|
||||
|
||||
/**
|
||||
* Search and index cleanup (CONCEPT.md D33).
|
||||
* - `releases.search_text`: name, archive listing and readable file-name
|
||||
* stems, normalised (lowercase; dots, dashes, underscores and spaces as
|
||||
* one space), kept by triggers; one trigram index replaces the four the
|
||||
* old per-table OR search could hardly use.
|
||||
* - Spot status indexes become partial indexes over the few rows the worker
|
||||
* and search look for.
|
||||
* - Unused release indexes go (size: never scanned; complete: a boolean).
|
||||
* Runs online: indexes are built and dropped concurrently.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
|
||||
// For IndexMaintenance; needs a superuser (it is one in the compose setup).
|
||||
await q(`CREATE EXTENSION IF NOT EXISTS pgstattuple`).catch((error) => console.warn(`pgstattuple not installed: ${error.message}`))
|
||||
|
||||
// Spots: the worker only looks for pending/processing and un-detailed imported spots, search for hidden ones.
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_todo_idx ON spots (article_number) WHERE status IN ('pending', 'processing')`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_hidden_release_idx ON spots (release_id) WHERE status IN ('removed', 'dummy')`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_details_todo_idx ON spots (posted_at DESC) WHERE status = 'imported' AND details_at IS NULL`)
|
||||
for (const index of ['spots_status_posted_idx', 'spots_status_article_idx', 'spots_status_created_at_index', 'spots_status_release_id_index']) {
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
|
||||
}
|
||||
for (const index of ['releases_size_index', 'releases_complete_idx']) {
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
|
||||
}
|
||||
|
||||
// Search text.
|
||||
await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS search_files text, ADD COLUMN IF NOT EXISTS search_text text`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION search_normalize(t text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT btrim(lower(regexp_replace(coalesce(t, ''), '[._[:space:]-]+', ' ', 'g')))
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION release_search_text(name text, contents jsonb, files text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT left(concat_ws(' ',
|
||||
search_normalize(name),
|
||||
nullif(search_normalize((SELECT string_agg(e->>'name', ' ') FROM (
|
||||
SELECT e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) e LIMIT 200) x)), ''),
|
||||
nullif(files, '')), 8000)
|
||||
$$`)
|
||||
// Visible releases only; computed when one becomes visible or its name, listing or file names change.
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION releases_search_text() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT (NEW.file_count > 0 OR NEW.source = 'spot') THEN
|
||||
NEW.search_text := NULL;
|
||||
ELSIF TG_OP = 'INSERT' OR NEW.search_text IS NULL
|
||||
OR NEW.name IS DISTINCT FROM OLD.name OR NEW.contents IS DISTINCT FROM OLD.contents
|
||||
OR NEW.search_files IS DISTINCT FROM OLD.search_files THEN
|
||||
NEW.search_text := release_search_text(NEW.name, NEW.contents, NEW.search_files);
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
|
||||
await q(`CREATE TRIGGER releases_search_text BEFORE INSERT OR UPDATE ON releases FOR EACH ROW EXECUTE FUNCTION releases_search_text()`)
|
||||
// A file's readable stem joins its release's search text when it adds something.
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_search() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
n text := coalesce(NEW.real_name, NEW.filename);
|
||||
s text;
|
||||
BEGIN
|
||||
IF NEW.release_id IS NULL OR NOT is_readable_name(n) THEN RETURN NULL; END IF;
|
||||
s := search_normalize(name_stem(n));
|
||||
IF s = '' THEN RETURN NULL; END IF;
|
||||
UPDATE releases SET search_files = left(concat_ws(' ', search_files, s), 4000)
|
||||
WHERE id = NEW.release_id AND position(s IN coalesce(search_text, search_normalize(name))) = 0
|
||||
AND length(coalesce(search_files, '')) < 4000;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
|
||||
// Fill: readable file stems per release (one pass over files), then every visible release in batches.
|
||||
await q(`DROP TABLE IF EXISTS search_stems_tmp`)
|
||||
await q(`
|
||||
CREATE UNLOGGED TABLE search_stems_tmp AS
|
||||
SELECT release_id, left(string_agg(DISTINCT s, ' '), 4000) AS stems FROM (
|
||||
SELECT f.release_id, search_normalize(name_stem(coalesce(f.real_name, f.filename))) AS s
|
||||
FROM files f WHERE f.release_id IS NOT NULL AND is_readable_name(coalesce(f.real_name, f.filename))) x
|
||||
WHERE s <> '' GROUP BY release_id`)
|
||||
await q(`CREATE UNIQUE INDEX ON search_stems_tmp (release_id)`)
|
||||
let last = 0
|
||||
let done = 0
|
||||
while (true) {
|
||||
const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
|
||||
if (rows.length === 0) break
|
||||
const to = Number(rows[rows.length - 1].id)
|
||||
await q(
|
||||
`UPDATE releases r SET search_files = (
|
||||
SELECT nullif(array_to_string(array(
|
||||
SELECT w FROM unnest(string_to_array(t.stems, ' ')) w
|
||||
WHERE position(w IN search_normalize(r.name)) = 0), ' '), '')
|
||||
FROM search_stems_tmp t WHERE t.release_id = r.id),
|
||||
search_text = NULL
|
||||
WHERE r.id > ? AND r.id <= ? AND ${VISIBLE.replaceAll('file_count', 'r.file_count').replace('source', 'r.source')}`,
|
||||
[last, to]
|
||||
)
|
||||
last = to
|
||||
done += rows.length
|
||||
if (done % 200_000 < BATCH) console.log(`Search text for ${done} releases.`)
|
||||
}
|
||||
await q(`DROP TABLE search_stems_tmp`)
|
||||
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_search_trgm ON releases USING gin (search_text gin_trgm_ops) WHERE ${VISIBLE}`)
|
||||
for (const index of ['releases_name_trgm', 'releases_contents_trgm', 'files_filename_trgm', 'files_real_name_trgm']) {
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops) WHERE ${VISIBLE}`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops) WHERE ${VISIBLE}`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)`)
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_search_trgm`)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
|
||||
await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
|
||||
await q(`DROP FUNCTION IF EXISTS files_release_search()`)
|
||||
await q(`DROP FUNCTION IF EXISTS releases_search_text()`)
|
||||
await q(`DROP FUNCTION IF EXISTS release_search_text(text, jsonb, text)`)
|
||||
await q(`DROP FUNCTION IF EXISTS search_normalize(text)`)
|
||||
await q(`ALTER TABLE releases DROP COLUMN IF EXISTS search_files, DROP COLUMN IF EXISTS search_text`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_size_index ON releases (size) WHERE ${VISIBLE}`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_complete_idx ON releases (complete) WHERE ${VISIBLE}`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_created_at_index ON spots (status, created_at)`)
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_release_id_index ON spots (status, release_id)`)
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_todo_idx`)
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_hidden_release_idx`)
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_details_todo_idx`)
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* is_readable_name(): also hex with a Unix time, UUIDs, digit-only names
|
||||
* (and rclone's "83111763 n"), lowercase base32/36 tokens and short tokens
|
||||
* with capitals all over. The same rules as looksRandom() in ArchiveService.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
|
||||
AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
|
||||
OR stem ~* '^[0-9a-f]{16,}$'
|
||||
OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
|
||||
OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||
OR stem ~ '^\\d{6,}( n){0,1}$'
|
||||
OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
|
||||
AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
|
||||
OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
|
||||
AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3))
|
||||
FROM s
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
|
||||
AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
|
||||
OR stem ~* '^[0-9a-f]{16,}$')
|
||||
FROM s
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* SFV listings (CONCEPT.md D34): the files an SFV names belong to its
|
||||
* release; files arriving later are matched by name.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('sfv_entries', (table) => {
|
||||
table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
|
||||
table.text('name').notNullable()
|
||||
table.string('crc', 8).notNullable()
|
||||
table.text('poster')
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.primary(['release_id', 'name'])
|
||||
table.index(['name'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('sfv_entries')
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** SFV grouping (CONCEPT.md D34) looks files up by their name. */
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
// On the NVMe (default tablespace), like the files table's other indexes.
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_name_idx ON files ((coalesce(real_name, filename)))`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_name_idx`)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* is_readable_name(): generic folder/file names (VIDEO_TS, Sample, CD1) are
|
||||
* no name either. Same list as GENERIC_NAMES in lib/release/release_key.ts.
|
||||
*/
|
||||
const FUNCTION = (generic: boolean) => `
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
|
||||
AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
|
||||
OR stem ~* '^[0-9a-f]{16,}$'
|
||||
OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
|
||||
OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||
OR stem ~ '^\\d{6,}( n){0,1}$'
|
||||
OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
|
||||
AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
|
||||
OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
|
||||
AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3)
|
||||
${generic ? `OR btrim(stem) ~* '^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples{0,1}|subs{0,1}|subtitles{0,1}|proof|cover|covers|extras{0,1}|featurettes{0,1}|(cd|dvd|disc|disk)[ ._-]{0,1}\\d{1,2})$'` : ''})
|
||||
FROM s
|
||||
$$`
|
||||
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(FUNCTION(true))
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(FUNCTION(false))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Staged segments (CONCEPT.md D35): a header segment whose file is new and
|
||||
* alone in its batch waits here until a second segment of the file shows
|
||||
* up; ~97% never do (uploads with a random subject per article). Unlogged:
|
||||
* no WAL, emptied by a crash, which only costs those waiting segments.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`
|
||||
CREATE UNLOGGED TABLE staged_segments (
|
||||
file_key text PRIMARY KEY,
|
||||
segments jsonb NOT NULL,
|
||||
staged_at timestamptz NOT NULL
|
||||
)`)
|
||||
// Rows arrive in time order, so a BRIN index serves the expiry cheaply.
|
||||
await this.db.rawQuery(`CREATE INDEX staged_segments_staged_at_brin ON staged_segments USING brin (staged_at)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TABLE staged_segments`)
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Counter grouping (CONCEPT.md D36): staged segments also carry the key of
|
||||
* the file they'd belong to by poster, file counter `[i/M]`, part count and
|
||||
* posting day, so uploads with a random subject per article become files.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(`ALTER TABLE staged_segments ADD COLUMN IF NOT EXISTS counter_key text`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS staged_segments_counter_key_idx ON staged_segments (counter_key) WHERE counter_key IS NOT NULL`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS staged_segments_counter_key_idx`)
|
||||
await this.db.rawQuery(`ALTER TABLE staged_segments DROP COLUMN IF EXISTS counter_key`)
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
const VISIBLE = `(file_count > 0 OR source = 'spot')`
|
||||
const BATCH = 20_000
|
||||
|
||||
/**
|
||||
* `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from
|
||||
* its files' names and its archive listing, for the search's extension
|
||||
* filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names
|
||||
* enter it as stems without extension, and matching the long, TOASTed text
|
||||
* took seconds per search.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
|
||||
|
||||
await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`)
|
||||
// Two to four characters with a letter (mkv, flac, 7z, par2); split
|
||||
// archive volumes count as their archive type (.7z.001, .r00).
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$')
|
||||
WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar'
|
||||
ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$')
|
||||
END
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT array(SELECT DISTINCT e FROM (
|
||||
SELECT name_extension(c->>'name') AS e
|
||||
FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x
|
||||
WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e)
|
||||
$$`)
|
||||
|
||||
// A new archive listing adds its entries' types.
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN
|
||||
NEW.extensions := nullif(array(
|
||||
SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}');
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
|
||||
await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`)
|
||||
|
||||
// A file adds its type to its release (once: most files of a release share it).
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
e text := name_extension(coalesce(NEW.real_name, NEW.filename));
|
||||
BEGIN
|
||||
IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF;
|
||||
UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x)
|
||||
WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]);
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
|
||||
// Fill: file types per release in one sequential pass over files (random
|
||||
// reads per release would take over an hour on the HDD), then the
|
||||
// visible releases in batches by id, adding their listing's types.
|
||||
await q(`DROP TABLE IF EXISTS file_extensions_tmp`)
|
||||
await q(`
|
||||
CREATE UNLOGGED TABLE file_extensions_tmp AS
|
||||
SELECT release_id, array_agg(DISTINCT e) AS exts FROM (
|
||||
SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x
|
||||
WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`)
|
||||
await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`)
|
||||
let last = 0
|
||||
let done = 0
|
||||
while (true) {
|
||||
const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
|
||||
if (rows.length === 0) break
|
||||
const to = Number(rows[rows.length - 1].id)
|
||||
await q(
|
||||
`UPDATE releases r SET extensions = x.exts FROM (
|
||||
SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts
|
||||
FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id
|
||||
WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x
|
||||
WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`,
|
||||
[last, to]
|
||||
)
|
||||
last = to
|
||||
done += rows.length
|
||||
if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`)
|
||||
}
|
||||
await q(`DROP TABLE file_extensions_tmp`)
|
||||
|
||||
await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
|
||||
await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
|
||||
await q(`DROP FUNCTION IF EXISTS files_release_extension()`)
|
||||
await q(`DROP FUNCTION IF EXISTS releases_extensions()`)
|
||||
await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`)
|
||||
await q(`DROP FUNCTION IF EXISTS name_extension(text)`)
|
||||
await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`)
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The seed orchestrator (CONCEPT.md D38): nodes that lease article ranges
|
||||
* over HTTPS and submit what they aggregated (D37), and the ranges planned
|
||||
* for them. A range's id is its priority: plans insert newest first.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('seed_nodes', (table) => {
|
||||
table.increments('id')
|
||||
table.text('name').notNullable().unique()
|
||||
// sha256 of the bearer token; the token itself is shown once, when it's made.
|
||||
table.string('token_hash', 64).notNullable().unique()
|
||||
table.boolean('disabled').notNullable().defaultTo(false)
|
||||
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
|
||||
table.timestamp('last_seen_at', { useTz: true }).nullable()
|
||||
})
|
||||
|
||||
this.schema.createTable('seed_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('start_id').notNullable()
|
||||
table.bigInteger('end_id').notNullable()
|
||||
// First article not reported yet; a re-leased range resumes here.
|
||||
table.bigInteger('next_id').notNullable()
|
||||
table.string('status', 8).notNullable().defaultTo('queued')
|
||||
table.integer('node_id').nullable().references('id').inTable('seed_nodes').onDelete('SET NULL')
|
||||
table.integer('attempts').notNullable().defaultTo(0)
|
||||
table.timestamp('leased_at', { useTz: true }).nullable()
|
||||
table.timestamp('heartbeat_at', { useTz: true }).nullable()
|
||||
table.timestamp('done_at', { useTz: true }).nullable()
|
||||
table.bigInteger('headers').notNullable().defaultTo(0)
|
||||
table.bigInteger('files').notNullable().defaultTo(0)
|
||||
table.bigInteger('fragments').notNullable().defaultTo(0)
|
||||
table.text('error').nullable()
|
||||
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
|
||||
table.index(['group_name', 'start_id'])
|
||||
})
|
||||
this.schema.raw(`CREATE INDEX seed_ranges_open_idx ON seed_ranges (id) WHERE status IN ('queued', 'leased')`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('seed_ranges')
|
||||
this.schema.dropTable('seed_nodes')
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Ids per partition: 4M files is a few days of indexing, ~2 GB of heap (FilePartitions uses the same). */
|
||||
const FILE_PARTITION_SIZE = 4_000_000
|
||||
|
||||
/**
|
||||
* `files` partitioned by id, i.e. by age (CONCEPT.md D39). New rows went into
|
||||
* free space scattered through the 9.6 GB heap on the HDD (left by
|
||||
* inspection's updates), one random read per insert; now they go into a
|
||||
* small recent partition that stays in memory. The existing table becomes
|
||||
* the first partition as it is: only a CHECK scan, no rewrite.
|
||||
*
|
||||
* Postgres can't enforce a unique file_key across partitions: every
|
||||
* partition has its own unique index, and FileStore.insert looks keys up in
|
||||
* the older ones. Run with the writers stopped (a few minutes).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
const { rows } = await q(`SELECT coalesce(max(id), 0) AS max FROM files`)
|
||||
// The first new partition starts at the next million.
|
||||
const boundary = (Math.floor(Number(rows[0].max) / 1_000_000) + 1) * 1_000_000
|
||||
// The HDD tablespace on homelab (D32), the default elsewhere; new partitions inherit the parent's.
|
||||
const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
|
||||
const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
|
||||
|
||||
// The old table, renamed; its constraint and index names get a files_p0 prefix.
|
||||
await q(`ALTER TABLE files ADD CONSTRAINT files_p0_bound CHECK (id < ${boundary}) NOT VALID`)
|
||||
await q(`ALTER TABLE files VALIDATE CONSTRAINT files_p0_bound`)
|
||||
await q(`ALTER TABLE files RENAME TO files_p0`)
|
||||
await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_pkey TO files_p0_pkey`)
|
||||
await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_file_key_unique TO files_p0_file_key_unique`)
|
||||
await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_release_id_foreign TO files_p0_release_id_foreign`)
|
||||
for (const index of ['files_content_type_index', 'files_first_message_id_idx', 'files_name_idx', 'files_release_id_index']) {
|
||||
await q(`ALTER INDEX ${index} RENAME TO ${index.replace(/^files_/, 'files_p0_')}`)
|
||||
}
|
||||
// Recreated on the parent, which clones them to every partition.
|
||||
for (const trigger of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
|
||||
await q(`DROP TRIGGER ${trigger} ON files_p0`)
|
||||
}
|
||||
|
||||
// The parent: same columns and defaults (the id sequence), partitioned by id, in the old table's tablespace.
|
||||
await q(`CREATE TABLE files (LIKE files_p0 INCLUDING DEFAULTS) PARTITION BY RANGE (id)${tablespace}`)
|
||||
await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
|
||||
await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
|
||||
await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
|
||||
await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
|
||||
await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
|
||||
await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
|
||||
await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
|
||||
// Matching indexes and the foreign key of files_p0 are taken over, not rebuilt; the CHECK spares the scan.
|
||||
await q(`ALTER TABLE files ATTACH PARTITION files_p0 FOR VALUES FROM (MINVALUE) TO (${boundary})`)
|
||||
await q(`ALTER TABLE files_p0 DROP CONSTRAINT files_p0_bound`)
|
||||
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
|
||||
// binaries.file_id followed the rename to files_p0; point it at the parent (no scan needed).
|
||||
await q(`ALTER TABLE binaries DROP CONSTRAINT binaries_file_id_foreign`)
|
||||
await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
|
||||
|
||||
// The next two partitions; FilePartitions.ensure() (gc:scheduler) keeps two ahead from now on.
|
||||
for (const start of [boundary, boundary + FILE_PARTITION_SIZE]) {
|
||||
const name = `files_p${start}`
|
||||
await q(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${start}) TO (${start + FILE_PARTITION_SIZE})`)
|
||||
await q(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
throw new Error('Not reversible: merging partitions back means rewriting the files table.')
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
const TRIGGERS = [
|
||||
`files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`,
|
||||
`files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`,
|
||||
`files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`,
|
||||
`files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`,
|
||||
]
|
||||
|
||||
/**
|
||||
* files.id (and binaries.file_id, the id sequence) to 64 bits (CONCEPT.md
|
||||
* D39): seeding adds ~14M files a day, and 32 bits run out at 2.1 billion.
|
||||
* Postgres can't change the type of a partition key, so the partitions are
|
||||
* detached, rewritten one by one as plain tables (files_p0: ~100 GB with its
|
||||
* segment lists, 1–2 hours on homelab's HDD) and attached to a new 64-bit
|
||||
* parent. A CHECK per partition, added before its rewrite, spares the attach
|
||||
* scan. Run with the writers stopped and Postgres on wal_level=minimal, or the
|
||||
* rewrite also goes through the WAL (LOG.md 2026-10-05 (8)).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
// One transaction: a failure anywhere leaves files as it was (it drops and recreates the parent).
|
||||
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
const { rows: partitions } = await q(
|
||||
`SELECT c.relname AS name, pg_get_expr(c.relpartbound, c.oid) AS bound
|
||||
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
|
||||
)
|
||||
const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
|
||||
const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
|
||||
|
||||
// The sequence belongs to the old parent: keep it when that goes.
|
||||
await q(`ALTER SEQUENCE files_id_seq OWNED BY NONE`)
|
||||
await q(`ALTER SEQUENCE files_id_seq AS bigint`)
|
||||
await q(`ALTER TABLE binaries DROP CONSTRAINT IF EXISTS binaries_file_id_foreign`)
|
||||
await q(`ALTER TABLE binaries ALTER COLUMN file_id TYPE bigint`)
|
||||
|
||||
for (const p of partitions) {
|
||||
await q(`ALTER TABLE files DETACH PARTITION ${p.name}`)
|
||||
// Triggers cloned from the parent may stay behind on a detached partition; the new parent clones them again.
|
||||
for (const t of TRIGGERS) await q(`DROP TRIGGER IF EXISTS ${t.split(' ')[0]} ON ${p.name}`)
|
||||
const [, from, to] = /FROM \((.+)\) TO \((.+)\)/.exec(p.bound.replaceAll("'", ''))!
|
||||
const conditions = [from === 'MINVALUE' ? null : `id >= ${from}`, to === 'MAXVALUE' ? null : `id < ${to}`].filter(Boolean)
|
||||
if (conditions.length > 0) await q(`ALTER TABLE ${p.name} ADD CONSTRAINT ${p.name}_bound CHECK (${conditions.join(' AND ')})`)
|
||||
}
|
||||
await q(`DROP TABLE files`)
|
||||
|
||||
for (const p of partitions) {
|
||||
await q(`ALTER TABLE ${p.name} ALTER COLUMN id TYPE bigint`)
|
||||
}
|
||||
|
||||
await q(`
|
||||
CREATE TABLE files (
|
||||
id bigint NOT NULL DEFAULT nextval('files_id_seq'),
|
||||
filename varchar(255) NOT NULL,
|
||||
poster varchar(255) NOT NULL,
|
||||
date bigint NOT NULL,
|
||||
parts integer NOT NULL,
|
||||
groups jsonb,
|
||||
created_at timestamptz,
|
||||
updated_at timestamptz,
|
||||
file_key varchar(40),
|
||||
release_id bigint,
|
||||
archive jsonb,
|
||||
real_name text,
|
||||
size bigint,
|
||||
content_type varchar(16),
|
||||
segments bytea NOT NULL,
|
||||
first_message_id text
|
||||
) PARTITION BY RANGE (id)${tablespace}`)
|
||||
await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
|
||||
await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
|
||||
await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
|
||||
await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
|
||||
await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
|
||||
await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
|
||||
await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
|
||||
for (const p of partitions) {
|
||||
await q(`ALTER TABLE files ATTACH PARTITION ${p.name} ${p.bound}`)
|
||||
await q(`ALTER TABLE ${p.name} DROP CONSTRAINT IF EXISTS ${p.name}_bound`)
|
||||
}
|
||||
for (const t of TRIGGERS) await q(`CREATE TRIGGER ${t}`)
|
||||
await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
throw new Error('Not reversible: ids above 2^31 may exist.')
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** When an archive walk (CONCEPT.md D40) looked at a release, so it isn't walked again. */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.timestamp('contents_walked_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('contents_walked_at'))
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
|
||||
* par2 set listing n files was inserted into release_files row by row, each
|
||||
* row's trigger ran release_complete() twice, and that matched every listed
|
||||
* file against all binaries in the database: n × 2 × n × (all binaries). One
|
||||
* insert of a few thousand rows ran for over 20 minutes at full CPU.
|
||||
*
|
||||
* - release_complete(): the release's files are read once, by release id,
|
||||
* and each listed file is matched against those by size or by name. The
|
||||
* correlated subquery read every binary in the database per listed file
|
||||
* (binaries_size_index), filtering by release only afterwards.
|
||||
* - refresh_release_complete(): computes it once.
|
||||
* - release_files: one refresh per release per INSERT statement.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
|
||||
-- The release's files, once, by index (read only when there's a par2 listing to check).
|
||||
have AS MATERIALIZED (
|
||||
SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = rid AND rf.size >= 65536
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
|
||||
ELSE false END
|
||||
FROM r), false)
|
||||
$$`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
// Swapping the trigger locks release_files: inserts still running the old cubic check
|
||||
// (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||||
await this.db.rawQuery(`
|
||||
SELECT pg_cancel_backend(pid) FROM pg_stat_activity
|
||||
WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
|
||||
AND now() - query_start > interval '30 seconds'`)
|
||||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||||
REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION release_files_release_complete_statement()`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||||
FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
|
||||
await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Cheaper release bookkeeping for file inserts (LOG.md 2026-10-06 (6)).
|
||||
*
|
||||
* Every inserted file fired four row triggers, each updating its release:
|
||||
* file_count/size, search_files, extensions and completeness, so a seed
|
||||
* batch of thousands of files rewrote its releases thousands of times (a
|
||||
* quarter of Postgres' active time on homelab). Now one statement-level
|
||||
* trigger updates each release once per INSERT, with all four.
|
||||
*
|
||||
* Completeness (D24), the expensive part, is no longer computed in triggers:
|
||||
* refresh_release_complete() only notes the release in release_dirty, and
|
||||
* gc:scheduler recomputes the noted releases every 30 minutes
|
||||
* (recompute_release_complete). Counts, sizes, search text and extensions
|
||||
* stay current: code decides on them right away (weak releases, deleting
|
||||
* empty releases).
|
||||
*
|
||||
* Also an index on release_files (release_id, size) for the par2 size
|
||||
* match's "listed once in its set" check (IngestService.matchPar2Releases).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
// Swapping triggers locks files and its partitions. Taken first and in one statement: a
|
||||
// DROP TRIGGER later in the transaction deadlocked with ingest (2026-10-06). Give up rather
|
||||
// than queue every writer behind it for long.
|
||||
await q(`SET LOCAL lock_timeout = '2min'`)
|
||||
await q(`LOCK TABLE files, release_files IN ACCESS EXCLUSIVE MODE`)
|
||||
// Built CONCURRENTLY by hand on homelab (2026-10-06); IF NOT EXISTS keeps that one.
|
||||
await q(`CREATE INDEX IF NOT EXISTS release_files_release_size_idx ON release_files (release_id, size)`)
|
||||
|
||||
// Append-only (no key): writers never wait on each other or on the sweep.
|
||||
await q(`CREATE TABLE IF NOT EXISTS release_dirty (release_id bigint NOT NULL)`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION recompute_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
// Every trigger that called this (files, release_files, binaries.size, releases.file_total) now just notes the release.
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL
|
||||
$$`)
|
||||
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
WITH added AS (
|
||||
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
||||
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
||||
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
||||
FROM (
|
||||
SELECT release_id, size, name_extension(nm) AS e,
|
||||
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
||||
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
||||
) x
|
||||
GROUP BY release_id
|
||||
)
|
||||
UPDATE releases r SET
|
||||
file_count = r.file_count + a.n,
|
||||
size = r.size + a.bytes,
|
||||
extensions = CASE
|
||||
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
||||
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
||||
search_files = CASE
|
||||
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
||||
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
||||
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
||||
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
||||
FROM added a
|
||||
WHERE r.id = a.release_id;
|
||||
|
||||
INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
|
||||
// The row triggers keep UPDATE and DELETE (renames, moves between releases); INSERT is the statement trigger.
|
||||
for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
|
||||
await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
|
||||
}
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
await q(`CREATE TRIGGER files_release_totals AFTER DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
await q(`CREATE TRIGGER files_zz_release_complete AFTER DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
await q(`CREATE TRIGGER files_release_insert AFTER INSERT ON files REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION files_release_insert_statement()`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_insert ON files`)
|
||||
for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
|
||||
await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
|
||||
}
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
await q(`DROP FUNCTION IF EXISTS files_release_insert_statement()`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
await q(`DROP FUNCTION IF EXISTS recompute_release_complete(bigint)`)
|
||||
await q(`DROP TABLE IF EXISTS release_dirty`)
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Seed ranges of a higher priority are leased first (`seed:plan --priority`), e.g. a gap in recent history. */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('seed_ranges', (table) => {
|
||||
table.smallint('priority').notNullable().defaultTo(0)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('seed_ranges', (table) => table.dropColumn('priority'))
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Stored parts per binary, kept by IngestService as it inserts parts
|
||||
* (LOG.md 2026-10-06 (7)). claimCompleted counted every binary's parts in
|
||||
* every batch (15% of Postgres' active time); a binary of thousands of
|
||||
* parts that gets a few per batch was counted thousands of times. NULL:
|
||||
* not counted yet (binaries from before), counted exactly on the next new
|
||||
* part. Only parts of binaries without a file are deleted, so it stays exact
|
||||
* until a binary completes.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||||
await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS part_count integer`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS part_count`)
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
|
||||
* file_key is unique per files partition only (D39), so FileStore.insert
|
||||
* looked every key up in every partition: 7 partitions and growing, 35,000
|
||||
* random index reads for a batch of 5,000 files. With file_keys complete it
|
||||
* asks file_keys first, and only keys whose hash is there (a duplicate, or
|
||||
* rarely a collision) are looked up in files.
|
||||
*
|
||||
* New keys come from the files insert statement trigger. The keys from
|
||||
* before are copied in the background (FileKeys.fill, gc:scheduler):
|
||||
* copying them here took over half an hour of HDD reads and held up the
|
||||
* deploy. Until file_keys_fill.complete, FileStore checks every partition.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
|
||||
// One row: the fill's position in files.id, the id it must reach, and whether it got there.
|
||||
await q(`
|
||||
CREATE TABLE IF NOT EXISTS file_keys_fill (
|
||||
id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
next_id bigint NOT NULL DEFAULT 0,
|
||||
until_id bigint,
|
||||
complete boolean NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now())`)
|
||||
await q(`INSERT INTO file_keys_fill (id) VALUES (1) ON CONFLICT DO NOTHING`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
WITH added AS (
|
||||
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
||||
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
||||
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
||||
FROM (
|
||||
SELECT release_id, size, name_extension(nm) AS e,
|
||||
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
||||
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
||||
) x
|
||||
GROUP BY release_id
|
||||
)
|
||||
UPDATE releases r SET
|
||||
file_count = r.file_count + a.n,
|
||||
size = r.size + a.bytes,
|
||||
extensions = CASE
|
||||
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
||||
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
||||
search_files = CASE
|
||||
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
||||
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
||||
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
||||
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
||||
FROM added a
|
||||
WHERE r.id = a.release_id;
|
||||
|
||||
INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
||||
INSERT INTO file_keys (key_hash)
|
||||
SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys_fill`)
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A completed binary whose file is queued for content inspection keeps its
|
||||
* row until inspection has stored the file's fingerprint on it (LOG.md
|
||||
* 2026-10-06 (11)): par2 naming (D16) matches par2 listings against binary
|
||||
* fingerprints and attaches sets to binaries. With the inspection queue
|
||||
* days behind, the GC deleted them first (after 6 hours unfingerprinted,
|
||||
* 48 hours stale). TRUE while waiting; cleared by inspection.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||||
await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS awaiting_inspection boolean`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS awaiting_inspection`)
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Indexes for the GC's deletes of finished binaries, without the ones
|
||||
* waiting for inspection (LOG.md 2026-10-06 (11)): those can be millions
|
||||
* when inspection is behind, and the deletes must not read past them every
|
||||
* sweep. Built CONCURRENTLY: binaries takes writes all the time.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_done_last_seen_idx ON binaries (last_seen_at)
|
||||
WHERE file_id IS NOT NULL AND awaiting_inspection IS NOT TRUE`
|
||||
)
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_awaiting_inspection_idx ON binaries (last_seen_at)
|
||||
WHERE awaiting_inspection IS TRUE`
|
||||
)
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_open_last_seen_idx ON binaries (last_seen_at)
|
||||
WHERE completed_at IS NULL OR conflicted_at IS NOT NULL`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_open_last_seen_idx`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_awaiting_inspection_idx`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_done_last_seen_idx`)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Release completeness (D24) for fewer releases, and without binaries
|
||||
* (LOG.md 2026-10-07 (7)).
|
||||
*
|
||||
* - Only releases whose completeness can change are noted for the sweep: a
|
||||
* par2 listing, a file_total, or complete now (it can become incomplete).
|
||||
* For any other release release_complete() is false by definition; noting
|
||||
* every release a file went into made a sweep check 430,000 releases in
|
||||
* 51 minutes.
|
||||
* - release_complete() matches listed files by name against the files
|
||||
* themselves (coalesce(real_name, filename)); the size match still goes
|
||||
* through a binary when there is one. Seeding no longer creates binaries
|
||||
* for most files and the GC deletes them, so a name only reachable through
|
||||
* a binary was lost.
|
||||
* - Notes that don't qualify are dropped.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete))
|
||||
OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
|
||||
-- The release's files, once, by index (read only when there's a par2 listing to check).
|
||||
have AS MATERIALIZED (
|
||||
SELECT b.size, coalesce(f.real_name, f.filename) AS name
|
||||
FROM files f LEFT JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = rid)
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = rid AND rf.size >= 65536
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name))
|
||||
ELSE false END
|
||||
FROM r), false)
|
||||
$$`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
WITH added AS (
|
||||
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
||||
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
||||
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
||||
FROM (
|
||||
SELECT release_id, size, name_extension(nm) AS e,
|
||||
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
||||
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
||||
) x
|
||||
GROUP BY release_id
|
||||
)
|
||||
UPDATE releases r SET
|
||||
file_count = r.file_count + a.n,
|
||||
size = r.size + a.bytes,
|
||||
extensions = CASE
|
||||
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
||||
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
||||
search_files = CASE
|
||||
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
||||
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
||||
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
||||
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
||||
FROM added a
|
||||
WHERE r.id = a.release_id;
|
||||
|
||||
-- Only releases whose completeness can change (LOG.md 2026-10-07 (7)).
|
||||
INSERT INTO release_dirty (release_id)
|
||||
SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n
|
||||
WHERE release_completeness_matters(n.release_id);
|
||||
INSERT INTO file_keys (key_hash)
|
||||
SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
// The previous definitions are in migrations …180, …190 and …220.
|
||||
throw new Error('Not reversible: re-run the earlier definitions by hand if needed.')
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* The content inspection backlog (LOG.md 2026-10-07 (8)): files waiting to be
|
||||
* inspected, in Postgres instead of as BullMQ jobs. 5.4M waiting jobs took
|
||||
* 3.3 GB of Redis memory and most of its single thread; InspectionQueue.feed
|
||||
* keeps at most INSPECTION_FEED_MAX of them in BullMQ, where the workers and
|
||||
* nodes take them. Lower priority first, then oldest.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS inspection_queue (
|
||||
file_id bigint PRIMARY KEY,
|
||||
priority smallint NOT NULL DEFAULT 0,
|
||||
queued_at timestamptz NOT NULL DEFAULT now())`)
|
||||
await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS inspection_queue_order_idx ON inspection_queue (priority, queued_at)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS inspection_queue`)
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Dropping garbage uploads early (CONCEPT.md D42): random-named uploads that
|
||||
* never get a real name or password.
|
||||
* - upload_signatures: per posting tool (from the Message-ID) and first group,
|
||||
* how many probed random-named releases got a real name; `dropped` ones
|
||||
* aren't stored any more (GarbageFilter).
|
||||
* - signature_probes: a sample of probed random-named releases, what
|
||||
* upload_signatures is learned from (the outcome is read from the release
|
||||
* at learning time, so names that come later count too).
|
||||
* - dropped_releases: releases deleted after their probe, by release_key, so
|
||||
* seeding and ingest don't add them again.
|
||||
* - is_real_release_name(): a name someone would search for. is_readable_name
|
||||
* lets spam through (usenetszene net, VTS_01_1.VOB, Nieuwe map, 100 MiB):
|
||||
* this also wants a year, episode, resolution, codec or language tag, or
|
||||
* three words (LOG.md 2026-10-08 (5)). No `?` in its regexes: knex takes it
|
||||
* for a binding (hence {0,1}).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS upload_signatures (
|
||||
tool varchar(32) NOT NULL,
|
||||
group_name text NOT NULL,
|
||||
probed integer NOT NULL DEFAULT 0,
|
||||
named integer NOT NULL DEFAULT 0,
|
||||
dropped boolean NOT NULL DEFAULT false,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (tool, group_name))`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS signature_probes (
|
||||
release_id bigint PRIMARY KEY,
|
||||
tool varchar(32) NOT NULL,
|
||||
group_name text NOT NULL,
|
||||
probed_at timestamptz NOT NULL DEFAULT now())`)
|
||||
await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS signature_probes_probed_at_idx ON signature_probes (probed_at)`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS dropped_releases (
|
||||
release_key varchar(40) PRIMARY KEY,
|
||||
reason varchar(16) NOT NULL,
|
||||
dropped_at timestamptz NOT NULL DEFAULT now())`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_real_release_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT name IS NOT NULL AND is_readable_name(name)
|
||||
AND NOT (
|
||||
name ~* '(usenet|video_ts|nieuwe map|new folder|(^|[^a-z0-9])mib([^a-z0-9]|$))'
|
||||
OR name ~* '^(vts_|sample|cd[0-9])'
|
||||
OR name ~ '^\\(.*\\)$'
|
||||
OR name ~ '^[A-Za-z0-9]{8,}$')
|
||||
AND (
|
||||
name ~* '(^|[^a-z0-9])((19|20)[0-9]{2}|(480|576|720|1080|2160)[pi]|dl|nl)([^a-z0-9]|$)'
|
||||
OR name ~* '(s[0-9]{2}e[0-9]{2}|x26[45]|h\\.{0,1}26[45]|hevc|web[- .]{0,1}(dl|rip)|bluray|remux|german|dutch|multi|flac|mp3|epub|pdf|ac3|dts|xxx)'
|
||||
OR (SELECT count(*) FROM regexp_matches(name, '[A-Za-z]{3,}', 'g')) >= 3)
|
||||
$$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP FUNCTION IF EXISTS is_real_release_name(text)`)
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS dropped_releases`)
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS signature_probes`)
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS upload_signatures`)
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Listed spots (D29) were only read from their overview: no description, no
|
||||
* NZB, so neither a password in the spot's text nor the NZB's own
|
||||
* `<meta type="password">` was known (LOG.md 2026-10-08 (10)). nzb_read_at:
|
||||
* when the spot's article and NZB were read for that (SpotService.readListing),
|
||||
* also set while a reader holds it. The index finds the ones still to read,
|
||||
* newest first.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS nzb_read_at timestamptz`)
|
||||
await this.db.rawQuery(
|
||||
`CREATE INDEX IF NOT EXISTS spots_listing_unread_idx ON spots (posted_at DESC) WHERE status = 'listed' AND nzb_read_at IS NULL`
|
||||
)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP INDEX IF EXISTS spots_listing_unread_idx`)
|
||||
await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS nzb_read_at`)
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* What inspection learns about a file (content type, archive listing) lives
|
||||
* in file_details, on the NVMe, instead of in `files` (LOG.md 2026-10-08 (14)).
|
||||
* `files` sits on the HDD and was updated after its insert by every probe:
|
||||
* a new row version on a random page and an entry in each of its indexes
|
||||
* (3.7M updates against 49.5M inserts in a day, 27% HOT). Its rows are now
|
||||
* only inserted (and moved between releases or renamed, rarely).
|
||||
* The existing values are copied by `files:move-details` after this deploy;
|
||||
* a later migration drops the columns.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
// The foreign key locks `files` briefly: take the lock first, so it can't deadlock with writers.
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '60s'`)
|
||||
await this.db.rawQuery(`LOCK TABLE files IN SHARE ROW EXCLUSIVE MODE`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS file_details (
|
||||
file_id bigint PRIMARY KEY REFERENCES files (id) ON DELETE CASCADE,
|
||||
content_type varchar(16),
|
||||
archive jsonb)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TABLE IF EXISTS file_details`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user