implemented a lot of features
This commit is contained in:
Daan Meijer committed
1 parent
62f1767df3
commit
bcb9be5da9
141 files changed
+10705
-385
No files matched your search
@@ -0,0 +1,32 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Files that are still being collected from XOVER. One row per uploaded file,
|
||||
* identified by file_key (see lib/subject/file_key.ts).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'binaries'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('file_key', 40).notNullable().unique()
|
||||
table.text('poster').notNullable()
|
||||
table.text('subject').notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.integer('total_parts').notNullable()
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable().index()
|
||||
table.timestamp('completed_at', { useTz: true })
|
||||
// Set when two different articles claim the same part number (see IngestService).
|
||||
table.timestamp('conflicted_at', { useTz: true })
|
||||
table.integer('file_id').unsigned().references('files.id').onDelete('SET NULL')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Segments (articles) of a binary. The primary key makes re-ingesting the
|
||||
* same range a no-op.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'parts'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE')
|
||||
table.integer('number').notNullable()
|
||||
table.text('message_id').notNullable()
|
||||
table.integer('bytes').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.primary(['binary_id', 'number'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'files'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.string('file_key', 40).unique()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.dropColumn('file_key')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release is one download: the files of one post (archive volumes, par2
|
||||
* set, nfo). See lib/release/release_key.ts and CONCEPT.md (D16).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'releases'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('release_key', 40).notNullable().unique()
|
||||
table.text('name').notNullable()
|
||||
// Null for obfuscated posts, whose poster is random per article.
|
||||
table.text('poster')
|
||||
// Number of files announced by the [i/M] subject counter, if any.
|
||||
table.integer('file_total')
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable()
|
||||
})
|
||||
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('release_id'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('release_id'))
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Par2-based grouping of obfuscated posts (CONCEPT.md D16): the par2 file
|
||||
* list of a release, and the exact file size of body-derived binaries to
|
||||
* match against it.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
// Exact file size from =ybegin; only known for body-derived binaries.
|
||||
table.bigInteger('size').index()
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// How the release was recognised: subject, stem, weak-stem or par2.
|
||||
table.string('source', 16)
|
||||
})
|
||||
this.schema.createTable('release_files', (table) => {
|
||||
table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
|
||||
table.string('par2_file_id', 32).notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.bigInteger('size').notNullable().index()
|
||||
table.string('md5_16k', 32).notNullable()
|
||||
table.primary(['release_id', 'par2_file_id'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('release_files')
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('source'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('size'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Article ranges handed to worker:fetch, so a range only counts as indexed
|
||||
* once it was fetched successfully (CONCEPT.md D18).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('groups', (table) => {
|
||||
// How far the scheduler has queued; last_indexed_id becomes the
|
||||
// watermark below which every article was fetched.
|
||||
table.bigInteger('last_queued_id').notNullable().defaultTo(0)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('UPDATE groups SET last_queued_id = last_indexed_id')
|
||||
})
|
||||
|
||||
this.schema.createTable('fetch_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.integer('group_id').notNullable().references('groups.id').onDelete('CASCADE')
|
||||
table.bigInteger('start_id').notNullable()
|
||||
table.bigInteger('end_id').notNullable()
|
||||
// queued | done | failed
|
||||
table.string('status', 8).notNullable().defaultTo('queued')
|
||||
table.integer('requeues').notNullable().defaultTo(0)
|
||||
table.text('error')
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('updated_at', { useTz: true }).notNullable()
|
||||
table.index(['group_id', 'status'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('fetch_ranges')
|
||||
this.schema.alterTable('groups', (table) => table.dropColumn('last_queued_id'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Archive listings from the first RAR volume (CONCEPT.md D19). */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
// RarInfo of a first volume: format, flags and entries.
|
||||
table.jsonb('archive')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// Password protected: encrypted file data or encrypted headers.
|
||||
table.boolean('encrypted')
|
||||
// Entries of the release's first archive volume.
|
||||
table.jsonb('contents')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('contents')
|
||||
table.dropColumn('encrypted')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('archive'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Splitting equal-sized obfuscated volumes (CONCEPT.md D20): the extra
|
||||
* articles of conflicted binaries are kept, and the split-off volumes point
|
||||
* to the conflicted binary they came from.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('part_conflicts', (table) => {
|
||||
table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE')
|
||||
table.integer('number').notNullable()
|
||||
table.text('message_id').notNullable()
|
||||
table.integer('bytes').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.primary(['binary_id', 'number', 'message_id'])
|
||||
})
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.bigInteger('parent_binary_id').references('binaries.id').onDelete('SET NULL').index()
|
||||
// Last time the splitter processed this conflicted binary.
|
||||
table.timestamp('split_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.dropColumn('split_at')
|
||||
table.dropColumn('parent_binary_id')
|
||||
})
|
||||
this.schema.dropTable('part_conflicts')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** Full-file MD5 from par2, to verify assembled (e.g. split) volumes. */
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'release_files'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.string('md5', 32)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => table.dropColumn('md5'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Binaries that were flagged conflicted before part_conflicts existed lost
|
||||
* their extra articles, so splitting them only reshuffles the first article
|
||||
* per part number, a mix of different volumes. Mark them as never to be
|
||||
* split, and remove the volumes and files the first splitter run made from
|
||||
* them (see LOG.md, 2026-09-28).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
WITH boundary AS (
|
||||
SELECT migration_time FROM adonis_schema WHERE name LIKE '%1768620764790%'
|
||||
), legacy AS (
|
||||
SELECT id FROM binaries
|
||||
WHERE conflicted_at IS NOT NULL AND parent_binary_id IS NULL
|
||||
AND conflicted_at < (SELECT migration_time FROM boundary)
|
||||
), instances AS (
|
||||
SELECT id, file_id FROM binaries WHERE parent_binary_id IN (SELECT id FROM legacy)
|
||||
), deleted_files AS (
|
||||
DELETE FROM files WHERE id IN (SELECT file_id FROM instances WHERE file_id IS NOT NULL)
|
||||
), deleted_instances AS (
|
||||
DELETE FROM binaries WHERE id IN (SELECT id FROM instances)
|
||||
)
|
||||
-- 'infinity': last_seen_at > split_at is never true, so SplitService skips them.
|
||||
UPDATE binaries SET split_at = 'infinity' WHERE id IN (SELECT id FROM legacy)
|
||||
`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
// The removed volumes were unverifiable; nothing to restore.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* yEnc CRCs of body-derived articles, so split volumes can be verified
|
||||
* without downloading them (CONCEPT.md D20): the combined part CRCs of a
|
||||
* volume must equal the whole-file crc32 in its last part.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
table.string('pcrc32', 8)
|
||||
// Decoded length of the part (=ypart end - begin + 1).
|
||||
table.integer('part_size')
|
||||
// Whole-file CRC32, when the poster included it (usually in the last part).
|
||||
table.string('file_crc32', 8)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
table.dropColumn('pcrc32')
|
||||
table.dropColumn('part_size')
|
||||
table.dropColumn('file_crc32')
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Names and groupings from file contents (CONCEPT.md D21): a fingerprint
|
||||
* (exact size + MD5 of the first 16 KiB, as par2 records it) and the real
|
||||
* filename recovered from par2.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.string('md5_16k', 32).index()
|
||||
// Filename recovered from file contents (par2); `name` stays as posted.
|
||||
table.text('real_name')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.text('real_name')
|
||||
})
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => {
|
||||
// Only for part 1: MD5 of the first 16 KiB of the file.
|
||||
table.string('md5_16k', 32)
|
||||
})
|
||||
}
|
||||
this.schema.alterTable('release_files', (table) => {
|
||||
table.index(['md5_16k'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('release_files', (table) => table.dropIndex(['md5_16k']))
|
||||
for (const tableName of ['parts', 'part_conflicts']) {
|
||||
this.schema.alterTable(tableName, (table) => table.dropColumn('md5_16k'))
|
||||
}
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('real_name'))
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.dropColumn('real_name')
|
||||
table.dropColumn('md5_16k')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Web interface (CONCEPT.md D22): trigram indexes for substring search, and
|
||||
* SQL versions of the "readable name" rule used for filtering.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('CREATE EXTENSION IF NOT EXISTS pg_trgm')
|
||||
// Per-file suffixes stripped, as in lib/release fileStem.
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(name,
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
// Not an obfuscated token (as in ArchiveService.looksRandom).
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '\\d'
|
||||
AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]')
|
||||
OR name_stem(name) ~* '^[0-9a-f]{16,}$')
|
||||
$$`)
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_posted_at_idx ON releases (posted_at DESC NULLS LAST)')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_posted_at_idx')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_real_name_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_filename_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_name_trgm')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS is_readable_name(text)')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS name_stem(text)')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Per-release totals for browsing and sorting (CONCEPT.md D22). A trigger on
|
||||
* `files` keeps `releases.file_count` and `releases.size` current, whichever
|
||||
* code path assigns files to releases.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.bigInteger('size')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.integer('file_count').notNullable().defaultTo(0).index()
|
||||
table.bigInteger('size').notNullable().defaultTo(0).index()
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
UPDATE files f SET size = s.size
|
||||
FROM (SELECT id, (SELECT coalesce(sum((v->>'size')::bigint), 0) FROM jsonb_each(message_ids) e(k, v)) AS size FROM files) s
|
||||
WHERE s.id = f.id`)
|
||||
await db.rawQuery(`
|
||||
UPDATE releases r SET file_count = t.n, size = t.size
|
||||
FROM (SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS size FROM files
|
||||
WHERE release_id IS NOT NULL GROUP BY release_id) t
|
||||
WHERE t.release_id = r.id`)
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION files_release_totals() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN
|
||||
UPDATE releases SET file_count = file_count - 1, size = size - coalesce(OLD.size, 0) WHERE id = OLD.release_id;
|
||||
END IF;
|
||||
IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN
|
||||
UPDATE releases SET file_count = file_count + 1, size = size + coalesce(NEW.size, 0) WHERE id = NEW.release_id;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER files_release_totals
|
||||
AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS files_release_totals ON files')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS files_release_totals()')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('size')
|
||||
table.dropColumn('file_count')
|
||||
})
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('size'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Stored completeness for the "Complete" filter (CONCEPT.md D22); computing
|
||||
* it per request took ~30 s on 34k releases. Triggers keep it current when
|
||||
* files, par2 lists, sizes or announced totals change.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS binaries_file_id_idx ON binaries (file_id)')
|
||||
await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops)')
|
||||
await db.rawQuery('ALTER TABLE releases ADD COLUMN complete boolean NOT NULL DEFAULT false')
|
||||
await db.rawQuery('CREATE INDEX releases_complete_idx ON releases (complete)')
|
||||
|
||||
// All files announced by [i/M] are there, or every par2-listed file
|
||||
// (>= 64 KiB) is present by exact size or real name.
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean STABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = r.id AND rf.size >= 65536
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name)))
|
||||
ELSE false END
|
||||
FROM releases r WHERE r.id = rid), false)
|
||||
$$`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases SET complete = release_complete(rid) WHERE id = rid AND complete IS DISTINCT FROM release_complete(rid)
|
||||
$$`)
|
||||
|
||||
// files: after the totals trigger has updated file_count (trigger names sort alphabetically).
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN PERFORM refresh_release_complete(OLD.release_id); END IF;
|
||||
IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN PERFORM refresh_release_complete(NEW.release_id); END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER files_zz_release_complete
|
||||
AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(NEW.release_id);
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete
|
||||
AFTER INSERT ON release_files FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
|
||||
|
||||
// An exact size from content inspection can make a par2-listed file "present".
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION binaries_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.file_id IS NOT NULL THEN
|
||||
PERFORM refresh_release_complete(f.release_id) FROM files f WHERE f.id = NEW.file_id AND f.release_id IS NOT NULL;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER binaries_release_complete
|
||||
AFTER UPDATE OF size ON binaries FOR EACH ROW
|
||||
WHEN (OLD.size IS DISTINCT FROM NEW.size) EXECUTE FUNCTION binaries_release_complete()`)
|
||||
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION releases_file_total_complete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(NEW.id);
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
await db.rawQuery(`
|
||||
CREATE TRIGGER releases_file_total_complete
|
||||
AFTER UPDATE OF file_total ON releases FOR EACH ROW
|
||||
WHEN (OLD.file_total IS DISTINCT FROM NEW.file_total) EXECUTE FUNCTION releases_file_total_complete()`)
|
||||
|
||||
await db.rawQuery('UPDATE releases SET complete = release_complete(id) WHERE file_count > 0')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS releases_file_total_complete ON releases')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS binaries_release_complete ON binaries')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS release_files_release_complete ON release_files')
|
||||
await db.rawQuery('DROP TRIGGER IF EXISTS files_zz_release_complete ON files')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS releases_file_total_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS binaries_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_files_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS files_release_complete()')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS refresh_release_complete(bigint)')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_complete(bigint)')
|
||||
await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS complete')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS releases_contents_trgm')
|
||||
await db.rawQuery('DROP INDEX IF EXISTS binaries_file_id_idx')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* par2 set id per listed file (traceability; see LOG.md 2026-09-28 for the
|
||||
* cross-post merges it helps diagnose), and a stricter "random name" rule:
|
||||
* long letter-only tokens with many capitals count as random too
|
||||
* (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('release_files', (table) => {
|
||||
table.string('set_id', 32)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]'
|
||||
AND (name_stem(name) ~ '\\d' OR length(regexp_replace(name_stem(name), '[^A-Z]', '', 'g')) >= 6))
|
||||
OR name_stem(name) ~* '^[0-9a-f]{16,}$')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('release_files', (table) => table.dropColumn('set_id'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Newznab category per release (CONCEPT.md D23), derived from the name and
|
||||
* the archive contents. A stored generated column, so it follows renames
|
||||
* and newly listed contents without any application code.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text, contents jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- TV: S01E02, 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[._ -])(S\\d{1,2}E\\d{1,3}|\\d{1,2}x\\d{2})([._ -]|$)' OR name ~* '[._ -]S\\d{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)\\d{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\\.?26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: lossless or mp3 by contents, otherwise scene music names (-WEB-2026-GROUP).
|
||||
WHEN coalesce(contents::text, '') ~* '\\.flac"' OR name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN coalesce(contents::text, '') ~* '\\.mp3"' OR name ~* '(^|[._ -])mp3([._ -]|$)'
|
||||
OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\\d{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN coalesce(contents::text, '') ~* '\\.(epub|mobi|azw3)"' OR name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020
|
||||
WHEN coalesce(contents::text, '') ~* '\\.(exe|msi|dmg)"' THEN 4000
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
await db.rawQuery(
|
||||
'ALTER TABLE releases ADD COLUMN category integer GENERATED ALWAYS AS (release_category(name, contents)) STORED'
|
||||
)
|
||||
await db.rawQuery('CREATE INDEX releases_category_idx ON releases (category)')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS category')
|
||||
await db.rawQuery('DROP FUNCTION IF EXISTS release_category(text, jsonb)')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A second article with the same part CRC is a repost of the same data, not
|
||||
* a different file (CONCEPT.md D20). Drop such recorded conflicts and unflag
|
||||
* binaries whose only conflicts were reposts. Binaries flagged before CRCs
|
||||
* were stored (split_at = 'infinity') cannot be checked and stay flagged.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
DELETE FROM part_conflicts c USING parts p
|
||||
WHERE p.binary_id = c.binary_id AND p.number = c.number
|
||||
AND p.pcrc32 IS NOT NULL AND c.pcrc32 IS NOT NULL AND p.pcrc32 = c.pcrc32`)
|
||||
// Split volumes made from reposts are copies of their parent's file.
|
||||
await db.rawQuery(`
|
||||
DELETE FROM binaries v USING binaries b
|
||||
WHERE v.parent_binary_id = b.id AND v.completed_at IS NULL
|
||||
AND b.conflicted_at IS NOT NULL AND b.split_at IS DISTINCT FROM 'infinity'
|
||||
AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`)
|
||||
await db.rawQuery(`
|
||||
UPDATE binaries b SET conflicted_at = NULL
|
||||
WHERE b.conflicted_at IS NOT NULL AND b.parent_binary_id IS NULL
|
||||
AND b.split_at IS DISTINCT FROM 'infinity'
|
||||
AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Body downloads per group (CONCEPT.md D24). Identifying obfuscated articles
|
||||
* costs one body download each; busy groups carry far more of them than the
|
||||
* line can fetch, so they are indexed header-only (normal subjects) unless
|
||||
* enabled.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'groups'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.boolean('fetch_bodies').notNullable().defaultTo(false)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`UPDATE groups SET fetch_bodies = true WHERE name = 'alt.binaries.test'`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => table.dropColumn('fetch_bodies'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* is_readable_name(): also judge the stem without an unknown short
|
||||
* extension, so `326678d1d07ef12a.bin` counts as random (CONCEPT.md D22).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
|
||||
SELECT name IS NOT NULL AND NOT (
|
||||
(stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
|
||||
AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
|
||||
OR stem ~* '^[0-9a-f]{16,}$')
|
||||
FROM s
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* File type by magic bytes (lib/content), and releases that are only an
|
||||
* encrypted container (CONCEPT.md D25).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.string('content_type', 16).index()
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// e.g. 'nzbe': an encrypted NZB, useless without its key.
|
||||
table.string('container', 16)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('container'))
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('content_type'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* NZBs posted to Usenet are read into releases (CONCEPT.md D26): the archive
|
||||
* password they may carry, and a lookup of files by their first segment.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// From <meta type="password"> of a posted NZB; passed on in our NZBs.
|
||||
table.text('password')
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('DROP INDEX IF EXISTS files_first_segment_idx')
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('password'))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** name_stem(): also strip subtitle extensions, as lib/release fileStem does. */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(name,
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* name_stem(): strip a 3-digit split suffix only after a file extension
|
||||
* (x.mkv.001), not codec names like H.264 / H.265, matching lib/release.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(
|
||||
regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'),
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/** name_stem(): same extension list as lib/release fileStem (adds bin, m3u, cue, log, bz2). */
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT regexp_replace(
|
||||
regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'),
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|srr|srs|txt|md5|sha1|tar|zst|gz|xz|bz2|mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log))+$',
|
||||
'', 'i')
|
||||
$$`)
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Files of uploaders whose bodies are not fetched right away (ngPost, see
|
||||
* CONCEPT.md D28): one row per file, from the headers alone. The row keeps
|
||||
* the article range, so the file can be found again, and is the queue for
|
||||
* worker:deferred, which fetches one body per file.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('deferred_files', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('file_key', 40).notNullable().unique()
|
||||
// File name from the subject (random for ngPost).
|
||||
table.text('name').notNullable()
|
||||
table.text('subject').notNullable()
|
||||
table.text('poster').notNullable()
|
||||
// Article numbers are counted in the group where the file was first seen.
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('first_article').notNullable()
|
||||
table.text('first_message_id').notNullable()
|
||||
table.bigInteger('last_article').notNullable()
|
||||
table.integer('article_count').notNullable()
|
||||
table.bigInteger('bytes').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable()
|
||||
// pending → sampled (one body read) or expanded (sent to the body queue) or failed.
|
||||
table.string('status', 16).notNullable().defaultTo('pending')
|
||||
table.timestamp('processed_at', { useTz: true })
|
||||
// From the sampled body.
|
||||
table.text('yenc_name')
|
||||
table.bigInteger('size')
|
||||
table.integer('total_parts')
|
||||
table.integer('sampled_part')
|
||||
table.string('content_type', 16)
|
||||
table.boolean('encrypted')
|
||||
table.text('error')
|
||||
table.index(['status', 'last_seen_at'])
|
||||
table.index(['first_seen_at'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('deferred_files')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Deferred articles whose headers don't say which file they belong to
|
||||
* (ngPost with random subjects and a new poster every few articles, see
|
||||
* CONCEPT.md D28). Only the article range per fetch batch is kept;
|
||||
* `deferred:expand-ranges` finds the articles again and queues their bodies.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('deferred_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('first_article').notNullable()
|
||||
table.bigInteger('last_article').notNullable()
|
||||
table.integer('article_count').notNullable()
|
||||
table.bigInteger('bytes').notNullable()
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('expanded_at', { useTz: true })
|
||||
table.index(['expanded_at', 'id'])
|
||||
table.index(['created_at'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('deferred_ranges')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Spotnet spots (CONCEPT.md D29): signed posts in free.pt that point to a
|
||||
* compressed NZB. Filled from headers by worker:fetch, then worker:spots
|
||||
* verifies each spot, reads its NZB and imports it as a release.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.createTable('spots', (table) => {
|
||||
table.text('message_id').primary()
|
||||
table.text('group_name').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.text('title').notNullable()
|
||||
table.text('tag')
|
||||
table.text('poster').notNullable()
|
||||
table.integer('category')
|
||||
table.jsonb('subcats')
|
||||
table.bigInteger('size')
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.integer('key_id')
|
||||
// Spotweb's spotter id (CRC of the poster's key), to check personal disposes.
|
||||
table.text('spotter_id')
|
||||
// pending → imported, or unverified / no-nzb / moderation / removed / failed.
|
||||
table.string('status', 16).notNullable().defaultTo('pending')
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL')
|
||||
table.text('website')
|
||||
table.text('error')
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('processed_at', { useTz: true })
|
||||
table.index(['status', 'created_at'])
|
||||
table.index(['release_id'])
|
||||
table.index(['status', 'release_id'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('spots')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Smaller indexes (LOG.md 2026-09-28 (17)). 99.5% of releases have no files
|
||||
* yet (most never will), and search only looks at releases with files; most
|
||||
* binaries have no size, fingerprint, file or parent. Indexing only the rows
|
||||
* that queries can match saves GBs and work on every insert. Built
|
||||
* concurrently, so ingest keeps running.
|
||||
*/
|
||||
const INDEXES: [table: string, name: string, definition: string][] = [
|
||||
['releases', 'releases_name_trgm', `USING gin (name gin_trgm_ops) WHERE file_count > 0`],
|
||||
['releases', 'releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops) WHERE file_count > 0`],
|
||||
['releases', 'releases_posted_at_idx', `(posted_at DESC NULLS LAST) WHERE file_count > 0`],
|
||||
['releases', 'releases_size_index', `(size) WHERE file_count > 0`],
|
||||
['releases', 'releases_file_count_index', `(file_count) WHERE file_count > 0`],
|
||||
['releases', 'releases_complete_idx', `(complete) WHERE file_count > 0`],
|
||||
['releases', 'releases_category_idx', `(category) WHERE file_count > 0`],
|
||||
['binaries', 'binaries_md5_16k_index', `(md5_16k) WHERE md5_16k IS NOT NULL`],
|
||||
['binaries', 'binaries_size_index', `(size) WHERE size IS NOT NULL`],
|
||||
['binaries', 'binaries_file_id_idx', `(file_id) WHERE file_id IS NOT NULL`],
|
||||
['binaries', 'binaries_parent_binary_id_index', `(parent_binary_id) WHERE parent_binary_id IS NOT NULL`],
|
||||
]
|
||||
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
for (const [table, name, definition] of INDEXES) {
|
||||
// Build the partial index under a temporary name, then swap it in.
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_partial`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_partial ON ${table} ${definition}`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`ALTER INDEX ${name}_partial RENAME TO ${name}`)
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
for (const [table, name, definition] of INDEXES) {
|
||||
const full = definition.replace(/ WHERE .*$/, '')
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name} ON ${table} ${full}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Spot listings (CONCEPT.md D29): old spots become a release without files
|
||||
* (source `spot`), whose NZB is fetched when first downloaded. Search shows
|
||||
* releases with files or spot listings, so the partial release indexes
|
||||
* cover both. Spots are verified from their overview when fetched.
|
||||
*/
|
||||
const VISIBLE = `(file_count > 0 OR source = 'spot')`
|
||||
const INDEXES: [name: string, definition: string][] = [
|
||||
['releases_name_trgm', `USING gin (name gin_trgm_ops)`],
|
||||
['releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops)`],
|
||||
['releases_posted_at_idx', `(posted_at DESC NULLS LAST)`],
|
||||
['releases_size_index', `(size)`],
|
||||
['releases_file_count_index', `(file_count)`],
|
||||
['releases_complete_idx', `(complete)`],
|
||||
['releases_category_idx', `(category)`],
|
||||
]
|
||||
|
||||
export default class extends BaseSchema {
|
||||
static disableTransactions = true
|
||||
|
||||
async up() {
|
||||
await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
|
||||
await this.swap(VISIBLE)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.swap('file_count > 0')
|
||||
await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
|
||||
}
|
||||
|
||||
private async swap(predicate: string) {
|
||||
for (const [name, definition] of INDEXES) {
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_next`)
|
||||
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_next ON releases ${definition} WHERE ${predicate}`)
|
||||
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
||||
await this.db.rawQuery(`ALTER INDEX ${name}_next RENAME TO ${name}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
+303
-2
@@ -32,14 +32,158 @@ export class AuthAccessTokenSchema extends BaseModel {
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class BinarySchema extends BaseModel {
|
||||
static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
|
||||
$columns = BinarySchema.$columns
|
||||
@column.dateTime()
|
||||
declare completedAt: DateTime | null
|
||||
@column.dateTime()
|
||||
declare conflictedAt: DateTime | null
|
||||
@column()
|
||||
declare fileId: number | null
|
||||
@column()
|
||||
declare fileKey: string
|
||||
@column.dateTime()
|
||||
declare firstSeenAt: DateTime
|
||||
@column()
|
||||
declare groups: any
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column.dateTime()
|
||||
declare lastSeenAt: DateTime
|
||||
@column()
|
||||
declare md516K: string | null
|
||||
@column()
|
||||
declare name: string
|
||||
@column()
|
||||
declare parentBinaryId: bigint | number | null
|
||||
@column.dateTime()
|
||||
declare postedAt: DateTime | null
|
||||
@column()
|
||||
declare poster: string
|
||||
@column()
|
||||
declare realName: string | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column.dateTime()
|
||||
declare splitAt: DateTime | null
|
||||
@column()
|
||||
declare subject: string
|
||||
@column()
|
||||
declare totalParts: number
|
||||
}
|
||||
|
||||
export class DeferredFileSchema extends BaseModel {
|
||||
static $columns = ['articleCount', 'bytes', 'contentType', 'encrypted', 'error', 'fileKey', 'firstArticle', 'firstMessageId', 'firstSeenAt', 'groupName', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'processedAt', 'sampledPart', 'size', 'status', 'subject', 'totalParts', 'yencName'] as const
|
||||
$columns = DeferredFileSchema.$columns
|
||||
@column()
|
||||
declare articleCount: number
|
||||
@column()
|
||||
declare bytes: bigint | number
|
||||
@column()
|
||||
declare contentType: string | null
|
||||
@column()
|
||||
declare encrypted: boolean | null
|
||||
@column()
|
||||
declare error: string | null
|
||||
@column()
|
||||
declare fileKey: string
|
||||
@column()
|
||||
declare firstArticle: bigint | number
|
||||
@column()
|
||||
declare firstMessageId: string
|
||||
@column.dateTime()
|
||||
declare firstSeenAt: DateTime
|
||||
@column()
|
||||
declare groupName: string
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column()
|
||||
declare lastArticle: bigint | number
|
||||
@column.dateTime()
|
||||
declare lastSeenAt: DateTime
|
||||
@column()
|
||||
declare name: string
|
||||
@column.dateTime()
|
||||
declare postedAt: DateTime | null
|
||||
@column()
|
||||
declare poster: string
|
||||
@column.dateTime()
|
||||
declare processedAt: DateTime | null
|
||||
@column()
|
||||
declare sampledPart: number | null
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column()
|
||||
declare status: string
|
||||
@column()
|
||||
declare subject: string
|
||||
@column()
|
||||
declare totalParts: number | null
|
||||
@column()
|
||||
declare yencName: string | null
|
||||
}
|
||||
|
||||
export class DeferredRangeSchema extends BaseModel {
|
||||
static $columns = ['articleCount', 'bytes', 'createdAt', 'expandedAt', 'firstArticle', 'groupName', 'id', 'lastArticle'] as const
|
||||
$columns = DeferredRangeSchema.$columns
|
||||
@column()
|
||||
declare articleCount: number
|
||||
@column()
|
||||
declare bytes: bigint | number
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column.dateTime()
|
||||
declare expandedAt: DateTime | null
|
||||
@column()
|
||||
declare firstArticle: bigint | number
|
||||
@column()
|
||||
declare groupName: string
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column()
|
||||
declare lastArticle: bigint | number
|
||||
}
|
||||
|
||||
export class FetchRangeSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
|
||||
$columns = FetchRangeSchema.$columns
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare endId: bigint | number
|
||||
@column()
|
||||
declare error: string | null
|
||||
@column()
|
||||
declare groupId: number
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column()
|
||||
declare requeues: number
|
||||
@column()
|
||||
declare startId: bigint | number
|
||||
@column()
|
||||
declare status: string
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
|
||||
export class FileSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'date', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'updatedAt'] as const
|
||||
static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'realName', 'releaseId', 'size', 'updatedAt'] as const
|
||||
$columns = FileSchema.$columns
|
||||
@column()
|
||||
declare archive: any | null
|
||||
@column()
|
||||
declare contentType: string | null
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime | null
|
||||
@column()
|
||||
declare date: bigint | number
|
||||
@column()
|
||||
declare fileKey: string | null
|
||||
@column()
|
||||
declare filename: string
|
||||
@column()
|
||||
declare groups: any | null
|
||||
@@ -51,27 +195,184 @@ export class FileSchema extends BaseModel {
|
||||
declare parts: number
|
||||
@column()
|
||||
declare poster: string
|
||||
@column()
|
||||
declare realName: string | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class GroupSchema extends BaseModel {
|
||||
static $columns = ['active', 'createdAt', 'id', 'lastIndexedId', 'name', 'updatedAt'] as const
|
||||
static $columns = ['active', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
|
||||
$columns = GroupSchema.$columns
|
||||
@column()
|
||||
declare active: boolean
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime | null
|
||||
@column()
|
||||
declare fetchBodies: boolean
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
@column()
|
||||
declare lastIndexedId: bigint | number | null
|
||||
@column()
|
||||
declare lastQueuedId: bigint | number
|
||||
@column()
|
||||
declare name: string
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime | null
|
||||
}
|
||||
|
||||
export class PartConflictSchema extends BaseModel {
|
||||
static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const
|
||||
$columns = PartConflictSchema.$columns
|
||||
@column()
|
||||
declare articleNumber: bigint | number | null
|
||||
@column({ isPrimary: true })
|
||||
declare binaryId: bigint | number
|
||||
@column()
|
||||
declare bytes: number
|
||||
@column()
|
||||
declare fileCrc32: string | null
|
||||
@column()
|
||||
declare md516K: string | null
|
||||
@column()
|
||||
declare messageId: string
|
||||
@column()
|
||||
declare number: number
|
||||
@column()
|
||||
declare partSize: number | null
|
||||
@column()
|
||||
declare pcrc32: string | null
|
||||
}
|
||||
|
||||
export class PartSchema extends BaseModel {
|
||||
static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const
|
||||
$columns = PartSchema.$columns
|
||||
@column()
|
||||
declare articleNumber: bigint | number | null
|
||||
@column({ isPrimary: true })
|
||||
declare binaryId: bigint | number
|
||||
@column()
|
||||
declare bytes: number
|
||||
@column()
|
||||
declare fileCrc32: string | null
|
||||
@column()
|
||||
declare md516K: string | null
|
||||
@column()
|
||||
declare messageId: string
|
||||
@column()
|
||||
declare number: number
|
||||
@column()
|
||||
declare partSize: number | null
|
||||
@column()
|
||||
declare pcrc32: string | null
|
||||
}
|
||||
|
||||
export class ReleaseFileSchema extends BaseModel {
|
||||
static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const
|
||||
$columns = ReleaseFileSchema.$columns
|
||||
@column()
|
||||
declare md5: string | null
|
||||
@column()
|
||||
declare md516K: string
|
||||
@column()
|
||||
declare name: string
|
||||
@column()
|
||||
declare par2FileId: string
|
||||
@column({ isPrimary: true })
|
||||
declare releaseId: bigint | number
|
||||
@column()
|
||||
declare setId: string | null
|
||||
@column()
|
||||
declare size: bigint | number
|
||||
}
|
||||
|
||||
export class ReleaseSchema extends BaseModel {
|
||||
static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'postedAt', 'poster', 'releaseKey', 'size', 'source'] as const
|
||||
$columns = ReleaseSchema.$columns
|
||||
@column()
|
||||
declare category: number | null
|
||||
@column()
|
||||
declare complete: boolean
|
||||
@column()
|
||||
declare container: string | null
|
||||
@column()
|
||||
declare contents: any | null
|
||||
@column()
|
||||
declare encrypted: boolean | null
|
||||
@column()
|
||||
declare fileCount: number
|
||||
@column()
|
||||
declare fileTotal: number | null
|
||||
@column.dateTime()
|
||||
declare firstSeenAt: DateTime
|
||||
@column()
|
||||
declare groups: any
|
||||
@column({ isPrimary: true })
|
||||
declare id: bigint | number
|
||||
@column.dateTime()
|
||||
declare lastSeenAt: DateTime
|
||||
@column()
|
||||
declare name: string
|
||||
@column({ serializeAs: null })
|
||||
declare password: string | null
|
||||
@column.dateTime()
|
||||
declare postedAt: DateTime | null
|
||||
@column()
|
||||
declare poster: string | null
|
||||
@column()
|
||||
declare releaseKey: string
|
||||
@column()
|
||||
declare size: bigint | number
|
||||
@column()
|
||||
declare source: string | null
|
||||
}
|
||||
|
||||
export class SpotSchema extends BaseModel {
|
||||
static $columns = ['articleNumber', 'category', 'createdAt', 'error', 'groupName', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'website'] as const
|
||||
$columns = SpotSchema.$columns
|
||||
@column()
|
||||
declare articleNumber: bigint | number | null
|
||||
@column()
|
||||
declare category: number | null
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column()
|
||||
declare error: string | null
|
||||
@column()
|
||||
declare groupName: string
|
||||
@column()
|
||||
declare keyId: number | null
|
||||
@column({ isPrimary: true })
|
||||
declare messageId: string
|
||||
@column.dateTime()
|
||||
declare postedAt: DateTime | null
|
||||
@column()
|
||||
declare poster: string
|
||||
@column.dateTime()
|
||||
declare processedAt: DateTime | null
|
||||
@column()
|
||||
declare releaseId: bigint | number | null
|
||||
@column()
|
||||
declare size: bigint | number | null
|
||||
@column()
|
||||
declare spotterId: string | null
|
||||
@column()
|
||||
declare status: string
|
||||
@column()
|
||||
declare subcats: any | null
|
||||
@column()
|
||||
declare tag: string | null
|
||||
@column()
|
||||
declare title: string
|
||||
@column()
|
||||
declare website: string | null
|
||||
}
|
||||
|
||||
export class UserSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
|
||||
$columns = UserSchema.$columns
|
||||
|
||||
Reference in new issue
Block a user