diff --git a/database/migrations/1770000000100_create_groups.ts b/database/migrations/1770000000100_create_groups.ts new file mode 100644 index 0000000..37d3136 --- /dev/null +++ b/database/migrations/1770000000100_create_groups.ts @@ -0,0 +1,27 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The indexed groups (REBUILD.md, group list). `priority` decides a + * crosspost's owner: the indexed group in its Xref with the lowest number. + * New groups get numbers after the existing ones, so they never take + * articles from groups already done. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('groups', (table) => { + table.increments('id') + table.string('name', 255).notNullable().unique() + table.integer('priority').notNullable().unique() + table.boolean('active').notNullable().defaultTo(true) + // From GROUP when planned: the server's oldest and newest article. + table.bigInteger('low').nullable() + table.bigInteger('high').nullable() + table.timestamp('planned_at', { useTz: true }).nullable() + table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now()) + }) + } + + async down() { + this.schema.dropTable('groups') + } +} diff --git a/database/migrations/1770000000200_create_rule_sets.ts b/database/migrations/1770000000200_create_rule_sets.ts new file mode 100644 index 0000000..0b57a3e --- /dev/null +++ b/database/migrations/1770000000200_create_rule_sets.ts @@ -0,0 +1,21 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Versions of what workers drop (REBUILD.md, rules and recovery): dropped + * signatures and tools, the per-article limit, the sample share. Every range + * records the version it ran with. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('rule_sets', (table) => { + table.integer('version').primary() + table.jsonb('rules').notNullable() + table.text('note').nullable() + table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now()) + }) + } + + async down() { + this.schema.dropTable('rule_sets') + } +} diff --git a/database/migrations/1770000000300_create_ranges.ts b/database/migrations/1770000000300_create_ranges.ts new file mode 100644 index 0000000..f466d81 --- /dev/null +++ b/database/migrations/1770000000300_create_ranges.ts @@ -0,0 +1,38 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A group's history in leasable pieces (REBUILD.md, coordinator): + * planned → (spooled by the prefetcher) → leased → done. `next_id` is the + * resume point: everything before it has been stored. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('ranges', (table) => { + table.bigIncrements('id') + table.integer('group_id').notNullable().references('id').inTable('groups').onDelete('CASCADE') + table.bigInteger('start_id').notNullable() + table.bigInteger('end_id').notNullable() + table.bigInteger('next_id').notNullable() + table.string('status', 8).notNullable().defaultTo('planned') + // Lower first; the coordinator plans the oldest era first. + table.integer('priority').notNullable().defaultTo(0) + table.integer('rule_version').nullable() + table.integer('node_id').nullable() + table.integer('attempts').notNullable().defaultTo(0) + // Chunks of this range in the spool (prefetch); all of them when spooled_at is set. + table.integer('spooled_chunks').notNullable().defaultTo(0) + table.timestamp('spooled_at', { useTz: true }).nullable() + table.timestamp('leased_at', { useTz: true }).nullable() + table.timestamp('heartbeat_at', { useTz: true }).nullable() + table.timestamp('done_at', { useTz: true }).nullable() + table.bigInteger('headers').notNullable().defaultTo(0) + table.text('error').nullable() + table.unique(['group_id', 'start_id']) + }) + this.schema.raw(`CREATE INDEX ranges_todo_idx ON ranges (priority, id) WHERE status IN ('planned', 'leased')`) + } + + async down() { + this.schema.dropTable('ranges') + } +} diff --git a/database/migrations/1770000000400_create_releases.ts b/database/migrations/1770000000400_create_releases.ts new file mode 100644 index 0000000..6553fa0 --- /dev/null +++ b/database/migrations/1770000000400_create_releases.ts @@ -0,0 +1,41 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The release catalog (REBUILD.md, part 4): one row per release, the hot + * part on fast disk. Updated with aggregates once per batch; the files + * themselves are in release_chunks. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('releases', (table) => { + table.bigIncrements('id') + table.string('release_key', 40).notNullable().unique() + table.text('name').notNullable() + table.string('source', 16).notNullable() + table.text('poster').nullable() + table.jsonb('groups').notNullable() + table.timestamp('posted_at', { useTz: true }).nullable() + // Files announced by the [i/N] counter, when present. + table.integer('file_total').nullable() + table.integer('files').notNullable().defaultTo(0) + table.bigInteger('segments').notNullable().defaultTo(0) + table.bigInteger('bytes').notNullable().defaultTo(0) + table.integer('chunks').notNullable().defaultTo(0) + table.bigInteger('first_article').nullable() + table.bigInteger('last_article').nullable() + table.boolean('readable').notNullable().defaultTo(false) + table.boolean('has_par2').notNullable().defaultTo(false) + table.boolean('complete').nullable() + table.timestamp('settled_at', { useTz: true }).nullable() + table.timestamp('first_seen_at', { useTz: true }).notNullable().defaultTo(this.now()) + table.timestamp('last_seen_at', { useTz: true }).notNullable().defaultTo(this.now()) + }) + this.schema.raw(`CREATE EXTENSION IF NOT EXISTS pg_trgm`) + this.schema.raw(`CREATE INDEX releases_name_trgm ON releases USING gin (name gin_trgm_ops)`) + this.schema.raw(`CREATE INDEX releases_posted_at_idx ON releases (posted_at DESC)`) + } + + async down() { + this.schema.dropTable('releases') + } +} diff --git a/database/migrations/1770000000500_create_release_chunks.ts b/database/migrations/1770000000500_create_release_chunks.ts new file mode 100644 index 0000000..9c0a387 --- /dev/null +++ b/database/migrations/1770000000500_create_release_chunks.ts @@ -0,0 +1,34 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' +import env from '#start/env' + +/** + * The chunk store (REBUILD.md, part 3; decision a): one write-once row per + * release per batch, its files encoded by lib/chunks. Partitioned by month + * of writing, in CHUNK_TABLESPACE (the HDD) when set; ChunkPartitions keeps + * the next months' partitions ready. A re-run of a range replaces its chunks + * (by range_id). + */ +export default class extends BaseSchema { + async up() { + const tablespace = env.get('CHUNK_TABLESPACE') + const where = tablespace ? ` TABLESPACE ${tablespace}` : '' + this.schema.raw(` + CREATE TABLE release_chunks ( + id bigserial, + release_id bigint NOT NULL, + range_id bigint NOT NULL, + batch_seq integer NOT NULL, + files integer NOT NULL, + segments integer NOT NULL, + payload bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at)${where}`) + this.schema.raw(`CREATE INDEX release_chunks_release_idx ON release_chunks (release_id)`) + this.schema.raw(`CREATE INDEX release_chunks_range_idx ON release_chunks (range_id)`) + } + + async down() { + this.schema.raw(`DROP TABLE IF EXISTS release_chunks CASCADE`) + } +} diff --git a/database/migrations/1770000000600_create_fragments.ts b/database/migrations/1770000000600_create_fragments.ts new file mode 100644 index 0000000..a020974 --- /dev/null +++ b/database/migrations/1770000000600_create_fragments.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Segments of files cut off by a range's edge (REBUILD.md, fragments): kept + * by range, merged when the neighbouring range is done, then written as a + * chunk. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('fragments', (table) => { + table.bigIncrements('id') + table.bigInteger('range_id').notNullable() + table.string('file_key', 40).notNullable() + table.bigInteger('article_number').notNullable() + table.jsonb('segment').notNullable() + table.index(['file_key']) + table.index(['range_id']) + }) + } + + async down() { + this.schema.dropTable('fragments') + } +} diff --git a/database/migrations/1770000000700_create_probe_candidates.ts b/database/migrations/1770000000700_create_probe_candidates.ts new file mode 100644 index 0000000..c075187 --- /dev/null +++ b/database/migrations/1770000000700_create_probe_candidates.ts @@ -0,0 +1,25 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The probe queue (REBUILD.md, part 6): per release the few articles phase 2 + * should read. Phase 1 only fills it. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('probe_candidates', (table) => { + table.bigIncrements('id') + table.bigInteger('release_id').notNullable() + table.string('kind', 8).notNullable() + table.text('message_id').notNullable() + table.text('file').notNullable() + table.string('status', 8).notNullable().defaultTo('waiting') + table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now()) + table.unique(['release_id', 'kind', 'message_id']) + }) + this.schema.raw(`CREATE INDEX probe_candidates_waiting_idx ON probe_candidates (id) WHERE status = 'waiting'`) + } + + async down() { + this.schema.dropTable('probe_candidates') + } +} diff --git a/database/migrations/1770000000800_create_rule_bookkeeping.ts b/database/migrations/1770000000800_create_rule_bookkeeping.ts new file mode 100644 index 0000000..caca523 --- /dev/null +++ b/database/migrations/1770000000800_create_rule_bookkeeping.ts @@ -0,0 +1,35 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * What the rules dropped, per range (REBUILD.md, rules and recovery), and + * the sample of it kept in full. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('rule_counts', (table) => { + table.bigInteger('range_id').notNullable() + table.string('rule', 12).notNullable() + table.text('key').notNullable() + table.bigInteger('articles').notNullable().defaultTo(0) + table.bigInteger('files').notNullable().defaultTo(0) + table.primary(['range_id', 'rule', 'key']) + }) + this.schema.createTable('rule_samples', (table) => { + table.bigIncrements('id') + table.bigInteger('range_id').notNullable() + table.string('rule', 12).notNullable() + table.text('rule_key').notNullable() + table.string('release_key', 40).notNullable() + table.text('name').notNullable() + table.integer('files').notNullable() + table.integer('segments').notNullable() + table.binary('payload').notNullable() + table.index(['rule', 'rule_key']) + }) + } + + async down() { + this.schema.dropTable('rule_samples') + this.schema.dropTable('rule_counts') + } +} diff --git a/database/migrations/1770000000900_create_nodes.ts b/database/migrations/1770000000900_create_nodes.ts new file mode 100644 index 0000000..d6fd437 --- /dev/null +++ b/database/migrations/1770000000900_create_nodes.ts @@ -0,0 +1,30 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** Edge workers and their API tokens (the `node` guard). */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('nodes', (table) => { + table.increments('id') + table.string('name', 64).notNullable().unique() + table.timestamp('last_seen_at', { useTz: true }).nullable() + table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now()) + }) + this.schema.createTable('node_access_tokens', (table) => { + table.increments('id') + table.integer('tokenable_id').notNullable().unsigned().references('id').inTable('nodes').onDelete('CASCADE') + table.string('type').notNullable() + table.string('name').nullable() + table.string('hash').notNullable() + table.text('abilities').notNullable() + table.timestamp('created_at') + table.timestamp('updated_at') + table.timestamp('last_used_at').nullable() + table.timestamp('expires_at').nullable() + }) + } + + async down() { + this.schema.dropTable('node_access_tokens') + this.schema.dropTable('nodes') + } +} diff --git a/database/schema.ts b/database/schema.ts index e1e2811..a3dafb8 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -32,6 +32,957 @@ export class AuthAccessTokenSchema extends BaseModel { declare updatedAt: DateTime | null } +export class AuthAccessTokenSchema extends BaseModel { + static $columns = ['abilities', 'createdAt', 'expiresAt', 'hash', 'id', 'lastUsedAt', 'name', 'tokenableId', 'type', 'updatedAt'] as const + $columns = AuthAccessTokenSchema.$columns + @column() + declare abilities: string + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column.dateTime() + declare expiresAt: DateTime | null + @column() + declare hash: string + @column({ isPrimary: true }) + declare id: number + @column.dateTime() + declare lastUsedAt: DateTime | null + @column() + declare name: string | null + @column() + declare tokenableId: number + @column() + declare type: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class BinarySchema extends BaseModel { + static $columns = ['awaitingInspection', 'completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const + $columns = BinarySchema.$columns + @column() + declare awaitingInspection: boolean | null + @column.dateTime() + declare completedAt: DateTime | null + @column.dateTime() + declare conflictedAt: DateTime | null + @column() + declare fileId: bigint | number | null + @column() + declare fileKey: string + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groups: any + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare md516K: string | null + @column() + declare name: string + @column() + declare parentBinaryId: bigint | number | null + @column() + declare partCount: number | null + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare size: bigint | number | null + @column.dateTime() + declare splitAt: DateTime | null + @column() + declare subject: string + @column() + declare totalParts: number +} + +export class DeferredFileSchema extends BaseModel { + static $columns = ['articleCount', 'bytes', 'contentType', 'encrypted', 'error', 'fileKey', 'firstArticle', 'firstMessageId', 'firstSeenAt', 'groupName', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'processedAt', 'sampledPart', 'size', 'status', 'subject', 'totalParts', 'yencName'] as const + $columns = DeferredFileSchema.$columns + @column() + declare articleCount: number + @column() + declare bytes: bigint | number + @column() + declare contentType: string | null + @column() + declare encrypted: boolean | null + @column() + declare error: string | null + @column() + declare fileKey: string + @column() + declare firstArticle: bigint | number + @column() + declare firstMessageId: string + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groupName: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare lastArticle: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare name: string + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column.dateTime() + declare processedAt: DateTime | null + @column() + declare sampledPart: number | null + @column() + declare size: bigint | number | null + @column() + declare status: string + @column() + declare subject: string + @column() + declare totalParts: number | null + @column() + declare yencName: string | null +} + +export class DeferredRangeSchema extends BaseModel { + static $columns = ['articleCount', 'bytes', 'createdAt', 'expandedAt', 'firstArticle', 'groupName', 'id', 'lastArticle'] as const + $columns = DeferredRangeSchema.$columns + @column() + declare articleCount: number + @column() + declare bytes: bigint | number + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column.dateTime() + declare expandedAt: DateTime | null + @column() + declare firstArticle: bigint | number + @column() + declare groupName: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare lastArticle: bigint | number +} + +export class DroppedReleaseSchema extends BaseModel { + static $columns = ['droppedAt', 'reason', 'releaseKey'] as const + $columns = DroppedReleaseSchema.$columns + @column.dateTime() + declare droppedAt: DateTime + @column() + declare reason: string + @column({ isPrimary: true }) + declare releaseKey: string +} + +export class FetchRangeSchema extends BaseModel { + static $columns = ['backfill', 'createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const + $columns = FetchRangeSchema.$columns + @column() + declare backfill: boolean + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare endId: bigint | number + @column() + declare error: string | null + @column() + declare groupId: number + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare requeues: number + @column() + declare startId: bigint | number + @column() + declare status: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime +} + +export class FileDetailSchema extends BaseModel { + static $columns = ['archive', 'contentType', 'fileId'] as const + $columns = FileDetailSchema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null + @column({ isPrimary: true }) + declare fileId: bigint | number +} + +export class FileKeySchema extends BaseModel { + static $columns = ['keyHash'] as const + $columns = FileKeySchema.$columns + @column({ isPrimary: true }) + declare keyHash: bigint | number +} + +export class FileKeysFillSchema extends BaseModel { + static $columns = ['complete', 'id', 'nextId', 'untilId', 'updatedAt'] as const + $columns = FileKeysFillSchema.$columns + @column() + declare complete: boolean + @column({ isPrimary: true }) + declare id: number + @column() + declare nextId: bigint | number + @column() + declare untilId: bigint | number | null + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime +} + +export class FileSchema extends BaseModel { + static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const + $columns = FileSchema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column() + declare date: bigint | number + @column() + declare fileKey: string | null + @column() + declare filename: string + @column() + declare firstMessageId: string | null + @column() + declare groups: any | null + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare parts: number + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare segments: Buffer + @column() + declare size: bigint | number | null + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class FilesP0Schema extends BaseModel { + static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const + $columns = FilesP0Schema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column() + declare date: bigint | number + @column() + declare fileKey: string | null + @column() + declare filename: string + @column() + declare firstMessageId: string | null + @column() + declare groups: any | null + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare parts: number + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare segments: Buffer + @column() + declare size: bigint | number | null + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class FilesP1000000Schema extends BaseModel { + static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const + $columns = FilesP1000000Schema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column() + declare date: bigint | number + @column() + declare fileKey: string | null + @column() + declare filename: string + @column() + declare firstMessageId: string | null + @column() + declare groups: any | null + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare parts: number + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare segments: Buffer + @column() + declare size: bigint | number | null + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class FilesP5000000Schema extends BaseModel { + static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const + $columns = FilesP5000000Schema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column() + declare date: bigint | number + @column() + declare fileKey: string | null + @column() + declare filename: string + @column() + declare firstMessageId: string | null + @column() + declare groups: any | null + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare parts: number + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare segments: Buffer + @column() + declare size: bigint | number | null + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class FragmentSchema extends BaseModel { + static $columns = ['articleNumber', 'fileKey', 'id', 'rangeId', 'segment'] as const + $columns = FragmentSchema.$columns + @column() + declare articleNumber: bigint | number + @column() + declare fileKey: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare rangeId: bigint | number + @column() + declare segment: any +} + +export class GroupSchema extends BaseModel { + static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority'] as const + $columns = GroupSchema.$columns + @column() + declare active: boolean + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare high: bigint | number | null + @column({ isPrimary: true }) + declare id: number + @column() + declare low: bigint | number | null + @column() + declare name: string + @column.dateTime() + declare plannedAt: DateTime | null + @column() + declare priority: number +} + +export class GroupSchema extends BaseModel { + static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const + $columns = GroupSchema.$columns + @column() + declare active: boolean + @column() + declare backfillNextId: bigint | number | null + @column() + declare backfillUntilId: bigint | number | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column() + declare fetchBodies: boolean + @column({ isPrimary: true }) + declare id: number + @column() + declare lastIndexedId: bigint | number | null + @column() + declare lastQueuedId: bigint | number + @column() + declare name: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class InspectionQueueSchema extends BaseModel { + static $columns = ['fileId', 'priority', 'queuedAt'] as const + $columns = InspectionQueueSchema.$columns + @column({ isPrimary: true }) + declare fileId: bigint | number + @column() + declare priority: number + @column.dateTime() + declare queuedAt: DateTime +} + +export class NodeAccessTokenSchema extends BaseModel { + static $columns = ['abilities', 'createdAt', 'expiresAt', 'hash', 'id', 'lastUsedAt', 'name', 'tokenableId', 'type', 'updatedAt'] as const + $columns = NodeAccessTokenSchema.$columns + @column() + declare abilities: string + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column.dateTime() + declare expiresAt: DateTime | null + @column() + declare hash: string + @column({ isPrimary: true }) + declare id: number + @column.dateTime() + declare lastUsedAt: DateTime | null + @column() + declare name: string | null + @column() + declare tokenableId: number + @column() + declare type: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + +export class NodeSchema extends BaseModel { + static $columns = ['createdAt', 'id', 'lastSeenAt', 'name'] as const + $columns = NodeSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column({ isPrimary: true }) + declare id: number + @column.dateTime() + declare lastSeenAt: DateTime | null + @column() + declare name: string +} + +export class PartConflictSchema extends BaseModel { + static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const + $columns = PartConflictSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column({ isPrimary: true }) + declare binaryId: bigint | number + @column() + declare bytes: number + @column() + declare fileCrc32: string | null + @column() + declare md516K: string | null + @column() + declare messageId: string + @column() + declare number: number + @column() + declare partSize: number | null + @column() + declare pcrc32: string | null +} + +export class PartSchema extends BaseModel { + static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const + $columns = PartSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column({ isPrimary: true }) + declare binaryId: bigint | number + @column() + declare bytes: number + @column() + declare fileCrc32: string | null + @column() + declare md516K: string | null + @column() + declare messageId: string + @column() + declare number: number + @column() + declare partSize: number | null + @column() + declare pcrc32: string | null +} + +export class ProbeCandidateSchema extends BaseModel { + static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const + $columns = ProbeCandidateSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare file: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare kind: string + @column() + declare messageId: string + @column() + declare releaseId: bigint | number + @column() + declare status: string +} + +export class RangeSchema extends BaseModel { + static $columns = ['attempts', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const + $columns = RangeSchema.$columns + @column() + declare attempts: number + @column.dateTime() + declare doneAt: DateTime | null + @column() + declare endId: bigint | number + @column() + declare error: string | null + @column() + declare groupId: number + @column() + declare headers: bigint | number + @column.dateTime() + declare heartbeatAt: DateTime | null + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare leasedAt: DateTime | null + @column() + declare nextId: bigint | number + @column() + declare nodeId: number | null + @column() + declare priority: number + @column() + declare ruleVersion: number | null + @column.dateTime() + declare spooledAt: DateTime | null + @column() + declare spooledChunks: number + @column() + declare startId: bigint | number + @column() + declare status: string +} + +export class ReleaseChunkSchema extends BaseModel { + static $columns = ['batchSeq', 'createdAt', 'files', 'id', 'payload', 'rangeId', 'releaseId', 'segments'] as const + $columns = ReleaseChunkSchema.$columns + @column() + declare batchSeq: number + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare files: number + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare payload: Buffer + @column() + declare rangeId: bigint | number + @column() + declare releaseId: bigint | number + @column() + declare segments: number +} + +export class ReleaseDirtySchema extends BaseModel { + static $columns = ['releaseId'] as const + $columns = ReleaseDirtySchema.$columns + @column() + declare releaseId: bigint | number +} + +export class ReleaseFileSchema extends BaseModel { + static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const + $columns = ReleaseFileSchema.$columns + @column() + declare md5: string | null + @column() + declare md516K: string + @column() + declare name: string + @column() + declare par2FileId: string + @column({ isPrimary: true }) + declare releaseId: bigint | number + @column() + declare setId: string | null + @column() + declare size: bigint | number +} + +export class ReleaseSchema extends BaseModel { + static $columns = ['bytes', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groups', 'hasPar2', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const + $columns = ReleaseSchema.$columns + @column() + declare bytes: bigint | number + @column() + declare chunks: number + @column() + declare complete: boolean | null + @column() + declare fileTotal: number | null + @column() + declare files: number + @column() + declare firstArticle: bigint | number | null + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groups: any + @column() + declare hasPar2: boolean + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare lastArticle: bigint | number | null + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare name: string + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string | null + @column() + declare readable: boolean + @column() + declare releaseKey: string + @column() + declare segments: bigint | number + @column.dateTime() + declare settledAt: DateTime | null + @column() + declare source: string +} + +export class ReleaseSchema extends BaseModel { + static $columns = ['category', 'complete', 'container', 'contents', 'contentsWalkedAt', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const + $columns = ReleaseSchema.$columns + @column() + declare category: number | null + @column() + declare complete: boolean + @column() + declare container: string | null + @column() + declare contents: any | null + @column.dateTime() + declare contentsWalkedAt: DateTime | null + @column() + declare encrypted: boolean | null + @column() + declare extensions: any | null + @column() + declare fileCount: number + @column() + declare fileTotal: number | null + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groups: any + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare name: string + @column({ serializeAs: null }) + declare password: string | null + @column() + declare passwordSource: string | null + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string | null + @column() + declare releaseKey: string + @column() + declare searchFiles: string | null + @column() + declare searchText: string | null + @column() + declare size: bigint | number + @column() + declare source: string | null +} + +export class RuleCountSchema extends BaseModel { + static $columns = ['articles', 'files', 'key', 'rangeId', 'rule'] as const + $columns = RuleCountSchema.$columns + @column() + declare articles: bigint | number + @column() + declare files: bigint | number + @column() + declare key: string + @column({ isPrimary: true }) + declare rangeId: bigint | number + @column() + declare rule: string +} + +export class RuleSampleSchema extends BaseModel { + static $columns = ['files', 'id', 'name', 'payload', 'rangeId', 'releaseKey', 'rule', 'ruleKey', 'segments'] as const + $columns = RuleSampleSchema.$columns + @column() + declare files: number + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare name: string + @column() + declare payload: Buffer + @column() + declare rangeId: bigint | number + @column() + declare releaseKey: string + @column() + declare rule: string + @column() + declare ruleKey: string + @column() + declare segments: number +} + +export class RuleSetSchema extends BaseModel { + static $columns = ['createdAt', 'note', 'rules', 'version'] as const + $columns = RuleSetSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare note: string | null + @column() + declare rules: any + @column({ isPrimary: true }) + declare version: number +} + +export class SeedNodeSchema extends BaseModel { + static $columns = ['createdAt', 'disabled', 'id', 'lastSeenAt', 'name', 'tokenHash'] as const + $columns = SeedNodeSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare disabled: boolean + @column({ isPrimary: true }) + declare id: number + @column.dateTime() + declare lastSeenAt: DateTime | null + @column() + declare name: string + @column() + declare tokenHash: string +} + +export class SeedRangeSchema extends BaseModel { + static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'startId', 'status'] as const + $columns = SeedRangeSchema.$columns + @column() + declare attempts: number + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column.dateTime() + declare doneAt: DateTime | null + @column() + declare endId: bigint | number + @column() + declare error: string | null + @column() + declare files: bigint | number + @column() + declare fragments: bigint | number + @column() + declare groupName: string + @column() + declare headers: bigint | number + @column.dateTime() + declare heartbeatAt: DateTime | null + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare leasedAt: DateTime | null + @column() + declare nextId: bigint | number + @column() + declare nodeId: number | null + @column() + declare priority: number + @column() + declare startId: bigint | number + @column() + declare status: string +} + +export class SfvEntrySchema extends BaseModel { + static $columns = ['crc', 'name', 'postedAt', 'poster', 'releaseId'] as const + $columns = SfvEntrySchema.$columns + @column() + declare crc: string + @column() + declare name: string + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string | null + @column({ isPrimary: true }) + declare releaseId: bigint | number +} + +export class SignatureProbeSchema extends BaseModel { + static $columns = ['groupName', 'probedAt', 'releaseId', 'tool'] as const + $columns = SignatureProbeSchema.$columns + @column() + declare groupName: string + @column.dateTime() + declare probedAt: DateTime + @column({ isPrimary: true }) + declare releaseId: bigint | number + @column() + declare tool: string +} + +export class SpotSchema extends BaseModel { + static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'nzbReadAt', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const + $columns = SpotSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column() + declare category: number | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare description: string | null + @column.dateTime() + declare detailsAt: DateTime | null + @column() + declare error: string | null + @column() + declare groupName: string + @column() + declare image: any | null + @column() + declare keyId: number | null + @column({ isPrimary: true }) + declare messageId: string + @column.dateTime() + declare nzbReadAt: DateTime | null + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column.dateTime() + declare processedAt: DateTime | null + @column() + declare releaseId: bigint | number | null + @column() + declare size: bigint | number | null + @column() + declare spotterId: string | null + @column() + declare status: string + @column() + declare subcats: any | null + @column() + declare tag: string | null + @column() + declare title: string + @column() + declare verified: boolean | null + @column() + declare website: string | null +} + +export class StagedSegmentSchema extends BaseModel { + static $columns = ['counterKey', 'fileKey', 'segments', 'stagedAt'] as const + $columns = StagedSegmentSchema.$columns + @column() + declare counterKey: string | null + @column({ isPrimary: true }) + declare fileKey: string + @column() + declare segments: any + @column.dateTime() + declare stagedAt: DateTime +} + +export class UploadSignatureSchema extends BaseModel { + static $columns = ['dropped', 'groupName', 'named', 'probed', 'tool', 'updatedAt'] as const + $columns = UploadSignatureSchema.$columns + @column() + declare dropped: boolean + @column() + declare groupName: string + @column() + declare named: number + @column() + declare probed: number + @column({ isPrimary: true }) + declare tool: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime +} + +export class UserSchema extends BaseModel { + static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const + $columns = UserSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare email: string + @column() + declare fullName: string | null + @column({ isPrimary: true }) + declare id: number + @column({ serializeAs: null }) + declare password: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + export class UserSchema extends BaseModel { static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const $columns = UserSchema.$columns diff --git a/start/env.ts b/start/env.ts index 358c134..362f1dd 100644 --- a/start/env.ts +++ b/start/env.ts @@ -33,6 +33,8 @@ export default await Env.create(new URL('../', import.meta.url), { DB_DATABASE: Env.schema.string(), DB_POOL_MAX: Env.schema.number.optional(), DB_SEARCH_PATH: Env.schema.string.optional(), + // Tablespace for the chunk store (the HDD on homelab). + CHUNK_TABLESPACE: Env.schema.string.optional(), DB_DEBUG: Env.schema.boolean.optional(), // NNTP (the coordinator's relay; edge:bench --capture)