diff --git a/app/controllers/newznab_controller.ts b/app/controllers/newznab_controller.ts index d78d020..7ee24d6 100644 --- a/app/controllers/newznab_controller.ts +++ b/app/controllers/newznab_controller.ts @@ -101,7 +101,9 @@ export default class NewznabController { postedAt: r.postedAt ? new Date(r.postedAt) : null, poster: r.poster, groups: r.groups ?? [], - password: Boolean(r.encrypted), + // 1 only when a password is needed that we can't give; a known one is in our NZB. + password: r.passwordState === 'missing', + description: r.spotDescription ?? undefined, season: tv ? `S${tv[1].padStart(2, '0')}` : undefined, episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined, } diff --git a/app/controllers/releases_controller.ts b/app/controllers/releases_controller.ts index b777061..8c7a347 100644 --- a/app/controllers/releases_controller.ts +++ b/app/controllers/releases_controller.ts @@ -18,6 +18,8 @@ export default class ReleasesController { sort: qs.sort as SortOrder | undefined, page: Number(qs.page) || 1, perPage: Number(qs.per_page) || 50, + // The page shows "1,000+"; counting further made filter clicks take seconds. + countLimit: 1000, }) } @@ -37,6 +39,16 @@ export default class ReleasesController { return response.send(nzb.xml) } + /** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */ + async image({ params, response }: HttpContext) { + const image = await SearchService.spotImage(Number(params.id)) + if (!image) return response.notFound({ error: 'No image' }) + response.header('Content-Type', image.type) + // A spot's image never changes. + response.header('Cache-Control', 'public, max-age=604800, immutable') + return response.send(image.data) + } + /** GET /api/v1/stats */ async stats() { return SearchService.stats() diff --git a/app/models/file.ts b/app/models/file.ts index d0a8231..a82f5ed 100644 --- a/app/models/file.ts +++ b/app/models/file.ts @@ -1,6 +1,8 @@ import { DateTime } from 'luxon' import { column } from '@adonisjs/lucid/orm' import BaseModel from '#models/base_model' +import { loadedSegments, storedSegments } from '#services/FileStore' +import type { Segment } from '#lib/nzb/index' export default class File extends BaseModel { @column({ isPrimary: true }) @@ -21,8 +23,12 @@ export default class File extends BaseModel { @column() declare parts: number - @column({ columnName: 'message_ids' }) - declare messageIds: Record + /** Stored compressed (see FileStore); the model sees the list. */ + @column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) }) + declare segments: Segment[] + + @column({ columnName: 'first_message_id' }) + declare firstMessageId: string | null @column() declare groups: string[] diff --git a/app/services/ArchiveService.ts b/app/services/ArchiveService.ts index 980df7b..04a5726 100644 --- a/app/services/ArchiveService.ts +++ b/app/services/ArchiveService.ts @@ -7,7 +7,7 @@ import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import IngestService from '#services/IngestService' import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index' -import { fileStem, rarReleaseIdentity } from '#lib/release/index' +import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index' /** Stored per file and release; a listing of thousands of files is not useful. */ const MAX_ENTRIES = 500 @@ -59,7 +59,9 @@ class ArchiveService { update.encrypted = true } const better = releaseNameFromContents(info) - const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined + // A spot's title is kept (CONCEPT.md D29). + const renamed = + release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined if (renamed) update.name = renamed if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update) @@ -138,7 +140,7 @@ export async function renameFromStoredContents(): Promise { .select('id', 'name', 'source', 'contents') let renamed = 0 for (const release of releases) { - if (!looksRandom(release.name) && release.source !== 'post') continue + if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue const better = releaseNameFromContents({ entries: release.contents }) if (!better || better === release.name) continue await db.from('releases').where('id', release.id).update({ name: better }) @@ -147,11 +149,27 @@ export async function renameFromStoredContents(): Promise { return renamed } -/** Obfuscated names: one long token of mixed letters and digits, no separators. */ +/** + * Obfuscated names: one long token of mixed letters and digits, no + * separators, and the other generated shapes seen on Usenet (LOG.md + * 2026-10-01 (3)). Kept in step with SQL's is_readable_name(). + */ export function looksRandom(name: string): boolean { // An unknown short extension (e.g. ".bin") says nothing about the name. name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') + // VIDEO_TS, Sample, CD1: real words, but no name. + if (isGenericName(name)) return true if (/^[0-9a-f]{16,}$/i.test(name)) return true + // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time) + if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true + // UUIDs + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true + // 1650777167 (yEncBin), "83111763 n" (rclone uploads) + if (/^\d{6,}( n)?$/.test(name)) return true + // prvjr7hapd0i1ny14pz4, and rclone's long base32 names + if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true + // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts + if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 diff --git a/app/services/BackfillService.ts b/app/services/BackfillService.ts new file mode 100644 index 0000000..82e4b77 --- /dev/null +++ b/app/services/BackfillService.ts @@ -0,0 +1,104 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import indexerConfig from '#config/indexer' +import NntpService from '#services/NntpService' +import FetchRangeService from '#services/FetchRangeService' +import { articleAtDate } from '#lib/nntp/date_search' + +/** Articles XOVERed to find a posting date near an article number. */ +const PROBE = 200n + +/** + * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from + * where live indexing began back to the article posted N days ago; `tick` + * (every scheduler run) queues low-priority, header-only fetch ranges from + * that cursor downwards, newest first, while few are waiting and the + * database stays within its size limits. + */ +class BackfillService { + /** Sets the group's backfill to reach back `days` days. Returns its range. */ + async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> { + const group = await db.from('groups').where('name', groupName).first() + if (!group) throw new Error(`Unknown group ${groupName}`) + const conn = await NntpService.acquire() + let failed = false + try { + const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`) + const dateOf = async (n: bigint) => { + const { overviews } = await NntpService.timed(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`) + const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1] + const date = header ? Date.parse(header.date) : Number.NaN + return Number.isNaN(date) ? null : date + } + const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf) + // Live indexing began a little before the group was added: continue from there. + const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf) + const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began + await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() }) + return next > until ? { from: until, to: next } : null + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + } + + /** Stops a group's backfill (queued ranges still run). */ + async stop(groupName: string) { + await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null }) + } + + /** Queues the next backfill ranges. Returns how many. */ + async tick(): Promise { + const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill + const groups = await db + .from('groups') + .where('active', true) + .whereNotNull('backfill_until_id') + .whereRaw('backfill_next_id > backfill_until_id') + .orderBy('id') + if (groups.length === 0) return 0 + + const room = await this.room(maxDefaultGb, maxHddGb) + if (room !== true) { + logger.warn(`Backfill paused: ${room}.`) + return 0 + } + const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count') + let free = maxQueued - Number(count) + let queued = 0 + // Round-robin over the groups, one range each per pass. + while (free > 0 && groups.length > 0) { + for (const group of [...groups]) { + if (free <= 0) break + const next = BigInt(group.backfill_next_id) + const until = BigInt(group.backfill_until_id) + const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n + await FetchRangeService.queue(group, start, next, { backfill: true }) + group.backfill_next_id = (start - 1n).toString() + await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id }) + free-- + queued++ + if (start - 1n <= until) { + groups.splice(groups.indexOf(group), 1) + logger.info(`${group.name}: backfill fully queued.`) + } + } + } + return queued + } + + /** True when both parts of the database are under their limits, else why not. */ + private async room(maxDefaultGb: number, maxHddGb: number): Promise { + const { rows } = await db.rawQuery( + `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')` + ) + const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3 + if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})` + if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})` + return true + } +} + +export default new BackfillService() diff --git a/app/services/ContentService.ts b/app/services/ContentService.ts index 9ed221e..d0b47e2 100644 --- a/app/services/ContentService.ts +++ b/app/services/ContentService.ts @@ -6,10 +6,25 @@ import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '# import NameService from '#services/NameService' import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index' import NzbImportService, { type NzbImportResult } from '#services/NzbImportService' -import { sniffContentType, type ContentType } from '#lib/content/index' +import { + isArchiveStart, + NICOVIDEO_FILE, + NICOVIDEO_RELEASE, + nicovideoInfo, + nicovideoReleaseName, + sniffContentType, + zstdPrefix, + type ContentType, + type NicovideoInfo, +} from '#lib/content/index' import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index' -import { fileStem } from '#lib/release/index' +import { fileStem, releaseIdentity } from '#lib/release/index' +import IngestService from '#services/IngestService' import { looksLikeFilename } from '#lib/subject/index' +import { sevenZipVolume } from '#lib/sevenzip/index' +import SevenZipService from '#services/SevenZipService' +import SfvService from '#services/SfvService' +import { segmentsOf } from '#services/FileStore' const FIRST_CHUNK = 16 * 1024 @@ -35,6 +50,12 @@ export interface ContentResult { nzb?: NzbImportResult /** NZBs read from inside an archive. */ archiveNzbs?: NzbImportResult[] + /** Files an SFV's listing moved into its release. */ + sfvMoved?: number + /** A niconico capture's release name from its API response (D34). */ + nicovideo?: string + /** The real name from the yEnc header, for a file posted under a random subject (D36). */ + yencName?: string } /** @@ -48,14 +69,65 @@ class ContentService { /** Random or missing names, and named first RAR volumes (for contents and password flag). */ isCandidate(name: string) { if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true + if (isArchiveStart(name)) return true if (/\.nzb$/i.test(name)) return true + // SFVs name the files of their release; niconico captures carry their title (D34). + if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true + // Every 7z volume: the set is listed when its last volume is in (see inspect). + if (sevenZipVolume(name) !== null) return true return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) } + /** + * Uploads that randomise the subject per article often keep the real file + * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`; + * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and + * leaves its weak one-file release for the release its name gives, where + * the other files of the upload gather. Returns the name, or undefined. + */ + private async nameFromYenc(file: any, binary: any, yencName: string): Promise { + const current = String(file.real_name ?? file.filename) + if (file.real_name || !looksRandom(fileStem(current))) return undefined + if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined + + const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null + const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1) + let releaseId = file.release_id + if (weak) { + const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true }) + if (identity) { + const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [ + { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true }, + ]) + releaseId = ids.get(identity.key) ?? releaseId + } + } + await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId }) + if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId }) + return yencName + } + + /** + * Names a niconico capture's release after the video, while it still has + * the archiver's `