From 92480174b9ff4d435c1b79a254349319f3307dc2 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 11 Oct 2026 19:33:23 +0200 Subject: [PATCH] Search and the web interface use the prober's findings - Search matches a word in the name or the inner name (trigram index on inner_name, partial, built concurrently); "Readable names" includes releases with an inner name. - unlocked=1 (chip "No password") leaves out passworded releases. - Results carry passworded and innerName: a Password badge, the inner name under the release name. - The release view: Inside and Inspected facts, a passworded notice, and Contents (per probed article its yEnc name, kind, type, encryption and the names inside) from release_inspections. - Newznab: the password attribute from the prober; an obfuscated release with an inner name is included, titled by it (without its extension). Co-Authored-By: Claude Opus 5.5 --- app/controllers/newznab_controller.ts | 9 ++-- app/controllers/releases_controller.ts | 3 +- app/services/search_service.ts | 31 +++++++++--- .../1770000001900_releases_inner_name_trgm.ts | 18 +++++++ public/app.css | 49 +++++++++++++++++++ public/app.js | 48 ++++++++++++++++-- public/index.html | 2 + tests/functional/releases_api.spec.ts | 33 +++++++++++++ 8 files changed, 177 insertions(+), 16 deletions(-) create mode 100644 database/migrations/1770000001900_releases_inner_name_trgm.ts diff --git a/app/controllers/newznab_controller.ts b/app/controllers/newznab_controller.ts index 0f17e02..92f6720 100644 --- a/app/controllers/newznab_controller.ts +++ b/app/controllers/newznab_controller.ts @@ -32,7 +32,7 @@ function int(value: unknown): number | undefined { /** * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1): * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=… - * Releases with readable names that aren't known to miss files. + * Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files. */ export default class NewznabController { async handle({ request, response }: HttpContext) { @@ -91,15 +91,16 @@ export default class NewznabController { const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name) return { id: r.id, - title: r.name, + // An obfuscated release by what the prober found inside, without its extension (what clients parse). + title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''), category: r.category ?? 8010, size: r.size, files: r.fileCount, postedAt: r.postedAt ? new Date(r.postedAt) : null, poster: r.poster, groups: r.groups ?? [], - // Passwords come with phase 2 (archive inspection). - password: false, + // From the prober (phase 2): encrypted archive entries or headers. + password: r.passworded === true, season: tv ? `S${tv[1].padStart(2, '0')}` : undefined, episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined, } diff --git a/app/controllers/releases_controller.ts b/app/controllers/releases_controller.ts index b551e05..0dda7c5 100644 --- a/app/controllers/releases_controller.ts +++ b/app/controllers/releases_controller.ts @@ -13,7 +13,7 @@ function megabytes(value: unknown): number | undefined { /** The catalog for the web interface (REBUILD.md, part 9). */ export default class ReleasesController { - /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */ + /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */ async index({ request }: HttpContext) { const qs = request.qs() const categories = String(qs.cat ?? '') @@ -25,6 +25,7 @@ export default class ReleasesController { categories: categories.length ? expandCategories(categories) : undefined, readable: flag(qs.readable), complete: flag(qs.complete), + unlocked: flag(qs.unlocked), source: typeof qs.source === 'string' && qs.source ? qs.source : undefined, poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined, minSize: megabytes(qs.min_size), diff --git a/app/services/search_service.ts b/app/services/search_service.ts index 3dc0c88..53c9131 100644 --- a/app/services/search_service.ts +++ b/app/services/search_service.ts @@ -7,12 +7,14 @@ export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' /** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */ export interface SearchParams { q?: string - /** Only releases with a file whose name is readable (not an obfuscated token). */ + /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */ readable?: boolean /** Only releases known to have all their files. */ complete?: boolean /** Leave out releases known to miss files (Newznab). */ notIncomplete?: boolean + /** Leave out releases the prober found passworded (encrypted archive entries or headers). */ + unlocked?: boolean /** How the release was recognised: stem, subject or post. */ source?: string poster?: string @@ -49,6 +51,10 @@ export interface ReleaseSummary { complete: boolean | null category: number settled: boolean + /** The prober found encrypted archive entries or headers; null: not probed (yet). */ + passworded: boolean | null + /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */ + innerName: string | null } const SORTS: Record = { @@ -75,14 +81,15 @@ class SearchService { async search(params: SearchParams) { const where: string[] = ['r.files > 0'] const bindings: unknown[] = [] - // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike. + // Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike. for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) { - where.push('r.name ILIKE ?') - bindings.push(contains(word)) + where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)') + bindings.push(contains(word), contains(word)) } - if (params.readable) where.push('r.readable') + if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)') if (params.complete) where.push('r.complete') if (params.notIncomplete) where.push('r.complete IS NOT FALSE') + if (params.unlocked) where.push('r.passworded IS NOT TRUE') if (params.source) { where.push('r.source = ?') bindings.push(params.source) @@ -122,7 +129,7 @@ class SearchService { const countLimit = offset + (params.countLimit ?? COUNT_LIMIT) const [{ rows }, { rows: counted }] = await Promise.all([ db.rawQuery( - `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at + `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`, [...bindings, perPage, offset] ), @@ -149,6 +156,8 @@ class SearchService { complete: r.complete, category: r.category, settled: r.settled_at !== null, + passworded: r.passworded, + innerName: r.inner_name, })), } } @@ -157,7 +166,10 @@ class SearchService { async release(id: number) { const r = await db.from('releases').where('id', id).first() if (!r) return null - const files = await NzbService.files(id) + const [files, { rows: inspections }] = await Promise.all([ + NzbService.files(id), + db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]), + ]) return { id: Number(r.id), name: r.name, @@ -176,6 +188,11 @@ class SearchService { firstSeenAt: r.first_seen_at, lastSeenAt: r.last_seen_at, settledAt: r.settled_at, + passworded: r.passworded, + innerName: r.inner_name, + inspectedAt: r.inspected_at, + /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */ + inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })), files: files.map((f) => ({ name: f.name, subject: f.subject, diff --git a/database/migrations/1770000001900_releases_inner_name_trgm.ts b/database/migrations/1770000001900_releases_inner_name_trgm.ts new file mode 100644 index 0000000..f55df7a --- /dev/null +++ b/database/migrations/1770000001900_releases_inner_name_trgm.ts @@ -0,0 +1,18 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Search matches the prober's inner names too (SearchService): a trigram + * index on them, only where there is one. Built concurrently: the catalog + * keeps taking batches meanwhile. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL`) + } + + async down() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_inner_name_trgm`) + } +} diff --git a/public/app.css b/public/app.css index 7156910..5185aec 100644 --- a/public/app.css +++ b/public/app.css @@ -395,6 +395,17 @@ kbd { color: var(--muted); } +.release-inner { + font-size: 12.5px; + color: var(--muted); + overflow-wrap: anywhere; +} + +.release-inner::before { + content: "↳ "; + color: var(--faint); +} + .release-meta { font-size: 12.5px; color: var(--faint); @@ -842,3 +853,41 @@ kbd { display: none; } } + +.contents { + display: grid; + gap: 10px; +} + +.inspection { + border: 1px solid var(--border); + border-radius: 8px; + padding: 10px 12px; +} + +.inspection-head { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; +} + +.inspection-head .name { + font-family: var(--mono); + font-size: 12.5px; + overflow-wrap: anywhere; + margin-right: auto; +} + +.inspection-names { + margin: 8px 0 0; + padding-left: 18px; + font-size: 13px; + overflow-wrap: anywhere; +} + +.inspection small { + display: block; + margin-top: 4px; + color: var(--faint); +} diff --git a/public/app.js b/public/app.js index 5f1f234..eca044a 100644 --- a/public/app.js +++ b/public/app.js @@ -2,9 +2,16 @@ // text is rendered with textContent, never as HTML. const $ = (selector) => document.querySelector(selector) -const FILTERS = ['readable', 'complete'] +const FILTERS = ['readable', 'complete', 'unlocked'] /** Text inputs that narrow a search: size bounds in MB. */ const FIELDS = ['min_size', 'max_size'] +/** What a probed article was (probe_candidates.kind). */ +const KIND_LABELS = { + archive: 'Archive start', + par2: 'Par2 index', + lone: 'Single file', + '7z-tail': '7z end', +} const SOURCE_LABELS = { stem: 'Filename', subject: 'Subject', @@ -37,6 +44,7 @@ const state = { max_size: '', readable: false, complete: false, + unlocked: false, page: 1, release: null, } @@ -200,12 +208,19 @@ function releaseRow(release) { release.groups?.length ? release.groups[0] + (release.groups.length > 1 ? ` +${release.groups.length - 1}` : '') : '', ].filter(Boolean) row.querySelector('.release-meta').textContent = meta.join(' · ') + if (release.innerName) { + const inner = row.querySelector('.release-inner') + inner.textContent = release.innerName + inner.title = 'Found inside by reading its articles' + inner.hidden = false + } const badges = row.querySelector('.badges') const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}` if (release.complete) badges.append(badge(files, 'good', 'Complete')) else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced')) else badges.append(badge(files)) + if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed')) if (!release.settled) badges.append(badge('Collecting', 'accent', 'Recently seen: more files may still arrive')) if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name)')) if (release.source && release.source !== 'stem' && release.source !== 'subject') { @@ -323,15 +338,22 @@ function renderDetail(release) { fact('Posted', release.postedAt ? `${new Date(release.postedAt).toLocaleString()} (${formatAge(release.postedAt)})` : null), fact('Poster', poster), fact('Groups', release.groups?.join(', ')), - fact('Indexed', release.firstSeenAt ? `${new Date(release.firstSeenAt).toLocaleString()} (${formatAge(release.firstSeenAt)})` : null) + fact('Indexed', release.firstSeenAt ? `${new Date(release.firstSeenAt).toLocaleString()} (${formatAge(release.firstSeenAt)})` : null), + fact('Inside', release.innerName), + fact('Inspected', release.inspectedAt ? `${new Date(release.inspectedAt).toLocaleString()} (${formatAge(release.inspectedAt)})` : null) ) const notices = [] if (!release.settledAt) { notices.push(el('div', { class: 'notice notice-warn', text: 'Recently seen: more files of this release may still arrive. The NZB has what is indexed so far.' })) } - if (!release.readable) { - notices.push(el('div', { class: 'notice', text: 'Obfuscated file names: what this release holds is only known once its articles are read (phase 2).' })) + if (release.passworded) { + notices.push(el('div', { class: 'notice notice-bad', text: 'Passworded: its archive has encrypted entries or headers. The password is not in the index.' })) + } + if (!release.readable && !release.innerName) { + notices.push(el('div', { class: 'notice', text: release.inspectedAt + ? 'Obfuscated file names, and its articles did not say more.' + : 'Obfuscated file names: what this release holds is only known once its articles are read (phase 2).' })) } const files = el('table', { class: 'file-table' }, @@ -341,10 +363,28 @@ function renderDetail(release) { el('td', { class: 'num', text: formatSize(f.size) }), el('td', { class: 'num', text: `${f.parts} ${f.parts === 1 ? 'part' : 'parts'}` }))))) + const inspections = release.inspections ?? [] + const contents = inspections.length + ? el('div', { class: 'contents' }, inspections.map((i) => + el('div', { class: 'inspection' }, + el('div', { class: 'inspection-head' }, + el('span', { class: 'name', text: i.yencName || '(no name)' }), + badge(KIND_LABELS[i.kind] ?? i.kind), + i.type ? badge(i.type) : null, + i.passworded ? badge('Encrypted', 'bad') : null), + i.names?.length + ? el('ul', { class: 'inspection-names' }, i.names.map((n) => el('li', { text: n }))) + : null, + i.entries > (i.names?.length ?? 0) + ? el('small', { text: `${number.format(i.entries)} in all; the largest shown` }) + : null))) + : null + $('#detail').replaceChildren( actions, ...notices, facts, + ...(contents ? section('Contents', `${inspections.length} ${inspections.length === 1 ? 'article' : 'articles'} read`, contents) : []), ...section('Files', `${release.files.length} posted`, files) ) } diff --git a/public/index.html b/public/index.html index 9d89fb0..6b73f85 100644 --- a/public/index.html +++ b/public/index.html @@ -39,6 +39,7 @@
+