Search and the web interface use the prober's findings

- Search matches a word in the name or the inner name (trigram index on
  inner_name, partial, built concurrently); "Readable names" includes
  releases with an inner name.
- unlocked=1 (chip "No password") leaves out passworded releases.
- Results carry passworded and innerName: a Password badge, the inner
  name under the release name.
- The release view: Inside and Inspected facts, a passworded notice, and
  Contents (per probed article its yEnc name, kind, type, encryption and
  the names inside) from release_inspections.
- Newznab: the password attribute from the prober; an obfuscated release
  with an inner name is included, titled by it (without its extension).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/controllers/newznab_controller.ts

@@ -32,7 +32,7 @@ function int(value: unknown): number | undefined {
 /**
  * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
  * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
- * Releases with readable names that aren't known to miss files.
+ * Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
  */
 export default class NewznabController {
   async handle({ request, response }: HttpContext) {
@@ -91,15 +91,16 @@ export default class NewznabController {
       const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
       return {
         id: r.id,
-        title: r.name,
+        // An obfuscated release by what the prober found inside, without its extension (what clients parse).
+        title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''),
         category: r.category ?? 8010,
         size: r.size,
         files: r.fileCount,
         postedAt: r.postedAt ? new Date(r.postedAt) : null,
         poster: r.poster,
         groups: r.groups ?? [],
-        // Passwords come with phase 2 (archive inspection).
-        password: false,
+        // From the prober (phase 2): encrypted archive entries or headers.
+        password: r.passworded === true,
         season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
         episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
       }

app/controllers/releases_controller.ts

@@ -13,7 +13,7 @@ function megabytes(value: unknown): number | undefined {
 
 /** The catalog for the web interface (REBUILD.md, part 9). */
 export default class ReleasesController {
-  /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
+  /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
   async index({ request }: HttpContext) {
     const qs = request.qs()
     const categories = String(qs.cat ?? '')
@@ -25,6 +25,7 @@ export default class ReleasesController {
       categories: categories.length ? expandCategories(categories) : undefined,
       readable: flag(qs.readable),
       complete: flag(qs.complete),
+      unlocked: flag(qs.unlocked),
       source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
       poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
       minSize: megabytes(qs.min_size),

app/services/search_service.ts

@@ -7,12 +7,14 @@ export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
 /** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
 export interface SearchParams {
   q?: string
-  /** Only releases with a file whose name is readable (not an obfuscated token). */
+  /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
   readable?: boolean
   /** Only releases known to have all their files. */
   complete?: boolean
   /** Leave out releases known to miss files (Newznab). */
   notIncomplete?: boolean
+  /** Leave out releases the prober found passworded (encrypted archive entries or headers). */
+  unlocked?: boolean
   /** How the release was recognised: stem, subject or post. */
   source?: string
   poster?: string
@@ -49,6 +51,10 @@ export interface ReleaseSummary {
   complete: boolean | null
   category: number
   settled: boolean
+  /** The prober found encrypted archive entries or headers; null: not probed (yet). */
+  passworded: boolean | null
+  /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
+  innerName: string | null
 }
 
 const SORTS: Record<SortOrder, string> = {
@@ -75,14 +81,15 @@ class SearchService {
   async search(params: SearchParams) {
     const where: string[] = ['r.files > 0']
     const bindings: unknown[] = []
-    // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike.
+    // Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
     for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
-      where.push('r.name ILIKE ?')
-      bindings.push(contains(word))
+      where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
+      bindings.push(contains(word), contains(word))
     }
-    if (params.readable) where.push('r.readable')
+    if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
     if (params.complete) where.push('r.complete')
     if (params.notIncomplete) where.push('r.complete IS NOT FALSE')
+    if (params.unlocked) where.push('r.passworded IS NOT TRUE')
     if (params.source) {
       where.push('r.source = ?')
       bindings.push(params.source)
@@ -122,7 +129,7 @@ class SearchService {
     const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
     const [{ rows }, { rows: counted }] = await Promise.all([
       db.rawQuery(
-        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at
+        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name
          FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
         [...bindings, perPage, offset]
       ),
@@ -149,6 +156,8 @@ class SearchService {
         complete: r.complete,
         category: r.category,
         settled: r.settled_at !== null,
+        passworded: r.passworded,
+        innerName: r.inner_name,
       })),
     }
   }
@@ -157,7 +166,10 @@ class SearchService {
   async release(id: number) {
     const r = await db.from('releases').where('id', id).first()
     if (!r) return null
-    const files = await NzbService.files(id)
+    const [files, { rows: inspections }] = await Promise.all([
+      NzbService.files(id),
+      db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
+    ])
     return {
       id: Number(r.id),
       name: r.name,
@@ -176,6 +188,11 @@ class SearchService {
       firstSeenAt: r.first_seen_at,
       lastSeenAt: r.last_seen_at,
       settledAt: r.settled_at,
+      passworded: r.passworded,
+      innerName: r.inner_name,
+      inspectedAt: r.inspected_at,
+      /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
+      inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
       files: files.map((f) => ({
         name: f.name,
         subject: f.subject,

database/migrations/1770000001900_releases_inner_name_trgm.ts

@@ -0,0 +1,18 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Search matches the prober's inner names too (SearchService): a trigram
+ * index on them, only where there is one. Built concurrently: the catalog
+ * keeps taking batches meanwhile.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_inner_name_trgm`)
+  }
+}

public/app.css

@@ -395,6 +395,17 @@ kbd {
   color: var(--muted);
 }
 
+.release-inner {
+  font-size: 12.5px;
+  color: var(--muted);
+  overflow-wrap: anywhere;
+}
+
+.release-inner::before {
+  content: "↳ ";
+  color: var(--faint);
+}
+
 .release-meta {
   font-size: 12.5px;
   color: var(--faint);
@@ -842,3 +853,41 @@ kbd {
     display: none;
   }
 }
+
+.contents {
+  display: grid;
+  gap: 10px;
+}
+
+.inspection {
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 10px 12px;
+}
+
+.inspection-head {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 6px;
+}
+
+.inspection-head .name {
+  font-family: var(--mono);
+  font-size: 12.5px;
+  overflow-wrap: anywhere;
+  margin-right: auto;
+}
+
+.inspection-names {
+  margin: 8px 0 0;
+  padding-left: 18px;
+  font-size: 13px;
+  overflow-wrap: anywhere;
+}
+
+.inspection small {
+  display: block;
+  margin-top: 4px;
+  color: var(--faint);
+}

public/app.js

@@ -2,9 +2,16 @@
 // text is rendered with textContent, never as HTML.
 
 const $ = (selector) => document.querySelector(selector)
-const FILTERS = ['readable', 'complete']
+const FILTERS = ['readable', 'complete', 'unlocked']
 /** Text inputs that narrow a search: size bounds in MB. */
 const FIELDS = ['min_size', 'max_size']
+/** What a probed article was (probe_candidates.kind). */
+const KIND_LABELS = {
+  archive: 'Archive start',
+  par2: 'Par2 index',
+  lone: 'Single file',
+  '7z-tail': '7z end',
+}
 const SOURCE_LABELS = {
   stem: 'Filename',
   subject: 'Subject',
@@ -37,6 +44,7 @@ const state = {
   max_size: '',
   readable: false,
   complete: false,
+  unlocked: false,
   page: 1,
   release: null,
 }
@@ -200,12 +208,19 @@ function releaseRow(release) {
     release.groups?.length ? release.groups[0] + (release.groups.length > 1 ? ` +${release.groups.length - 1}` : '') : '',
   ].filter(Boolean)
   row.querySelector('.release-meta').textContent = meta.join(' · ')
+  if (release.innerName) {
+    const inner = row.querySelector('.release-inner')
+    inner.textContent = release.innerName
+    inner.title = 'Found inside by reading its articles'
+    inner.hidden = false
+  }
 
   const badges = row.querySelector('.badges')
   const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}`
   if (release.complete) badges.append(badge(files, 'good', 'Complete'))
   else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
   else badges.append(badge(files))
+  if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
   if (!release.settled) badges.append(badge('Collecting', 'accent', 'Recently seen: more files may still arrive'))
   if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name)'))
   if (release.source && release.source !== 'stem' && release.source !== 'subject') {
@@ -323,15 +338,22 @@ function renderDetail(release) {
     fact('Posted', release.postedAt ? `${new Date(release.postedAt).toLocaleString()} (${formatAge(release.postedAt)})` : null),
     fact('Poster', poster),
     fact('Groups', release.groups?.join(', ')),
-    fact('Indexed', release.firstSeenAt ? `${new Date(release.firstSeenAt).toLocaleString()} (${formatAge(release.firstSeenAt)})` : null)
+    fact('Indexed', release.firstSeenAt ? `${new Date(release.firstSeenAt).toLocaleString()} (${formatAge(release.firstSeenAt)})` : null),
+    fact('Inside', release.innerName),
+    fact('Inspected', release.inspectedAt ? `${new Date(release.inspectedAt).toLocaleString()} (${formatAge(release.inspectedAt)})` : null)
   )
 
   const notices = []
   if (!release.settledAt) {
     notices.push(el('div', { class: 'notice notice-warn', text: 'Recently seen: more files of this release may still arrive. The NZB has what is indexed so far.' }))
   }
-  if (!release.readable) {
-    notices.push(el('div', { class: 'notice', text: 'Obfuscated file names: what this release holds is only known once its articles are read (phase 2).' }))
+  if (release.passworded) {
+    notices.push(el('div', { class: 'notice notice-bad', text: 'Passworded: its archive has encrypted entries or headers. The password is not in the index.' }))
+  }
+  if (!release.readable && !release.innerName) {
+    notices.push(el('div', { class: 'notice', text: release.inspectedAt
+      ? 'Obfuscated file names, and its articles did not say more.'
+      : 'Obfuscated file names: what this release holds is only known once its articles are read (phase 2).' }))
   }
 
   const files = el('table', { class: 'file-table' },
@@ -341,10 +363,28 @@ function renderDetail(release) {
         el('td', { class: 'num', text: formatSize(f.size) }),
         el('td', { class: 'num', text: `${f.parts} ${f.parts === 1 ? 'part' : 'parts'}` })))))
 
+  const inspections = release.inspections ?? []
+  const contents = inspections.length
+    ? el('div', { class: 'contents' }, inspections.map((i) =>
+        el('div', { class: 'inspection' },
+          el('div', { class: 'inspection-head' },
+            el('span', { class: 'name', text: i.yencName || '(no name)' }),
+            badge(KIND_LABELS[i.kind] ?? i.kind),
+            i.type ? badge(i.type) : null,
+            i.passworded ? badge('Encrypted', 'bad') : null),
+          i.names?.length
+            ? el('ul', { class: 'inspection-names' }, i.names.map((n) => el('li', { text: n })))
+            : null,
+          i.entries > (i.names?.length ?? 0)
+            ? el('small', { text: `${number.format(i.entries)} in all; the largest shown` })
+            : null)))
+    : null
+
   $('#detail').replaceChildren(
     actions,
     ...notices,
     facts,
+    ...(contents ? section('Contents', `${inspections.length} ${inspections.length === 1 ? 'article' : 'articles'} read`, contents) : []),
     ...section('Files', `${release.files.length} posted`, files)
   )
 }

public/index.html

@@ -39,6 +39,7 @@
         <div class="filters" role="group" aria-label="Filters">
           <button class="chip" type="button" data-filter="readable" aria-pressed="false">Readable names</button>
           <button class="chip" type="button" data-filter="complete" aria-pressed="false">Complete</button>
+          <button class="chip" type="button" data-filter="unlocked" aria-pressed="false" title="Leave out releases found passworded">No password</button>
           <label class="select select-small">
             <span class="visually-hidden">Category</span>
             <select id="cat">
@@ -96,6 +97,7 @@
       <li class="release">
         <button type="button" class="release-main">
           <span class="release-name"></span>
+          <span class="release-inner" hidden></span>
           <span class="release-meta"></span>
         </button>
         <div class="release-side">

tests/functional/releases_api.spec.ts

@@ -80,4 +80,37 @@ test.group('Releases API', (group) => {
     assert.include((await client.get('/api?t=get&id=999999')).text(), 'code="300"')
     assert.include((await client.get('/api?t=nope')).text(), 'code="202"')
   })
+
+  test("the prober's findings: inner names in search and Newznab titles, passworded filtered and marked", async ({ client, assert }) => {
+    const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+    const [n] = await db.table('nodes').insert({ name: 'api-node' }).returning('id')
+    const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id')
+    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
+    indexer.add([...file('Some.Show.S01E01.1080p.WEB.mkv', 'Some.Show.S01E01.1080p.WEB [1/1]', 2, 100), ...file('Other.Movie.2024.2160p.BluRay.x265.mkv', 'Other.Movie.2024.2160p.BluRay.x265 [1/1]', 1, 300)], 1001)
+    indexer.finish()
+    await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
+    const show = await db.from('releases').where('name', 'ilike', '%show%').firstOrFail()
+    const movie = await db.from('releases').where('name', 'ilike', '%movie%').firstOrFail()
+    // As the prober leaves them: the movie's archive is encrypted, the show holds a differently named file.
+    await db.from('releases').where('id', movie.id).update({ passworded: true, inspected_at: new Date() })
+    await db.from('releases').where('id', show.id).update({ passworded: false, inner_name: 'Hidden.Gem.2023.mkv', readable: false, inspected_at: new Date() })
+    await db.table('release_inspections').insert({ release_id: show.id, candidate_id: 1, kind: 'archive', result: JSON.stringify({ yencName: 'x.rar', size: 1, type: 'rar5', names: ['Hidden.Gem.2023.mkv'], entries: 1, passworded: false }) })
+
+    const byInner = (await client.get('/api/v1/releases?q=hidden gem')).body() as { releases: { id: number; innerName: string }[] }
+    assert.deepEqual(byInner.releases.map((x) => [x.id, x.innerName]), [[Number(show.id), 'Hidden.Gem.2023.mkv']])
+    // Readable by its inner name.
+    const readable = (await client.get('/api/v1/releases?readable=1')).body() as { releases: { id: number }[] }
+    assert.lengthOf(readable.releases, 2)
+    const unlocked = (await client.get('/api/v1/releases?unlocked=1')).body() as { releases: { id: number; passworded: boolean }[] }
+    assert.deepEqual(unlocked.releases.map((x) => x.id), [Number(show.id)])
+
+    const detail = (await client.get(`/api/v1/releases/${show.id}`)).body() as any
+    assert.equal(detail.innerName, 'Hidden.Gem.2023.mkv')
+    assert.deepEqual(detail.inspections.map((i: any) => [i.kind, i.names]), [['archive', ['Hidden.Gem.2023.mkv']]])
+
+    const key = process.env.NEWZNAB_API_KEY
+    const feed = (await client.get(`/api?t=search&q=&apikey=${key ?? ''}`)).text()
+    assert.include(feed, '<title>Hidden.Gem.2023</title>')
+    assert.match(feed, /Other Movie 2024[^]*?name="password" value="1"/)
+  })
 })