Poster search through a posters table and releases_poster_newest_idx; complete releases are final; the tail before everything (-3)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -243,6 +243,9 @@ class BatchStore {
           taken.set(r.key, n)
           return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
         }
+        // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
+        const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
+        if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
         let inserts = fresh.map(numbered)
         // Writers of other groups number the same key at the same time (a spam post crossposted to 67
         // groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.

app/services/release_judge.ts

@@ -78,7 +78,7 @@ export const RULES: Rule[] = [
     // Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
     sql: `SELECT r.id FROM releases r
           WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
-            AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at)
+            AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
             AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
             AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
             AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,

app/services/search_service.ts

@@ -61,6 +61,8 @@ export interface ReleaseSummary {
   innerName: string | null
 }
 
+/** Posters a poster search looks in at most (a short fragment matches thousands of random ones). */
+const MAX_POSTERS = 1000
 const SORTS: Record<SortOrder, string> = {
   newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
   oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
@@ -95,14 +97,17 @@ class SearchService {
     if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
     if (params.complete) where.push('r.complete')
     // Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
-    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at))')
+    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
     if (params.unlocked) where.push('r.passworded IS NOT TRUE')
     if (params.source) {
       where.push('r.source = ?')
       bindings.push(params.source)
     }
     if (params.poster) {
-      where.push('r.poster ILIKE ?')
+      // The matching posters first (posters, by trigram), then their releases by releases_poster_newest_idx: "poster ILIKE"
+      // on releases walked all 50M rows newest first, 30+ min for a poster with few releases (LOG.md 2026-10-16 (70)).
+      // By the first 200 characters, as posters and the index keep them (a btree entry holds at most 2,704 bytes).
+      where.push(`left(r.poster, 200) = ANY(ARRAY(SELECT p.poster FROM posters p WHERE p.poster ILIKE ? LIMIT ${MAX_POSTERS}))`)
       bindings.push(contains(params.poster))
     }
     if (params.categories?.length) {
@@ -137,7 +142,7 @@ class SearchService {
     const [{ rows }, { rows: counted }] = await Promise.all([
       db.rawQuery(
         `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
-                release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at) AS final
+                release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
          FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
         [...bindings, perPage, offset]
       ),
@@ -178,7 +183,7 @@ class SearchService {
     const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
       NzbService.files(id),
       db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
-      db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at]),
+      db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
     ])
     return {
       id: Number(r.id),

config/coordinator.ts

@@ -23,8 +23,11 @@ export default {
     everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
     minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
     maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
-    /** Before the history's 0 (oldest) … 1000 (newest). */
-    priority: -1,
+    /**
+     * Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the
+     * short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)).
+     */
+    priority: -3,
   },
   /** A leased range without a heartbeat for this long goes back to the queue. */
   leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),

database/migrations/1770000004000_posters.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Poster search (LOG.md 2026-10-16 (70)): "poster ILIKE '%…%'" on releases had no index, so it walked all 50M releases
+ * newest first; a poster with few releases took 30+ minutes. posters holds each poster once (3M, many random per file
+ * of an obfuscated upload) with a trigram index; releases_poster_newest_idx gives a poster's releases newest first.
+ * Both by the poster's first 200 characters: a spam poster of 3,024 bytes is over a btree entry's 2,704. The batch store
+ * adds new posters. Homelab built the releases index by hand first (concurrently); a no-op there.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE TABLE IF NOT EXISTS posters (poster text PRIMARY KEY)`)
+    this.schema.raw(`INSERT INTO posters (poster) SELECT DISTINCT left(poster, 200) FROM releases WHERE poster IS NOT NULL ON CONFLICT DO NOTHING`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_poster_newest_idx ON releases (left(poster, 200), posted_at DESC NULLS LAST, id DESC)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_poster_newest_idx`)
+    this.schema.raw(`DROP TABLE IF EXISTS posters`)
+  }
+}

database/migrations/1770000004100_release_final_complete.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * release_final() with the release's complete flag: a release with every file its counter announced is final as it is
+ * (nothing it lacks can come); the time window only decides for the incomplete and those without a counter (LOG.md
+ * 2026-10-16 (70): a complete episode posted an hour before was "pending" for six hours more).
+ */
+export default class extends BaseSchema {
+  async up() {
+    // Beside the five-argument one, which the services still running call until the deploy restarts them.
+    this.schema.raw(`
+      CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz, complete boolean)
+      RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT coalesce(complete, false) OR EXISTS (
+          SELECT 1 FROM range_blocks b
+          WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
+            AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
+            AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
+      $$`)
+  }
+
+  async down() {}
+}

tests/functional/final_completeness.spec.ts

@@ -61,7 +61,7 @@ test.group('Final completeness', (group) => {
     }
     const final = async (id: number) => {
       const r = await db.from('releases').where('id', id).firstOrFail()
-      const { rows } = await db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at])
+      const { rows } = await db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete])
       return rows[0].final as boolean
     }
 
@@ -76,6 +76,8 @@ test.group('Final completeness', (group) => {
     assert.isTrue(await final(middle))
     assert.isTrue(await final(first))
     assert.isFalse(await final(edge))
+    // Complete there: every file found, nothing to wait for.
+    assert.isTrue(await final(await release(196_000, '2026-01-01T22:30:00Z', { files: 5, complete: true, name: 'Complete.Edge' })))
     assert.isFalse(await final(island))
 
     // Search: an incomplete release that isn't final isn't left out as incomplete; the API says which is final.

tests/functional/poster_search.spec.ts

@@ -0,0 +1,37 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import BatchStore from '#services/batch_store'
+import SearchService from '#services/search_service'
+import { RangeIndexer } from '#lib/edge/index'
+import { DEFAULT_RULES } from '#lib/rules/index'
+import type { Overview } from '#lib/nntp/index'
+
+const GROUP = 'alt.binaries.posters'
+
+test.group('Poster search', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('the batch store keeps each poster once; a poster search finds releases by part of the name', async ({ assert }) => {
+    const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+    const [node] = await db.table('nodes').insert({ name: 'poster-node' }).returning('id')
+    const [range] = await db
+      .table('ranges')
+      .insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 })
+      .returning('id')
+    const post = (n: number, from: string, name: string): Overview => [
+      n,
+      { 'subject': `[1/1] - "${name}" yEnc (1/1)`, 'from': from, 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<ps${n}@x>`, ':bytes': '1000', 'xref': `x ${GROUP}:${n}` },
+    ]
+    const ix = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
+    ix.add([post(1, 'Kitchen Cook <cook@example.com>', 'Soup.Recipes.2026.rar'), post(2, 'Kitchen Cook <cook@example.com>', 'Bread.Recipes.2026.rar'), post(3, 'Gardener <garden@example.com>', 'Roses.2026.rar')], 4)
+    ix.finish()
+    await BatchStore.store(Number(range.id), Number(node.id), ix.take())
+
+    assert.sameMembers((await db.from('posters').select('poster')).map((p) => p.poster), ['Kitchen Cook <cook@example.com>', 'Gardener <garden@example.com>'])
+    const found = await SearchService.search({ poster: 'kitchen' })
+    assert.equal(found.total, 2)
+    assert.isTrue(found.releases.every((r) => r.poster === 'Kitchen Cook <cook@example.com>'))
+    assert.equal((await SearchService.search({ poster: 'nobody' })).total, 0)
+  })
+})