diff --git a/app/services/batch_store.ts b/app/services/batch_store.ts index bf880dd..8b57d4f 100644 --- a/app/services/batch_store.ts +++ b/app/services/batch_store.ts @@ -243,6 +243,9 @@ class BatchStore { taken.set(r.key, n) return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) } } + // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000). + const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))] + if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters]) let inserts = fresh.map(numbered) // Writers of other groups number the same key at the same time (a spam post crossposted to 67 // groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried. diff --git a/app/services/release_judge.ts b/app/services/release_judge.ts index 4930062..035bf5c 100644 --- a/app/services/release_judge.ts +++ b/app/services/release_judge.ts @@ -78,7 +78,7 @@ export const RULES: Rule[] = [ // Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)). sql: `SELECT r.id FROM releases r WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL - AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at) + AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id) AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id) AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`, diff --git a/app/services/search_service.ts b/app/services/search_service.ts index 5d92e2a..2df138b 100644 --- a/app/services/search_service.ts +++ b/app/services/search_service.ts @@ -61,6 +61,8 @@ export interface ReleaseSummary { innerName: string | null } +/** Posters a poster search looks in at most (a short fragment matches thousands of random ones). */ +const MAX_POSTERS = 1000 const SORTS: Record = { newest: 'r.posted_at DESC NULLS LAST, r.id DESC', oldest: 'r.posted_at ASC NULLS LAST, r.id ASC', @@ -95,14 +97,17 @@ class SearchService { if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)') if (params.complete) where.push('r.complete') // Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final). - if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at))') + if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))') if (params.unlocked) where.push('r.passworded IS NOT TRUE') if (params.source) { where.push('r.source = ?') bindings.push(params.source) } if (params.poster) { - where.push('r.poster ILIKE ?') + // The matching posters first (posters, by trigram), then their releases by releases_poster_newest_idx: "poster ILIKE" + // on releases walked all 50M rows newest first, 30+ min for a poster with few releases (LOG.md 2026-10-16 (70)). + // By the first 200 characters, as posters and the index keep them (a btree entry holds at most 2,704 bytes). + where.push(`left(r.poster, 200) = ANY(ARRAY(SELECT p.poster FROM posters p WHERE p.poster ILIKE ? LIMIT ${MAX_POSTERS}))`) bindings.push(contains(params.poster)) } if (params.categories?.length) { @@ -137,7 +142,7 @@ class SearchService { const [{ rows }, { rows: counted }] = await Promise.all([ db.rawQuery( `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name, - release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at) AS final + release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`, [...bindings, perPage, offset] ), @@ -178,7 +183,7 @@ class SearchService { const [files, { rows: inspections }, { rows: finality }] = await Promise.all([ NzbService.files(id), db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]), - db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at]), + db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]), ]) return { id: Number(r.id), diff --git a/config/coordinator.ts b/config/coordinator.ts index 82c4030..f9f75ec 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -23,8 +23,11 @@ export default { everySeconds: env.get('TAIL_EVERY_SECONDS', 120), minArticles: env.get('TAIL_MIN_ARTICLES', 100_000), maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15), - /** Before the history's 0 (oldest) … 1000 (newest). */ - priority: -1, + /** + * Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the + * short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)). + */ + priority: -3, }, /** A leased range without a heartbeat for this long goes back to the queue. */ leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60), diff --git a/database/migrations/1770000004000_posters.ts b/database/migrations/1770000004000_posters.ts new file mode 100644 index 0000000..5913b19 --- /dev/null +++ b/database/migrations/1770000004000_posters.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Poster search (LOG.md 2026-10-16 (70)): "poster ILIKE '%…%'" on releases had no index, so it walked all 50M releases + * newest first; a poster with few releases took 30+ minutes. posters holds each poster once (3M, many random per file + * of an obfuscated upload) with a trigram index; releases_poster_newest_idx gives a poster's releases newest first. + * Both by the poster's first 200 characters: a spam poster of 3,024 bytes is over a btree entry's 2,704. The batch store + * adds new posters. Homelab built the releases index by hand first (concurrently); a no-op there. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw(`CREATE TABLE IF NOT EXISTS posters (poster text PRIMARY KEY)`) + this.schema.raw(`INSERT INTO posters (poster) SELECT DISTINCT left(poster, 200) FROM releases WHERE poster IS NOT NULL ON CONFLICT DO NOTHING`) + this.schema.raw(`CREATE INDEX IF NOT EXISTS posters_trgm ON posters USING gin (poster gin_trgm_ops)`) + this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_poster_newest_idx ON releases (left(poster, 200), posted_at DESC NULLS LAST, id DESC)`) + } + + async down() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_poster_newest_idx`) + this.schema.raw(`DROP TABLE IF EXISTS posters`) + } +} diff --git a/database/migrations/1770000004100_release_final_complete.ts b/database/migrations/1770000004100_release_final_complete.ts new file mode 100644 index 0000000..f2c9dfa --- /dev/null +++ b/database/migrations/1770000004100_release_final_complete.ts @@ -0,0 +1,23 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * release_final() with the release's complete flag: a release with every file its counter announced is final as it is + * (nothing it lacks can come); the time window only decides for the incomplete and those without a counter (LOG.md + * 2026-10-16 (70): a complete episode posted an hour before was "pending" for six hours more). + */ +export default class extends BaseSchema { + async up() { + // Beside the five-argument one, which the services still running call until the deploy restarts them. + this.schema.raw(` + CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz, complete boolean) + RETURNS boolean LANGUAGE sql STABLE AS $$ + SELECT coalesce(complete, false) OR EXISTS ( + SELECT 1 FROM range_blocks b + WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article + AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours') + AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours') + $$`) + } + + async down() {} +} diff --git a/tests/functional/final_completeness.spec.ts b/tests/functional/final_completeness.spec.ts index 9c93d3e..d586045 100644 --- a/tests/functional/final_completeness.spec.ts +++ b/tests/functional/final_completeness.spec.ts @@ -61,7 +61,7 @@ test.group('Final completeness', (group) => { } const final = async (id: number) => { const r = await db.from('releases').where('id', id).firstOrFail() - const { rows } = await db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at]) + const { rows } = await db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]) return rows[0].final as boolean } @@ -76,6 +76,8 @@ test.group('Final completeness', (group) => { assert.isTrue(await final(middle)) assert.isTrue(await final(first)) assert.isFalse(await final(edge)) + // Complete there: every file found, nothing to wait for. + assert.isTrue(await final(await release(196_000, '2026-01-01T22:30:00Z', { files: 5, complete: true, name: 'Complete.Edge' }))) assert.isFalse(await final(island)) // Search: an incomplete release that isn't final isn't left out as incomplete; the API says which is final. diff --git a/tests/functional/poster_search.spec.ts b/tests/functional/poster_search.spec.ts new file mode 100644 index 0000000..1a6f227 --- /dev/null +++ b/tests/functional/poster_search.spec.ts @@ -0,0 +1,37 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import BatchStore from '#services/batch_store' +import SearchService from '#services/search_service' +import { RangeIndexer } from '#lib/edge/index' +import { DEFAULT_RULES } from '#lib/rules/index' +import type { Overview } from '#lib/nntp/index' + +const GROUP = 'alt.binaries.posters' + +test.group('Poster search', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('the batch store keeps each poster once; a poster search finds releases by part of the name', async ({ assert }) => { + const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id') + const [node] = await db.table('nodes').insert({ name: 'poster-node' }).returning('id') + const [range] = await db + .table('ranges') + .insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 }) + .returning('id') + const post = (n: number, from: string, name: string): Overview => [ + n, + { 'subject': `[1/1] - "${name}" yEnc (1/1)`, 'from': from, 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': ``, ':bytes': '1000', 'xref': `x ${GROUP}:${n}` }, + ] + const ix = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } }) + ix.add([post(1, 'Kitchen Cook ', 'Soup.Recipes.2026.rar'), post(2, 'Kitchen Cook ', 'Bread.Recipes.2026.rar'), post(3, 'Gardener ', 'Roses.2026.rar')], 4) + ix.finish() + await BatchStore.store(Number(range.id), Number(node.id), ix.take()) + + assert.sameMembers((await db.from('posters').select('poster')).map((p) => p.poster), ['Kitchen Cook ', 'Gardener ']) + const found = await SearchService.search({ poster: 'kitchen' }) + assert.equal(found.total, 2) + assert.isTrue(found.releases.every((r) => r.poster === 'Kitchen Cook ')) + assert.equal((await SearchService.search({ poster: 'nobody' })).total, 0) + }) +})