diff --git a/app/services/batch_store.ts b/app/services/batch_store.ts index 1938d31..aa41d8b 100644 --- a/app/services/batch_store.ts +++ b/app/services/batch_store.ts @@ -111,8 +111,9 @@ class BatchStore { for (let i = 0; i < sorted.length; i += 1000) { const chunk = sorted.slice(i, i + 1000) const { rows } = await trx.rawQuery( - `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2) - SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2 + `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete) + SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2, + CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean) ON CONFLICT (release_key) DO UPDATE SET files = releases.files + excluded.files, @@ -125,6 +126,9 @@ class BatchStore { last_article = greatest(releases.last_article, excluded.last_article), readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2, + -- As it stands (the counter's files all found); settling makes it final. + complete = CASE WHEN coalesce(releases.file_total, excluded.file_total) IS NULL THEN NULL + ELSE releases.files + excluded.files >= coalesce(releases.file_total, excluded.file_total) END, last_seen_at = now() RETURNING id, release_key, (xmax = 0) AS inserted`, [ diff --git a/app/services/planner.ts b/app/services/planner.ts index 694f448..71012b8 100644 --- a/app/services/planner.ts +++ b/app/services/planner.ts @@ -76,7 +76,7 @@ class Planner { * group took minutes for 1000 groups). */ async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> { - const groups = await db.from('groups').where('active', true).whereNotNull('planned_at').select('id', 'name', 'planned_at') + const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at') if (groups.length === 0) return { groups: 0, ranges: 0 } const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`) const plannedTo = new Map(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)])) diff --git a/app/services/search_service.ts b/app/services/search_service.ts index 348fd38..63b4c8f 100644 --- a/app/services/search_service.ts +++ b/app/services/search_service.ts @@ -222,12 +222,18 @@ class SearchService { const sampled = Number(s[0].sampled) const releases = sample ? Math.max(estimate, 0) : sampled const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0) + // History by articles; the pass: the history ranges before those deferred behind them (priority + // 2000 apart; e.g. groups added later that go first). const { rows: ranges } = await db.rawQuery( - `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total, - count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history, - max(done_at) AS last_done_at, - max(done_at) FILTER (WHERE priority < 0) AS tail_done_at - FROM ranges` + `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0), + pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h) + SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total, + coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history, + coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done, + coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass, + (SELECT max(done_at) FROM ranges) AS last_done_at, + (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at + FROM h` ) const g = ranges[0] return { @@ -236,7 +242,10 @@ class SearchService { readableReleases: scale(s[0].readable), files: scale(s[0].files), size: scale(s[0].bytes), - ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) }, + ranges: { done: Number(g.done), total: Number(g.total) }, + /** History and the current pass, in articles. */ + history: { done: Number(g.history_done), total: Number(g.history) }, + pass: { done: Number(g.pass_done), total: Number(g.pass) }, lastIndexedAt: g.last_done_at, tailIndexedAt: g.tail_done_at, } diff --git a/database/migrations/1770000001300_groups_tail.ts b/database/migrations/1770000001300_groups_tail.ts new file mode 100644 index 0000000..33f25f2 --- /dev/null +++ b/database/migrations/1770000001300_groups_tail.ts @@ -0,0 +1,15 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A group's live tail can be paused (its history and its place in the owner + * rule stay): e.g. to let other groups' history go first. + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(`ALTER TABLE groups ADD COLUMN IF NOT EXISTS tail boolean NOT NULL DEFAULT true`) + } + + async down() { + this.schema.raw(`ALTER TABLE groups DROP COLUMN IF EXISTS tail`) + } +} diff --git a/lib/release/identity.ts b/lib/release/identity.ts index 30c1c36..b066694 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -185,7 +185,9 @@ export function isPosterTag(remainder: string, stem: string): boolean { */ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject) - const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined + // The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's. + const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined + const fileTotal = counterTotal(input.subject) const stem = fileStem(input.name) if (input.weak) { @@ -223,8 +225,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { } // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the // [i/M] counter it groups the files of one post whose names are random. - if (!tag && fileTotal && isPostToken(remainder)) { - return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' } + if (!tag && postTotal && isPostToken(remainder)) { + return { key: hash('post', input.poster, remainder, postTotal), name: remainder, fileTotal, source: 'post' } } // An image without words around it: its set by its name (keyed as a stem, so "x.jpg" joins "x.part01.rar"). if (image) return { key: hash('stem', input.poster, prefix || stem), name: prefix || stem, fileTotal, source: 'stem' } diff --git a/public/app.js b/public/app.js index 71bd7fd..5f1f234 100644 --- a/public/app.js +++ b/public/app.js @@ -164,13 +164,17 @@ async function loadStats() { const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0 // Estimated from Postgres statistics and a sample. const about = stats.approximate ? '≈ ' : '' - const history = stats.ranges.history ? (100 * stats.ranges.historyDone) / stats.ranges.history : 0 + const percent = ({ done, total }) => { + const p = total ? (100 * done) / total : 0 + return `${p < 10 ? p.toFixed(1) : Math.round(p)}%` + } const items = [ ['Releases', about + number.format(stats.releases)], ['Readable', `${about}${readable}%`], ['Files', about + number.format(stats.files)], ['Size', about + formatSize(stats.size)], - ['History', `${history < 10 ? history.toFixed(1) : Math.round(history)}%`], + ['History', percent(stats.history)], + ...(stats.pass.total < stats.history.total ? [['Pass', percent(stats.pass)]] : []), ['Live', formatAge(stats.tailIndexedAt) || '–'], ] $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd })))) diff --git a/tests/functional/releases_api.spec.ts b/tests/functional/releases_api.spec.ts index b5a84e4..78599f8 100644 --- a/tests/functional/releases_api.spec.ts +++ b/tests/functional/releases_api.spec.ts @@ -57,9 +57,13 @@ test.group('Releases API', (group) => { const detail = (await client.get(`/api/v1/releases/${tv.releases[0].id}`)).body() as { files: { name: string; parts: number; size: number }[] } assert.deepEqual(detail.files, [{ ...detail.files[0], name: 'Some.Show.S01E01.1080p.WEB.mkv', parts: 2, size: 1400001 }]) ;(await client.get('/api/v1/releases/999999')).assertStatus(404) - const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number } } + // Complete as soon as the counter's files are all found (settling makes it final). + const complete = (await client.get('/api/v1/releases?complete=1')).body() as { releases: unknown[] } + assert.lengthOf(complete.releases, 2) + const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number }; history: { total: number } } assert.equal(stats.releases, 2) assert.equal(stats.ranges.total, 1) + assert.equal(stats.history.total, 1000) // Newznab: caps, a TV search with season and episode, the NZB; the key when one is set. const caps = await client.get('/api?t=caps') diff --git a/tools/edge-bench-rs/src/names.rs b/tools/edge-bench-rs/src/names.rs index e3a42cb..e0a619f 100644 --- a/tools/edge-bench-rs/src/names.rs +++ b/tools/edge-bench-rs/src/names.rs @@ -510,7 +510,9 @@ pub fn is_poster_tag(remainder: &str, stem: &str) -> bool { /// releaseIdentity for a named (not weak) file: never null. pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { - let file_total = file_counter_total(subject); + // The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal). + let post_total = file_counter_total(subject); + let file_total = counter_total(subject).and_then(|t| t.parse::().ok()); let stem = file_stem(name); let remainder = subject_remainder(subject); // A document or comic is a release of its own. @@ -527,7 +529,7 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let bound = counter_total(subject).unwrap_or_else(|| prefix.clone()); return Release { key: hash(&["subject", poster, &remainder, &bound]), name: remainder, source: "subject", file_total }; } - if let Some(total) = file_total.filter(|&n| n != 0 && !tag) { + if let Some(total) = post_total.filter(|&n| n != 0 && !tag) { if is_post_token(&remainder) { return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total }; }