Categories from the inner name, spots and groups; fix the name rules
Migration 1200 passed regexes with `?` through knex, which made them bindings: the live function held blu-$1ray, web-$2dl and h\.$326[45], so BluRay, WEB-DL and H.264 names without a resolution were Misc. 93% of recent releases were Misc, as the category came from the name alone. category becomes a plain column (DROP EXPRESSION, no rewrite) set by a trigger: the name's category (fixed, no `?` anywhere; adds XXX, audiobooks, console and PC games, and TV codes like S01e07[ and S04E01E02), else the prober's inner name's, else one given before (a spot's), else the groups'. The spot inspector sets a spot's category (spot_category) on Misc releases it links and on its own releases. The Newznab and web category lists gain Console, PC Games, Audiobook, Anime, XXX and Comics. releases:recategorize recomputes existing rows in id batches, writing only changed rows and waiting under a free-space floor. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
ebe9f8bb36
commit
61bdd43b34
8 files changed
+316
-11
No files matched your search
+7
-2
@@ -198,8 +198,13 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
again after a par2 merge and before the pruner drops a set's file list.
|
||||
Segments aren't compared. Phase 2 picks its work from settled releases.
|
||||
|
||||
9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
|
||||
search (names) and get. Web UI later.
|
||||
9. **NZB and API**. NZB built from a release's chunks, with a known
|
||||
password in its meta; Newznab search (names, inner names) and get; a web
|
||||
interface. Categories (Newznab's) by a trigger on releases (migration
|
||||
5200): from the name, else the inner name the prober found, else a spot's
|
||||
category (spot_category, set by the spot inspector), else the groups
|
||||
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
|
||||
in batches.
|
||||
|
||||
## Group list
|
||||
|
||||
|
||||
@@ -294,6 +294,12 @@ export class SpotInspector {
|
||||
.ignore()
|
||||
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
|
||||
}
|
||||
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
|
||||
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
|
||||
spot.category,
|
||||
JSON.stringify(spot.subcats ?? {}),
|
||||
[...perRelease.keys()],
|
||||
])
|
||||
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
|
||||
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
|
||||
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
|
||||
@@ -347,6 +353,8 @@ export class SpotInspector {
|
||||
readable: true,
|
||||
passworded: spot.password ? true : null,
|
||||
settled_at: new Date(),
|
||||
// Kept when its title says nothing (the category trigger, migration 5200).
|
||||
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
|
||||
})
|
||||
.onConflict('release_key')
|
||||
.ignore()
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
|
||||
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||
const CURSOR = 'recategorize'
|
||||
|
||||
/**
|
||||
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
|
||||
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
|
||||
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
|
||||
* the dead versions reusable) and goes on from its cursor when run again.
|
||||
*/
|
||||
export default class ReleasesRecategorize extends BaseCommand {
|
||||
static commandName = 'releases:recategorize'
|
||||
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Release ids per batch' })
|
||||
declare batch: number
|
||||
|
||||
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
|
||||
declare toId: number
|
||||
|
||||
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
|
||||
declare floorGb: number
|
||||
|
||||
@flags.boolean({ description: 'Count what would change, write nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||
declare restart: boolean
|
||||
|
||||
async run() {
|
||||
const batch = this.batch ?? 20_000
|
||||
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
|
||||
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||
const changes = new Map<string, number>()
|
||||
let changed = 0
|
||||
let looked = 0
|
||||
const started = Date.now()
|
||||
while (after < hi) {
|
||||
while (!this.dryRun && (await space.short())) await sleep(60_000)
|
||||
const to = Math.min(after + batch, hi)
|
||||
const new_ = `release_category_for(name, inner_name, groups, NULL)`
|
||||
if (this.dryRun) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT category AS was, ${new_} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
|
||||
[after, to]
|
||||
)
|
||||
for (const r of rows) {
|
||||
looked += r.n
|
||||
if (r.was === r.now) continue
|
||||
changed += r.n
|
||||
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
|
||||
}
|
||||
} else {
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE releases r SET category = x.c FROM (SELECT id, ${new_} AS c FROM releases WHERE id > ? AND id <= ?) x
|
||||
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
|
||||
[after, to]
|
||||
)
|
||||
changed += result.rowCount ?? 0
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, String(to)]
|
||||
)
|
||||
}
|
||||
after = to
|
||||
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
|
||||
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
|
||||
}
|
||||
}
|
||||
if (this.dryRun) {
|
||||
this.logger.info(`Would change ${changed} of ${looked}:`)
|
||||
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
|
||||
return
|
||||
}
|
||||
// The spots' categories for the linked releases whose names say nothing.
|
||||
const spots = await db.rawQuery(
|
||||
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
|
||||
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
|
||||
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
|
||||
)
|
||||
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release's category from more than its name (REBUILD.md, part 9; LOG.md 2026-10-23 (95)). Migration 1200 made it
|
||||
* a generated column of release_category(name), and knex took the `?` in its regexes for bindings: the function held
|
||||
* `blu-$1ray`, `web-$2dl`, `h\.$326[45]`, so BluRay, WEB-DL and H.264 names without a resolution were Misc. Now:
|
||||
* - release_category(name), fixed (no `?` in any regex here: `{0,1}`) and with XXX, audiobooks, console and PC games,
|
||||
* and TV codes the old one missed (`S01e07[`, `S04E01E02`);
|
||||
* - the inner name the prober found (an obfuscated release's real name) when the name says nothing;
|
||||
* - else a category given before (a spot's, below), else the groups' (erotica, sounds, ebooks, …);
|
||||
* - a spot's category (spot_category: its head category and subcategories), set by the spot inspector.
|
||||
* `category` becomes a plain column (DROP EXPRESSION keeps the values, no rewrite) set by a trigger; existing rows are
|
||||
* recategorized by `releases:recategorize` in batches, not here (a rewrite of the 70 GB table).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- XXX first: its names carry years, resolutions and episode-like codes too.
|
||||
WHEN name ~* '(^|[^a-z0-9])xxx([^a-z0-9]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 6045
|
||||
WHEN name ~* '720p|1080p|x26[45]|h\.{0,1}26[45]|hevc' THEN 6040
|
||||
WHEN name ~* 'imageset|picset|(^|[._ -])pics([._ -]|$)' THEN 6060 ELSE 6070 END
|
||||
-- TV: S01E02 (also S01e02[, S04E01E02), 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[^a-z0-9])(s[0-9]{1,2}[ ._-]{0,1}e[0-9]{1,3}|[0-9]{1,2}x[0-9]{2})([^0-9]|$)' OR name ~* '(^|[._ -])s[0-9]{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)[0-9]{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-{0,1}ray|web-{0,1}dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.{0,1}26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-{0,1}ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: audiobooks, lossless, mp3, or scene music names (-WEB-2026-GROUP).
|
||||
WHEN name ~* '(audiobook|audioboek|luisterboek|h(oe|ö)rbuch)' THEN 3030
|
||||
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)[0-9]{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN name ~* '(^|[._ -])(epub|ebook|e-book|mobi|azw3)([._ -]|$)' THEN 7020
|
||||
-- Games: console tags, PC scene groups and repackers.
|
||||
WHEN name ~* '(^|[._ -])(nsw|ps4|ps5|xboxone|xbox360|wiiu|3ds|nds)([._ -]|$)' THEN 1000
|
||||
WHEN name ~* '-(codex|plaza|skidrow|reloaded|flt|doge|tenoke|rune|empress|razor1911|tinyiso|darksiders|simplex)$' OR name ~* '(fitgirl|dodi|elamigos)[._ -]{0,1}repack' THEN 4050
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_group_category(groups jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
WHEN g ~* '(erotica|xxx|porn|erotic|nude|naked)' THEN CASE WHEN g ~* '(pictures|pics|images|photos)' THEN 6060 ELSE 6070 END
|
||||
WHEN g ~* 'anime' THEN 5070
|
||||
WHEN g ~* 'audiobook' THEN 3030
|
||||
WHEN g ~* '(sounds|mp3|music)' THEN CASE WHEN g ~* '(lossless|flac)' THEN 3040 ELSE 3010 END
|
||||
WHEN g ~* 'comics' THEN 7030
|
||||
WHEN g ~* 'e-{0,1}books{0,1}' THEN 7020
|
||||
WHEN g ~* '(games|nintendo|xbox|playstation|wii)' THEN 1000
|
||||
WHEN g ~* '(^|[. ])(tv|hdtv|teevee|tvseries)([. ]|$)' THEN 5000
|
||||
WHEN g ~* '(^|[. ])(movies|dvd|bluray|dvdr)([. ]|$)' THEN 2000
|
||||
ELSE 8010
|
||||
END
|
||||
FROM (SELECT coalesce(string_agg(value, ' '), '') AS g FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(groups) = 'array' THEN groups ELSE '[]' END)) x
|
||||
$$`)
|
||||
// The name's, else the inner name's, else one given before (a spot's), else the groups'.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION release_category_for(name text, inner_name text, groups jsonb, given integer) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE WHEN n <> 8010 THEN n WHEN i <> 8010 THEN i WHEN coalesce(given, 8010) <> 8010 THEN given ELSE release_group_category(groups) END
|
||||
FROM (SELECT release_category(name) AS n, coalesce(release_category(inner_name), 8010) AS i) x
|
||||
$$`)
|
||||
// Whether a spot's subcategories of one type letter ({"a": [9], "z": [0]}) hold one of `vals`.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_has(subcats jsonb, letter text, vals integer[]) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((SELECT bool_or((e #>> '{}')::integer = ANY(vals)) FROM jsonb_array_elements(
|
||||
CASE WHEN jsonb_typeof(subcats->letter) = 'array' THEN subcats->letter ELSE '[]' END) e), false)
|
||||
$$`)
|
||||
// A spot's category (lib/spotnet/categories.ts) in Newznab's; null for pictures and the unknown.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_category(category integer, subcats jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE category
|
||||
WHEN 0 THEN CASE
|
||||
WHEN spot_has(subcats, 'z', '{3}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 6045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 6040
|
||||
WHEN spot_has(subcats, 'a', '{12,13}') THEN 6060 ELSE 6070 END
|
||||
WHEN spot_has(subcats, 'z', '{1}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 5045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 5040 ELSE 5030 END
|
||||
WHEN spot_has(subcats, 'z', '{2}') OR spot_has(subcats, 'a', '{5,11}') THEN 7020
|
||||
WHEN spot_has(subcats, 'z', '{4}') THEN NULL
|
||||
ELSE CASE WHEN spot_has(subcats, 'a', '{15}') THEN 2045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 2040 ELSE 2030 END
|
||||
END
|
||||
WHEN 1 THEN CASE WHEN spot_has(subcats, 'z', '{3}') THEN 3030 WHEN spot_has(subcats, 'a', '{2,4,7,8}') OR spot_has(subcats, 'c', '{8}') THEN 3040 ELSE 3010 END
|
||||
WHEN 2 THEN CASE WHEN spot_has(subcats, 'a', '{0,1,2}') THEN 4050 ELSE 1000 END
|
||||
WHEN 3 THEN 4000
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, NEW.groups, CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`)
|
||||
// ACCESS EXCLUSIVE for a moment, while the workers write: a few tries instead of a queue behind them.
|
||||
this.schema.raw(`
|
||||
DO $$
|
||||
BEGIN
|
||||
FOR attempt IN 1..30 LOOP
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'releases' AND column_name = 'category' AND is_generated = 'ALWAYS') THEN
|
||||
ALTER TABLE releases ALTER COLUMN category DROP EXPRESSION;
|
||||
END IF;
|
||||
DROP TRIGGER IF EXISTS releases_category ON releases;
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, groups ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category();
|
||||
RETURN;
|
||||
EXCEPTION WHEN lock_not_available THEN
|
||||
PERFORM pg_sleep(2);
|
||||
END;
|
||||
END LOOP;
|
||||
RAISE EXCEPTION 'releases: no lock for the category trigger in 30 tries';
|
||||
END
|
||||
$$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS releases_category ON releases`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS releases_set_category()`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_category(integer, jsonb)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_has(jsonb, text, integer[])`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_category_for(text, text, jsonb, integer)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_group_category(jsonb)`)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */
|
||||
/** Newznab categories this indexer assigns (release_category_for() and spot_category() in SQL, migration …5200). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 1000, name: 'Console', subcats: [] },
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3030, name: 'Audiobook' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [{ id: 4050, name: 'Games' }] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }, { id: 5070, name: 'Anime' }] },
|
||||
{ id: 6000, name: 'XXX', subcats: [{ id: 6040, name: 'x264' }, { id: 6045, name: 'UHD' }, { id: 6060, name: 'ImgSet' }, { id: 6070, name: 'Other' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }, { id: 7030, name: 'Comics' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
|
||||
+7
-4
@@ -25,10 +25,13 @@ const PASSWORD_SOURCES = {
|
||||
}
|
||||
/** Newznab categories (lib/newznab/categories.ts). */
|
||||
const CATEGORIES = [
|
||||
[1000, 'Console', []],
|
||||
[2000, 'Movies', [[2030, 'SD'], [2040, 'HD'], [2045, 'UHD']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3040, 'Lossless']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD']]],
|
||||
[7000, 'Books', [[7020, 'Ebook']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3030, 'Audiobook'], [3040, 'Lossless']]],
|
||||
[4000, 'PC', [[4050, 'Games']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD'], [5070, 'Anime']]],
|
||||
[6000, 'XXX', [[6040, 'x264'], [6045, 'UHD'], [6060, 'ImgSet'], [6070, 'Other']]],
|
||||
[7000, 'Books', [[7020, 'Ebook'], [7030, 'Comics']]],
|
||||
[8000, 'Other', [[8010, 'Misc']]],
|
||||
]
|
||||
function categoryLabel(id) {
|
||||
@@ -229,7 +232,7 @@ function releaseRow(release) {
|
||||
if (release.passworded && release.passwordKnown) badges.append(badge('Password known', 'warn', 'Its archive is encrypted; the password is in the NZB'))
|
||||
else if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
|
||||
if (!release.settled) badges.append(badge('Collecting', 'accent', 'Recently seen: more files may still arrive'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name)'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name, the inner name, a spot or the groups)'))
|
||||
if (release.source && release.source !== 'stem' && release.source !== 'subject') {
|
||||
badges.append(badge(SOURCE_LABELS[release.source] ?? release.source, 'accent', 'How this release was recognised'))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
|
||||
const categoryOf = async (name: string) => Number((await db.rawQuery(`SELECT release_category(?) AS c`, [name])).rows[0].c)
|
||||
|
||||
test.group('Release categories', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('by name: the sources and codes the first function lost or missed, XXX, audio, books, games', async ({ assert }) => {
|
||||
const expected: [string, number][] = [
|
||||
// Lost to knex's bindings in migration 1200 (`blu-$1ray`, `web-$2dl`, `h\.$326[45]`).
|
||||
['Some.Movie.2020.BluRay.DTS-X', 2040],
|
||||
['Some.Movie.2020.WEB-DL.DDP5', 2030],
|
||||
['Some.Movie.2020.H.264', 2030],
|
||||
['Some.Movie.2020.2160p.WEB-DL', 2045],
|
||||
// TV codes it missed.
|
||||
['Rizzoli & Isles S01e07[Mux XviD', 5030],
|
||||
['Rush S04E01E02 WS PDTVXviD', 5030],
|
||||
['Show.S01E02.1080p.WEB', 5040],
|
||||
['Show.1x02.HDTV', 5030],
|
||||
['Resolution 1920x1080 wallpapers', 8010],
|
||||
['Studio.24.10.22.Some.Name.XXX.1080p.MP4-GRP', 6040],
|
||||
['Studio.Some.Name.XXX.2160p', 6045],
|
||||
['Some.Name.XXX.IMAGESET-GRP', 6060],
|
||||
['Artist-Album-WEB-2026-GRP', 3010],
|
||||
['Artist - Album (2020) FLAC', 3040],
|
||||
['Author - Title (Audiobook)', 3030],
|
||||
['Book.Title.EPUB', 7020],
|
||||
['Game.Title-CODEX', 4050],
|
||||
['Game.Title.NSW-VENOM', 1000],
|
||||
['TsIgNBewuFbuwWy', 8010],
|
||||
]
|
||||
const got: [string, number][] = []
|
||||
for (const [name] of expected) got.push([name, await categoryOf(name)])
|
||||
assert.deepEqual(got, expected)
|
||||
})
|
||||
|
||||
test("the name's, else the inner name's, else a given one (a spot's), else the groups'", async ({ assert }) => {
|
||||
const row = (key: string, extra: Record<string, unknown>) => ({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), source: 'stem', files: 1, groups: '["alt.binaries.misc"]', ...extra })
|
||||
await db.table('releases').multiInsert([
|
||||
row('a', { name: 'Show.S01E02.1080p.WEB' }),
|
||||
row('b', { name: 'TsIgNBewuFbuwWy' }),
|
||||
row('c', { name: 'TsIgNBewuFbuwWy', groups: '["alt.binaries.pictures.erotica.puffies"]' }),
|
||||
// A spot's release of its own, inserted with the spot's category.
|
||||
row('d', { name: 'Some spot title', category: 2040 }),
|
||||
])
|
||||
const byName = async () => Object.fromEntries((await db.from('releases').select('release_key', 'category')).map((r) => [r.release_key[0], r.category]))
|
||||
assert.deepEqual(await byName(), { a: 5040, b: 8010, c: 6060, d: 2040 })
|
||||
// The prober finds an inner name; a spot links a release (the spot inspector sets a Misc one's category).
|
||||
await db.from('releases').where('release_key', 'b'.padEnd(40, '0')).update({ inner_name: 'Some.Movie.2020.1080p.BluRay.x264-GRP.mkv' })
|
||||
await db.from('releases').where('release_key', 'c'.padEnd(40, '0')).update({ category: 5030 })
|
||||
await db.from('releases').where('release_key', 'c'.padEnd(40, '0')).update({ inner_name: 'nothing.bin' })
|
||||
assert.deepEqual(await byName(), { a: 5040, b: 2040, c: 5030, d: 2040 })
|
||||
})
|
||||
|
||||
test("a spot's category in Newznab's", async ({ assert }) => {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spot_category(0, '{"a":[9],"z":[0]}') AS movie_hd, spot_category(0, '{"a":[15],"z":[1]}') AS series_uhd, spot_category(0, '{"a":[0],"z":[3]}') AS erotica,
|
||||
spot_category(0, '{"a":[5]}') AS epub, spot_category(0, '{"z":[4]}') AS picture, spot_category(1, '{"a":[8]}') AS flac, spot_category(1, '{"a":[0],"z":[3]}') AS audiobook,
|
||||
spot_category(2, '{"a":[0]}') AS pc_game, spot_category(2, '{"a":[12]}') AS console, spot_category(3, '{}') AS app`
|
||||
)
|
||||
assert.deepEqual(rows[0], { movie_hd: 2040, series_uhd: 5045, erotica: 6070, epub: 7020, picture: null, flac: 3040, audiobook: 3030, pc_game: 4050, console: 1000, app: 4000 })
|
||||
})
|
||||
})
|
||||
@@ -102,6 +102,8 @@ test.group('Spots', (group) => {
|
||||
assert.isTrue(await inspector.standalone(SPOT))
|
||||
const release = await db.from('releases').where('source', 'spot').firstOrFail()
|
||||
assert.equal(release.name, 'Lonely File')
|
||||
// Its title says nothing: the spot's category (01, a09: a movie, x264).
|
||||
assert.equal(release.category, 2040)
|
||||
assert.equal(release.files, 1)
|
||||
assert.isNotNull(release.poster)
|
||||
assert.isNotNull(await db.from('posters').where('poster', release.poster).first())
|
||||
|
||||
Reference in new issue
Block a user