Daan MeijerandClaude Fable 5.1
81a5dce45a
Searches with the readable filter on returned scrambles: with a random
4-letter query nearly every hit was one ("Zx5AjNaK-mh7CFg6Hf8uu",
"Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ", "bd5b...-addon"), and
15% of a 6,000-row sample of the catalog's readable names were. looksRandom
knows single-token shapes only; separators, spaces, a random "extension"
of any length and a packaging word hid the rest.
isUnreadableName (lib/release/readable.ts) and names::is_unreadable_name
read a name token by token: words (one case or Capitalized), scrambles
(letter/digit and case switches, lone letters between capitals, inner
capital runs, few vowels, hex) and neutral tokens (digits, codes like
S02E10 and 1080p, CamelCase, packaging words); a name is unreadable when
its scrambles outweigh its words. Scene tags' "i" for "I" and the
NZBSPLIT hash are read as such. On the evaluation sets: 554 of 559
random-search names caught, 0 of 630 real names flagged (LOG.md (129)).
The edge's readable flag and the prober's inner names use it; the drop
rules keep looksRandom, which is narrower. An unnamed release goes to the
pending store (decision f), so more of the obfuscated groups' releases
wait for a probe now. releases:readable clears the flag on the existing
catalog's rows by name (a dry run by default).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
84 lines
3.4 KiB
TypeScript
84 lines
3.4 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import { isUnreadableName } from '#lib/release/index'
|
|
|
|
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
|
const CURSOR = 'readable'
|
|
|
|
/**
|
|
* The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
|
|
* readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
|
|
* false, so the readable filter hides them (an inner name from the prober still shows them). By the release name,
|
|
* which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
|
|
* Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
|
|
*/
|
|
export default class ReleasesReadable extends BaseCommand {
|
|
static commandName = 'releases:readable'
|
|
static description =
|
|
'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.number({ description: 'Release ids per batch' })
|
|
declare batch: number
|
|
|
|
@flags.number({ description: 'Stop after this id (a sample)' })
|
|
declare toId: number
|
|
|
|
@flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
|
|
declare apply: boolean
|
|
|
|
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
|
declare restart: boolean
|
|
|
|
async run() {
|
|
const batch = this.batch ?? 20_000
|
|
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
|
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
|
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
|
|
CURSOR,
|
|
])
|
|
let after =
|
|
this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
|
let looked = 0
|
|
let changed = 0
|
|
const sample: string[] = []
|
|
const started = Date.now()
|
|
while (after < hi) {
|
|
const to = Math.min(after + batch, hi)
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
|
|
[after, to]
|
|
)
|
|
looked += rows.length
|
|
const ids: number[] = []
|
|
for (const r of rows) {
|
|
if (!isUnreadableName(r.name)) continue
|
|
ids.push(Number(r.id))
|
|
if (sample.length < 40) sample.push(r.name)
|
|
}
|
|
changed += ids.length
|
|
if (this.apply) {
|
|
if (ids.length > 0)
|
|
await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
|
|
ids,
|
|
])
|
|
await db.rawQuery(
|
|
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
|
[CURSOR, String(to)]
|
|
)
|
|
}
|
|
after = to
|
|
if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
|
|
this.logger.info(
|
|
`Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
|
|
)
|
|
}
|
|
}
|
|
this.logger.info(
|
|
`${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
|
|
)
|
|
if (!this.apply) for (const n of sample) this.logger.info(` ${n}`)
|
|
}
|
|
}
|