Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e4044cb1d1 | ||
|
|
61bdd43b34 | ||
|
|
ebe9f8bb36 | ||
|
|
f75097d7ac | ||
|
|
d446bb57df |
No files matched your search
+16
-7
@@ -189,13 +189,22 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
number; merged when the neighbouring range is done, then written as a
|
||||
chunk. Never staged per segment.
|
||||
|
||||
8. **Settling**. A release is settled when the ranges covering its posting
|
||||
span are done and no chunk arrived for a day: completeness is computed
|
||||
then (expected vs found files and segments). Phase 2 picks its work from
|
||||
settled releases.
|
||||
8. **Settling**. A release is settled when nothing was added to it for 6
|
||||
hours (30 minutes once its last post is that old: history), and its
|
||||
completeness decided: complete when every file its [i/N] counter
|
||||
announced was found. Without a counter, by its par2 sets once their index
|
||||
is probed (`par2_completeness`): every file a set describes is there by
|
||||
name or by its first 16 KiB, or, renamed, as many files and bytes; decided
|
||||
again after a par2 merge and before the pruner drops a set's file list.
|
||||
Segments aren't compared. Phase 2 picks its work from settled releases.
|
||||
|
||||
9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
|
||||
search (names) and get. Web UI later.
|
||||
9. **NZB and API**. NZB built from a release's chunks, with a known
|
||||
password in its meta; Newznab search (names, inner names) and get; a web
|
||||
interface. Categories (Newznab's) by a trigger on releases (migration
|
||||
5200): from the name, else the inner name the prober found, else a spot's
|
||||
category (spot_category, set by the spot inspector), else the groups
|
||||
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
|
||||
in batches.
|
||||
|
||||
## Group list
|
||||
|
||||
@@ -260,7 +269,7 @@ with 68,032 releases with a password):
|
||||
| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
|
||||
| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
|
||||
| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
|
||||
| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed; one release keeps only the tails of its highest volume (the live tail's pieces each queued one; `seven_zip_tails`) |
|
||||
| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
|
||||
| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
|
||||
| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
|
||||
|
||||
|
||||
@@ -237,8 +237,9 @@ class BatchStore {
|
||||
last_article = greatest(r.last_article, a.last_article),
|
||||
readable = r.readable OR a.readable,
|
||||
has_par2 = r.has_par2 OR a.has_par2,
|
||||
-- As it stands (the counter's files all found); settling makes it final.
|
||||
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
|
||||
-- As it stands (the counter's files all found); settling makes it final. Without a counter, what its par2
|
||||
-- set decided stays (app/services/par2_completeness.ts).
|
||||
complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN r.complete ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
|
||||
last_seen_at = now()
|
||||
FROM a WHERE r.id = a.id`,
|
||||
[JSON.stringify(updates)]
|
||||
|
||||
@@ -3,15 +3,26 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
|
||||
/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
|
||||
const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
|
||||
|
||||
/**
|
||||
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
|
||||
* chunk store: every chunk decoded, its files in name order. A file stored
|
||||
* twice (the same name and first article, e.g. a range read again) is taken
|
||||
* once.
|
||||
* once. A known password (release_passwords) goes into the NZB's meta, where
|
||||
* SABnzbd and NZBGet take it for unpacking.
|
||||
*/
|
||||
class NzbService {
|
||||
async files(releaseId: number): Promise<ChunkFile[]> {
|
||||
const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
/** The release's known passwords, the most trusted first. */
|
||||
async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
|
||||
const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
|
||||
const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
|
||||
return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
|
||||
}
|
||||
|
||||
async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
|
||||
const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
const files = new Map<string, ChunkFile>()
|
||||
for (const chunk of chunks) {
|
||||
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
|
||||
@@ -22,10 +33,11 @@ class NzbService {
|
||||
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
|
||||
if (!release) return null
|
||||
const password = (await this.passwords(releaseId))[0]?.password
|
||||
// A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
|
||||
if (release.source === 'spot') {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
|
||||
if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
|
||||
if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
|
||||
}
|
||||
const xml = buildNzb(
|
||||
(await this.files(releaseId)).map((f) => ({
|
||||
@@ -35,10 +47,18 @@ class NzbService {
|
||||
groups: f.groups,
|
||||
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
})),
|
||||
{ title: release.name }
|
||||
{ title: release.name, password }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
}
|
||||
|
||||
/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
|
||||
export function withPassword(xml: string, password: string | undefined): string {
|
||||
if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
|
||||
const meta = `<meta type="password">${password.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')}</meta>`
|
||||
if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
|
||||
return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
|
||||
}
|
||||
|
||||
export default new NzbService()
|
||||
@@ -0,0 +1,53 @@
|
||||
import NzbService from '#services/nzb_service'
|
||||
|
||||
/** A file a par2 set describes (par2_files). */
|
||||
export interface SetFile {
|
||||
name: string
|
||||
length: number
|
||||
hash16k: string
|
||||
}
|
||||
|
||||
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
||||
|
||||
/**
|
||||
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
||||
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
||||
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
||||
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
||||
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
||||
*/
|
||||
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
||||
const names = new Set(files.map((f) => base(f.name)))
|
||||
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
||||
const byName = set.filter((f) => names.has(base(f.name))).length
|
||||
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
||||
if (byName > 0) return false
|
||||
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
||||
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
||||
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
||||
* a transaction that moved chunks (a merge) reads them as moved.
|
||||
*/
|
||||
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const { rows: releases } = await client.rawQuery(
|
||||
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
let decided = 0
|
||||
for (const { id } of releases) {
|
||||
const { rows: set } = await client.rawQuery(
|
||||
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (set.length === 0) continue
|
||||
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
||||
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
||||
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
||||
decided++
|
||||
}
|
||||
return decided
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { looksRandom } from '#lib/release/random'
|
||||
import { SidecarFolder } from '#services/sidecar_folder'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||
@@ -371,12 +372,13 @@ export class Par2Merger {
|
||||
WHERE t.id = ?`,
|
||||
[name ?? '', others, target.id]
|
||||
)
|
||||
// Completeness as the settler decides it, once settled.
|
||||
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
[target.id]
|
||||
)
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
||||
await decidePar2Completeness(trx, [target.id])
|
||||
return others.length
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
||||
const SETTLED_MS = 2 * 86400_000
|
||||
@@ -86,6 +87,11 @@ export class Par2Pruner {
|
||||
for (const reason of ['complete', 'judged', 'final'] as const) {
|
||||
const ids = done(reason)
|
||||
round[reason] += ids.length
|
||||
// The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
|
||||
if (reason !== 'judged' && ids.length > 0) {
|
||||
const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
|
||||
await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
|
||||
}
|
||||
round.rows += await this.drop(ids)
|
||||
}
|
||||
// Past the last set: the next batch starts the next pass.
|
||||
|
||||
@@ -8,6 +8,7 @@ import { YencError } from '#lib/yenc/index'
|
||||
import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
interface Candidate {
|
||||
id: number
|
||||
@@ -20,7 +21,8 @@ type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'w
|
||||
|
||||
/**
|
||||
* The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
|
||||
* then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
|
||||
*/
|
||||
const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'lone', unreadable: false },
|
||||
@@ -28,6 +30,8 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'archive', unreadable: true },
|
||||
{ kind: 'par2', unreadable: false },
|
||||
{ kind: 'archive', unreadable: false },
|
||||
{ kind: '7z-tail', unreadable: true },
|
||||
{ kind: '7z-tail', unreadable: false },
|
||||
]
|
||||
const FLUSH_MS = 2000
|
||||
|
||||
@@ -169,7 +173,7 @@ export class Prober {
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
@@ -303,6 +307,8 @@ export class Prober {
|
||||
[jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
)
|
||||
await this.storeForMerge(trx, done)
|
||||
// A par2 index read: the release's completeness by its set, when it has no counter.
|
||||
await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
|
||||
// Each touched release from all its inspections (earlier rounds may have stored others).
|
||||
const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
|
||||
const { rows } = await trx.rawQuery(
|
||||
|
||||
@@ -59,7 +59,7 @@ export const RULES: Rule[] = [
|
||||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||||
// an executable by name or by its first bytes, under a media name.
|
||||
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
|
||||
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||||
UNION
|
||||
|
||||
@@ -57,6 +57,8 @@ export interface ReleaseSummary {
|
||||
settled: boolean
|
||||
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
||||
passworded: boolean | null
|
||||
/** A password is known (release_passwords): it's in the NZB. */
|
||||
passwordKnown: boolean
|
||||
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
||||
innerName: string | null
|
||||
}
|
||||
@@ -163,6 +165,7 @@ class SearchService {
|
||||
const [{ rows }, { rows: counted }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
||||
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
||||
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...source.bindings, ...bindings, perPage, offset]
|
||||
@@ -192,6 +195,7 @@ class SearchService {
|
||||
category: r.category,
|
||||
settled: r.settled_at !== null,
|
||||
passworded: r.passworded,
|
||||
passwordKnown: r.password_known,
|
||||
innerName: r.inner_name,
|
||||
})),
|
||||
}
|
||||
@@ -201,10 +205,11 @@ class SearchService {
|
||||
async release(id: number) {
|
||||
const r = await db.from('releases').where('id', id).first()
|
||||
if (!r) return null
|
||||
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
|
||||
const [files, { rows: inspections }, { rows: finality }, passwords] = await Promise.all([
|
||||
NzbService.files(id),
|
||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
||||
NzbService.passwords(id),
|
||||
])
|
||||
return {
|
||||
id: Number(r.id),
|
||||
@@ -226,6 +231,8 @@ class SearchService {
|
||||
lastSeenAt: r.last_seen_at,
|
||||
settledAt: r.settled_at,
|
||||
passworded: r.passworded,
|
||||
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
|
||||
passwords,
|
||||
innerName: r.inner_name,
|
||||
inspectedAt: r.inspected_at,
|
||||
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
||||
|
||||
@@ -15,7 +15,8 @@ const BATCH = 50_000
|
||||
* Settles releases (REBUILD.md, part 8): once nothing has been added for
|
||||
* QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
|
||||
* [i/N] counter announced was found, unknown without a counter (phase 2
|
||||
* reads par2 sets for those). Phase 2 picks its work from settled releases.
|
||||
* decides those by their par2 sets: app/services/par2_completeness.ts).
|
||||
* Phase 2 picks its work from settled releases.
|
||||
*/
|
||||
class Settler {
|
||||
/** Up to `batches` batches of BATCH (all there are without). */
|
||||
@@ -25,7 +26,7 @@ class Settler {
|
||||
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH settled AS (
|
||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
|
||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END
|
||||
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
||||
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
||||
RETURNING id, readable, files, bytes, poster, sidecar_only),
|
||||
|
||||
@@ -294,6 +294,12 @@ export class SpotInspector {
|
||||
.ignore()
|
||||
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
|
||||
}
|
||||
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
|
||||
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
|
||||
spot.category,
|
||||
JSON.stringify(spot.subcats ?? {}),
|
||||
[...perRelease.keys()],
|
||||
])
|
||||
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
|
||||
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
|
||||
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
|
||||
@@ -347,6 +353,8 @@ export class SpotInspector {
|
||||
readable: true,
|
||||
passworded: spot.password ? true : null,
|
||||
settled_at: new Date(),
|
||||
// Kept when its title says nothing (the category trigger, migration 5200).
|
||||
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
|
||||
})
|
||||
.onConflict('release_key')
|
||||
.ignore()
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases per batch. */
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Decides completeness by par2 set (app/services/par2_completeness.ts) for the releases that had their par2 read
|
||||
* before it was built (2026-10-23): every holder of a set whose file list is still there, by release id. Once.
|
||||
*/
|
||||
export default class ReleasesPar2Complete extends BaseCommand {
|
||||
static commandName = 'releases:par2-complete'
|
||||
static description = 'Decides completeness by their par2 sets for settled releases without a counter, then exits'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
async run() {
|
||||
let after = 0
|
||||
let looked = 0
|
||||
let decided = 0
|
||||
for (;;) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT s.release_id FROM par2_sets s
|
||||
WHERE s.release_id > ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
ORDER BY s.release_id LIMIT ?`,
|
||||
[after, BATCH]
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
const ids = rows.map((r: { release_id: string }) => Number(r.release_id))
|
||||
decided += await decidePar2Completeness(db, ids)
|
||||
looked += ids.length
|
||||
after = ids.at(-1)!
|
||||
if (looked % 50_000 < BATCH) this.logger.info(`${looked} releases with a par2 set looked at, ${decided} decided (up to id ${after}).`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} releases with a par2 set looked at, ${decided} decided.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
|
||||
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||
const CURSOR = 'recategorize'
|
||||
|
||||
/**
|
||||
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
|
||||
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
|
||||
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
|
||||
* the dead versions reusable) and goes on from its cursor when run again.
|
||||
*/
|
||||
export default class ReleasesRecategorize extends BaseCommand {
|
||||
static commandName = 'releases:recategorize'
|
||||
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.number({ description: 'Release ids per batch' })
|
||||
declare batch: number
|
||||
|
||||
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
|
||||
declare toId: number
|
||||
|
||||
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
|
||||
declare floorGb: number
|
||||
|
||||
@flags.boolean({ description: 'Count what would change, write nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||
declare restart: boolean
|
||||
|
||||
async run() {
|
||||
const batch = this.batch ?? 20_000
|
||||
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
|
||||
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||
const changes = new Map<string, number>()
|
||||
let changed = 0
|
||||
let looked = 0
|
||||
const started = Date.now()
|
||||
while (after < hi) {
|
||||
while (!this.dryRun && (await space.short())) await sleep(60_000)
|
||||
const to = Math.min(after + batch, hi)
|
||||
const fresh = `release_category_for(name, inner_name, groups, NULL)`
|
||||
if (this.dryRun) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
|
||||
[after, to]
|
||||
)
|
||||
for (const r of rows) {
|
||||
looked += r.n
|
||||
if (r.was === r.now) continue
|
||||
changed += r.n
|
||||
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
|
||||
}
|
||||
} else {
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
|
||||
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
|
||||
[after, to]
|
||||
)
|
||||
changed += result.rowCount ?? 0
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, String(to)]
|
||||
)
|
||||
}
|
||||
after = to
|
||||
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
|
||||
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
|
||||
}
|
||||
}
|
||||
if (this.dryRun) {
|
||||
this.logger.info(`Would change ${changed} of ${looked}:`)
|
||||
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
|
||||
return
|
||||
}
|
||||
// The spots' categories for the linked releases whose names say nothing.
|
||||
const spots = await db.rawQuery(
|
||||
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
|
||||
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
|
||||
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
|
||||
)
|
||||
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release's category from more than its name (REBUILD.md, part 9; LOG.md 2026-10-23 (95)). Migration 1200 made it
|
||||
* a generated column of release_category(name), and knex took the `?` in its regexes for bindings: the function held
|
||||
* `blu-$1ray`, `web-$2dl`, `h\.$326[45]`, so BluRay, WEB-DL and H.264 names without a resolution were Misc. Now:
|
||||
* - release_category(name), fixed (no `?` in any regex here: `{0,1}`) and with XXX, audiobooks, console and PC games,
|
||||
* and TV codes the old one missed (`S01e07[`, `S04E01E02`);
|
||||
* - the inner name the prober found (an obfuscated release's real name) when the name says nothing;
|
||||
* - else a category given before (a spot's, below), else the groups' (erotica, sounds, ebooks, …);
|
||||
* - a spot's category (spot_category: its head category and subcategories), set by the spot inspector.
|
||||
* `category` becomes a plain column (DROP EXPRESSION keeps the values, no rewrite) set by a trigger; existing rows are
|
||||
* recategorized by `releases:recategorize` in batches, not here (a rewrite of the 70 GB table).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
-- XXX first: its names carry years, resolutions and episode-like codes too.
|
||||
WHEN name ~* '(^|[^a-z0-9])xxx([^a-z0-9]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 6045
|
||||
WHEN name ~* '720p|1080p|x26[45]|h\.{0,1}26[45]|hevc' THEN 6040
|
||||
WHEN name ~* 'imageset|picset|(^|[._ -])pics([._ -]|$)' THEN 6060 ELSE 6070 END
|
||||
-- TV: S01E02 (also S01e02[, S04E01E02), 1x02, or a season pack (S01).
|
||||
WHEN name ~* '(^|[^a-z0-9])(s[0-9]{1,2}[ ._-]{0,1}e[0-9]{1,3}|[0-9]{1,2}x[0-9]{2})([^0-9]|$)' OR name ~* '(^|[._ -])s[0-9]{1,2}([._ -]|$)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
|
||||
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
|
||||
-- Movies: a year plus a source or quality marker.
|
||||
WHEN name ~* '(^|[._ (-])(19|20)[0-9]{2}([._ )-]|$)'
|
||||
AND name ~* '(480p|576p|720p|1080p|2160p|blu-{0,1}ray|web-{0,1}dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.{0,1}26[45]|hevc|remux)' THEN
|
||||
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
|
||||
WHEN name ~* '720p|1080p|blu-{0,1}ray' THEN 2040 ELSE 2030 END
|
||||
-- Audio: audiobooks, lossless, mp3, or scene music names (-WEB-2026-GROUP).
|
||||
WHEN name ~* '(audiobook|audioboek|luisterboek|h(oe|ö)rbuch)' THEN 3030
|
||||
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
|
||||
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)[0-9]{2}-[A-Za-z0-9]+$' THEN 3010
|
||||
WHEN name ~* '(^|[._ -])(epub|ebook|e-book|mobi|azw3)([._ -]|$)' THEN 7020
|
||||
-- Games: console tags, PC scene groups and repackers.
|
||||
WHEN name ~* '(^|[._ -])(nsw|ps4|ps5|xboxone|xbox360|wiiu|3ds|nds)([._ -]|$)' THEN 1000
|
||||
WHEN name ~* '-(codex|plaza|skidrow|reloaded|flt|doge|tenoke|rune|empress|razor1911|tinyiso|darksiders|simplex)$' OR name ~* '(fitgirl|dodi|elamigos)[._ -]{0,1}repack' THEN 4050
|
||||
ELSE 8010
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(String.raw`
|
||||
CREATE OR REPLACE FUNCTION release_group_category(groups jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE
|
||||
WHEN g ~* '(erotica|xxx|porn|erotic|nude|naked)' THEN CASE WHEN g ~* '(pictures|pics|images|photos)' THEN 6060 ELSE 6070 END
|
||||
WHEN g ~* 'anime' THEN 5070
|
||||
WHEN g ~* 'audiobook' THEN 3030
|
||||
WHEN g ~* '(sounds|mp3|music)' THEN CASE WHEN g ~* '(lossless|flac)' THEN 3040 ELSE 3010 END
|
||||
WHEN g ~* 'comics' THEN 7030
|
||||
WHEN g ~* 'e-{0,1}books{0,1}' THEN 7020
|
||||
WHEN g ~* '(games|nintendo|xbox|playstation|wii)' THEN 1000
|
||||
WHEN g ~* '(^|[. ])(tv|hdtv|teevee|tvseries)([. ]|$)' THEN 5000
|
||||
WHEN g ~* '(^|[. ])(movies|dvd|bluray|dvdr)([. ]|$)' THEN 2000
|
||||
ELSE 8010
|
||||
END
|
||||
FROM (SELECT coalesce(string_agg(value, ' '), '') AS g FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(groups) = 'array' THEN groups ELSE '[]' END)) x
|
||||
$$`)
|
||||
// The name's, else the inner name's, else one given before (a spot's), else the groups'.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION release_category_for(name text, inner_name text, groups jsonb, given integer) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE WHEN n <> 8010 THEN n WHEN i <> 8010 THEN i WHEN coalesce(given, 8010) <> 8010 THEN given ELSE release_group_category(groups) END
|
||||
FROM (SELECT release_category(name) AS n, coalesce(release_category(inner_name), 8010) AS i) x
|
||||
$$`)
|
||||
// Whether a spot's subcategories of one type letter ({"a": [9], "z": [0]}) hold one of `vals`.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_has(subcats jsonb, letter text, vals integer[]) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT coalesce((SELECT bool_or((e #>> '{}')::integer = ANY(vals)) FROM jsonb_array_elements(
|
||||
CASE WHEN jsonb_typeof(subcats->letter) = 'array' THEN subcats->letter ELSE '[]' END) e), false)
|
||||
$$`)
|
||||
// A spot's category (lib/spotnet/categories.ts) in Newznab's; null for pictures and the unknown.
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION spot_category(category integer, subcats jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
|
||||
SELECT CASE category
|
||||
WHEN 0 THEN CASE
|
||||
WHEN spot_has(subcats, 'z', '{3}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 6045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 6040
|
||||
WHEN spot_has(subcats, 'a', '{12,13}') THEN 6060 ELSE 6070 END
|
||||
WHEN spot_has(subcats, 'z', '{1}') THEN CASE WHEN spot_has(subcats, 'a', '{15}') THEN 5045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 5040 ELSE 5030 END
|
||||
WHEN spot_has(subcats, 'z', '{2}') OR spot_has(subcats, 'a', '{5,11}') THEN 7020
|
||||
WHEN spot_has(subcats, 'z', '{4}') THEN NULL
|
||||
ELSE CASE WHEN spot_has(subcats, 'a', '{15}') THEN 2045 WHEN spot_has(subcats, 'a', '{4,6,7,8,9}') THEN 2040 ELSE 2030 END
|
||||
END
|
||||
WHEN 1 THEN CASE WHEN spot_has(subcats, 'z', '{3}') THEN 3030 WHEN spot_has(subcats, 'a', '{2,4,7,8}') OR spot_has(subcats, 'c', '{8}') THEN 3040 ELSE 3010 END
|
||||
WHEN 2 THEN CASE WHEN spot_has(subcats, 'a', '{0,1,2}') THEN 4050 ELSE 1000 END
|
||||
WHEN 3 THEN 4000
|
||||
END
|
||||
$$`)
|
||||
this.schema.raw(`
|
||||
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.category := release_category_for(NEW.name, NEW.inner_name, NEW.groups, CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`)
|
||||
// ACCESS EXCLUSIVE for a moment, while the workers write: a few tries instead of a queue behind them.
|
||||
this.schema.raw(`
|
||||
DO $$
|
||||
BEGIN
|
||||
FOR attempt IN 1..30 LOOP
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'releases' AND column_name = 'category' AND is_generated = 'ALWAYS') THEN
|
||||
ALTER TABLE releases ALTER COLUMN category DROP EXPRESSION;
|
||||
END IF;
|
||||
DROP TRIGGER IF EXISTS releases_category ON releases;
|
||||
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, groups ON releases
|
||||
FOR EACH ROW EXECUTE FUNCTION releases_set_category();
|
||||
RETURN;
|
||||
EXCEPTION WHEN lock_not_available THEN
|
||||
PERFORM pg_sleep(2);
|
||||
END;
|
||||
END LOOP;
|
||||
RAISE EXCEPTION 'releases: no lock for the category trigger in 30 tries';
|
||||
END
|
||||
$$`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TRIGGER IF EXISTS releases_category ON releases`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS releases_set_category()`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_category(integer, jsonb)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS spot_has(jsonb, text, integer[])`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_category_for(text, text, jsonb, integer)`)
|
||||
this.schema.raw(`DROP FUNCTION IF EXISTS release_group_category(jsonb)`)
|
||||
}
|
||||
}
|
||||
+40
-1
@@ -915,12 +915,17 @@ fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate>
|
||||
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
||||
for f in files {
|
||||
let name = &f[0].file.name;
|
||||
if let Some(volume) = names::seven_zip_volume(name).filter(|&v| v > 1) {
|
||||
let volume = names::seven_zip_volume(name);
|
||||
if let Some(volume) = volume.filter(|&v| v > 1) {
|
||||
if tail.is_none_or(|(v, _)| volume > v) {
|
||||
tail = Some((volume, f));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if volume == Some(0) && f.len() > 1 && tail.is_none() {
|
||||
tail = Some((0, f));
|
||||
}
|
||||
if names::is_archive_start(name) {
|
||||
out.push(c("archive", &f[0]));
|
||||
} else if names::is_par2(name) && !names::is_later_volume(name) {
|
||||
@@ -965,6 +970,40 @@ mod tests {
|
||||
assert_eq!(batch.releases[0].key, batch.releases[1].key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_candidates_as_in_the_typescript() {
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(GROUP.to_string(), 1);
|
||||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||||
let file = |at: i64, release: &str, name: &str, n: usize, parts: i64| -> Vec<String> {
|
||||
(0..parts).map(|i| line(at + i, &format!(r#"{release} [{n}/3] - "{name}" yEnc ({}/{parts})"#, i + 1), 0)).collect()
|
||||
};
|
||||
let mut lines = Vec::new();
|
||||
// A split 7z: its start, and the end of its highest volume.
|
||||
lines.extend(file(100, "Split.Seven", "Split.Seven.7z.001", 1, 2));
|
||||
lines.extend(file(110, "Split.Seven", "Split.Seven.7z.003", 3, 2));
|
||||
lines.extend(file(120, "Split.Seven", "Split.Seven.7z.002", 2, 2));
|
||||
// An unsplit 7z of 3 segments: its start and its end.
|
||||
lines.extend(file(200, "Whole.Seven", "Whole.Seven.7z", 1, 3));
|
||||
// A ZIP: its start.
|
||||
lines.extend(file(300, "Some.Zip", "Some.Zip.zip", 1, 2));
|
||||
ix.add_text(&lines.join("\r\n"), 10_001);
|
||||
ix.finish();
|
||||
let batch = ix.take();
|
||||
let mut got: Vec<(&str, &str, &str)> = batch.candidates.iter().map(|c| (c.kind, c.file.as_str(), c.message_id.as_str())).collect();
|
||||
got.sort();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
("7z-tail", "Split.Seven.7z.003", "u111@test"),
|
||||
("7z-tail", "Whole.Seven.7z", "u202@test"),
|
||||
("archive", "Some.Zip.zip", "u300@test"),
|
||||
("archive", "Split.Seven.7z.001", "u100@test"),
|
||||
("archive", "Whole.Seven.7z", "u200@test"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copies_and_fills_as_in_the_typescript() {
|
||||
let hour = 3_600_000i64;
|
||||
|
||||
+1
-1
@@ -1030,7 +1030,7 @@ static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(bin|zip)$"));
|
||||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||||
|
||||
@@ -434,6 +434,8 @@ export function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[],
|
||||
if (!tail || volume > tail.volume) tail = { volume, file: f }
|
||||
continue
|
||||
}
|
||||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||||
if (volume === 0 && f.segments.length > 1 && !tail) tail = { volume, file: f }
|
||||
if (isArchiveStart(name)) out.push({ releaseKey: key, upload, kind: 'archive', messageId: f.segments[0].messageId, file: name })
|
||||
else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, upload, kind: 'par2', messageId: f.segments[0].messageId, file: name })
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001: its type; the listing is at
|
||||
* its end, the 7z-tail), ZIP (its first entries) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
if (/\.(bin|zip)$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */
|
||||
/** Newznab categories this indexer assigns (release_category_for() and spot_category() in SQL, migration …5200). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 1000, name: 'Console', subcats: [] },
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3030, name: 'Audiobook' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [{ id: 4050, name: 'Games' }] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }, { id: 5070, name: 'Anime' }] },
|
||||
{ id: 6000, name: 'XXX', subcats: [{ id: 6040, name: 'x264' }, { id: 6045, name: 'UHD' }, { id: 6060, name: 'ImgSet' }, { id: 6070, name: 'Other' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }, { id: 7030, name: 'Comics' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
|
||||
+25
-3
@@ -2,6 +2,8 @@ import { createHash } from 'node:crypto'
|
||||
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||
import { parseZipStart } from '#lib/zip/index'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
|
||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||
@@ -55,8 +57,11 @@ function largestFirst(items: { name: string; size: number }[]): string[] {
|
||||
.map((i) => i.name.slice(0, MAX_NAME_LENGTH))
|
||||
}
|
||||
|
||||
/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
|
||||
export function inspect(body: Buffer): Inspection {
|
||||
/**
|
||||
* One article's body: its yEnc header, what the first bytes are, and for par2, RAR, ZIP and 7z what's inside.
|
||||
* Throws when it isn't yEnc. `kind` is the probe candidate's: a 7z-tail's article ends the archive.
|
||||
*/
|
||||
export function inspect(body: Buffer, kind?: string): Inspection {
|
||||
const segment = decodeBody(body)
|
||||
const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
|
||||
// The file's first 16 KiB, or all of a smaller file.
|
||||
@@ -85,6 +90,23 @@ export function inspect(body: Buffer): Inspection {
|
||||
result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
|
||||
}
|
||||
}
|
||||
if (result.type === 'zip') {
|
||||
const start = parseZipStart(segment.data)
|
||||
if (start) {
|
||||
result.entries = start.files.length
|
||||
result.names = largestFirst(start.files)
|
||||
result.passworded = start.encrypted
|
||||
}
|
||||
}
|
||||
// 7z keeps its listing at the end: the last segment of the last volume (a 7z-tail), or a small archive whole.
|
||||
if ((kind === '7z-tail' || result.type === '7z') && segment.offset + segment.data.length === result.size) {
|
||||
const listing = parseSevenZipTail(segment.data)
|
||||
if (listing) {
|
||||
result.entries = listing.files.length
|
||||
if (listing.files.length > 0) result.names = largestFirst(listing.files)
|
||||
result.passworded = listing.encrypted
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -97,7 +119,7 @@ export function inspect(body: Buffer): Inspection {
|
||||
export function innerName(inspections: { kind: string; result: Inspection }[], releaseName = ''): string | null {
|
||||
const of = (kind: string) => inspections.filter((i) => i.kind === kind).map((i) => i.result)
|
||||
const content = (kind: string) => of(kind).flatMap((i) => i.names ?? []).find((n) => !PACKAGING.test(n))
|
||||
const pick = content('archive') ?? content('par2') ?? of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const pick = content('archive') ?? content('7z-tail') ?? content('par2') ?? of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
|
||||
const name = pick?.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||
// The release's own name with an extension says nothing new (a random-named file's yEnc name is mostly its random name).
|
||||
return name && !name.toLowerCase().startsWith(releaseName.toLowerCase() || '\0') ? name : null
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export { parseSevenZipTail, type SevenZipListing } from '#lib/sevenzip/parse'
|
||||
export { lzmaDecode, LzmaError } from '#lib/sevenzip/lzma'
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* An LZMA decoder (raw stream, as a 7z coder stores it: the properties apart, the unpacked size known), after the
|
||||
* LZMA SDK's reference decoder (LzmaSpec.cpp). For 7z headers: those are small, so the whole output is one buffer.
|
||||
*/
|
||||
|
||||
const MODEL_BITS = 11
|
||||
const BIT_MODEL_TOTAL = 1 << MODEL_BITS
|
||||
const MOVE_BITS = 5
|
||||
const TOP = 1 << 24
|
||||
const POS_BITS_MAX = 4
|
||||
const STATES = 12
|
||||
const LEN_TO_POS_STATES = 4
|
||||
const ALIGN_BITS = 4
|
||||
const END_POS_MODEL_INDEX = 14
|
||||
const FULL_DISTANCES = 1 << (END_POS_MODEL_INDEX >> 1)
|
||||
const MATCH_MIN_LEN = 2
|
||||
|
||||
export class LzmaError extends Error {}
|
||||
|
||||
const probs = (n: number) => new Uint16Array(n).fill(BIT_MODEL_TOTAL >>> 1)
|
||||
|
||||
class RangeDecoder {
|
||||
private range = 0xffffffff
|
||||
private code = 0
|
||||
|
||||
constructor(
|
||||
private input: Uint8Array,
|
||||
private pos: number
|
||||
) {
|
||||
if (this.byte() !== 0) throw new LzmaError('LZMA: bad stream start')
|
||||
for (let i = 0; i < 4; i++) this.code = ((this.code << 8) | this.byte()) >>> 0
|
||||
if (this.code === this.range) throw new LzmaError('LZMA: corrupt stream')
|
||||
}
|
||||
|
||||
private byte(): number {
|
||||
if (this.pos >= this.input.length) throw new LzmaError('LZMA: input ended')
|
||||
return this.input[this.pos++]
|
||||
}
|
||||
|
||||
private normalize() {
|
||||
if (this.range < TOP) {
|
||||
this.range = (this.range << 8) >>> 0
|
||||
this.code = ((this.code << 8) | this.byte()) >>> 0
|
||||
}
|
||||
}
|
||||
|
||||
direct(bits: number): number {
|
||||
let result = 0
|
||||
do {
|
||||
this.range >>>= 1
|
||||
this.code = (this.code - this.range) >>> 0
|
||||
const t = this.code >>> 31 ? 0xffffffff : 0
|
||||
this.code = (this.code + (this.range & t)) >>> 0
|
||||
this.normalize()
|
||||
result = ((result << 1) | (t ? 0 : 1)) >>> 0
|
||||
} while (--bits)
|
||||
return result
|
||||
}
|
||||
|
||||
bit(p: Uint16Array, i: number): number {
|
||||
const v = p[i]
|
||||
const bound = ((this.range >>> MODEL_BITS) * v) >>> 0
|
||||
let symbol: number
|
||||
if (this.code < bound) {
|
||||
p[i] = v + ((BIT_MODEL_TOTAL - v) >>> MOVE_BITS)
|
||||
this.range = bound
|
||||
symbol = 0
|
||||
} else {
|
||||
p[i] = v - (v >>> MOVE_BITS)
|
||||
this.code = (this.code - bound) >>> 0
|
||||
this.range = (this.range - bound) >>> 0
|
||||
symbol = 1
|
||||
}
|
||||
this.normalize()
|
||||
return symbol
|
||||
}
|
||||
|
||||
tree(p: Uint16Array, offset: number, bits: number): number {
|
||||
let m = 1
|
||||
for (let i = 0; i < bits; i++) m = (m << 1) + this.bit(p, offset + m)
|
||||
return m - (1 << bits)
|
||||
}
|
||||
|
||||
reverseTree(p: Uint16Array, offset: number, bits: number): number {
|
||||
let m = 1
|
||||
let symbol = 0
|
||||
for (let i = 0; i < bits; i++) {
|
||||
const bit = this.bit(p, offset + m)
|
||||
m = (m << 1) + bit
|
||||
symbol |= bit << i
|
||||
}
|
||||
return symbol
|
||||
}
|
||||
}
|
||||
|
||||
class LenDecoder {
|
||||
private choice = probs(2)
|
||||
private low = probs((1 << POS_BITS_MAX) << 3)
|
||||
private mid = probs((1 << POS_BITS_MAX) << 3)
|
||||
private high = probs(1 << 8)
|
||||
|
||||
decode(rc: RangeDecoder, posState: number): number {
|
||||
if (rc.bit(this.choice, 0) === 0) return rc.tree(this.low, posState << 3, 3)
|
||||
if (rc.bit(this.choice, 1) === 0) return 8 + rc.tree(this.mid, posState << 3, 3)
|
||||
return 16 + rc.tree(this.high, 0, 8)
|
||||
}
|
||||
}
|
||||
|
||||
/** Decodes `size` bytes of a raw LZMA stream with its 5 property bytes (lc/lp/pb, dictionary size). */
|
||||
export function lzmaDecode(properties: Uint8Array, input: Uint8Array, size: number): Buffer {
|
||||
if (properties.length < 5) throw new LzmaError('LZMA: short properties')
|
||||
let d = properties[0]
|
||||
if (d >= 9 * 5 * 5) throw new LzmaError('LZMA: bad properties')
|
||||
const lc = d % 9
|
||||
d = Math.floor(d / 9)
|
||||
const lp = d % 5
|
||||
const pb = Math.floor(d / 5)
|
||||
const dictSize = Math.max(properties[1] | (properties[2] << 8) | (properties[3] << 16) | (properties[4] << 24), 1 << 12) >>> 0
|
||||
|
||||
const out = Buffer.alloc(size)
|
||||
let pos = 0
|
||||
const rc = new RangeDecoder(input, 0)
|
||||
const literals = probs(0x300 << (lc + lp))
|
||||
const posSlot = probs(LEN_TO_POS_STATES << 6)
|
||||
const align = probs(1 << ALIGN_BITS)
|
||||
const posDecoders = probs(1 + FULL_DISTANCES - END_POS_MODEL_INDEX)
|
||||
const isMatch = probs(STATES << POS_BITS_MAX)
|
||||
const isRep = probs(STATES)
|
||||
const isRepG0 = probs(STATES)
|
||||
const isRepG1 = probs(STATES)
|
||||
const isRepG2 = probs(STATES)
|
||||
const isRep0Long = probs(STATES << POS_BITS_MAX)
|
||||
const lenDecoder = new LenDecoder()
|
||||
const repLenDecoder = new LenDecoder()
|
||||
|
||||
const distance = (len: number): number => {
|
||||
const lenState = Math.min(len, LEN_TO_POS_STATES - 1)
|
||||
const slot = rc.tree(posSlot, lenState << 6, 6)
|
||||
if (slot < 4) return slot
|
||||
const directBits = (slot >>> 1) - 1
|
||||
let dist = ((2 | (slot & 1)) * 2 ** directBits) >>> 0
|
||||
if (slot < END_POS_MODEL_INDEX) {
|
||||
dist = (dist + rc.reverseTree(posDecoders, dist - slot, directBits)) >>> 0
|
||||
} else {
|
||||
dist = (dist + rc.direct(directBits - ALIGN_BITS) * 2 ** ALIGN_BITS) >>> 0
|
||||
dist = (dist + rc.reverseTree(align, 0, ALIGN_BITS)) >>> 0
|
||||
}
|
||||
return dist
|
||||
}
|
||||
|
||||
let state = 0
|
||||
let rep0 = 0
|
||||
let rep1 = 0
|
||||
let rep2 = 0
|
||||
let rep3 = 0
|
||||
while (pos < size) {
|
||||
const posState = pos & ((1 << pb) - 1)
|
||||
if (rc.bit(isMatch, (state << POS_BITS_MAX) + posState) === 0) {
|
||||
const prev = pos > 0 ? out[pos - 1] : 0
|
||||
const base = 0x300 * (((pos & ((1 << lp) - 1)) << lc) + (prev >>> (8 - lc)))
|
||||
let symbol = 1
|
||||
if (state >= 7) {
|
||||
let match = out[pos - rep0 - 1]
|
||||
do {
|
||||
const matchBit = (match >>> 7) & 1
|
||||
match <<= 1
|
||||
const bit = rc.bit(literals, base + ((1 + matchBit) << 8) + symbol)
|
||||
symbol = (symbol << 1) | bit
|
||||
if (matchBit !== bit) break
|
||||
} while (symbol < 0x100)
|
||||
}
|
||||
while (symbol < 0x100) symbol = (symbol << 1) | rc.bit(literals, base + symbol)
|
||||
out[pos++] = symbol - 0x100
|
||||
state = state < 4 ? 0 : state < 10 ? state - 3 : state - 6
|
||||
continue
|
||||
}
|
||||
let len: number
|
||||
if (rc.bit(isRep, state) !== 0) {
|
||||
if (pos === 0) throw new LzmaError('LZMA: repeat before any output')
|
||||
if (rc.bit(isRepG0, state) === 0) {
|
||||
if (rc.bit(isRep0Long, (state << POS_BITS_MAX) + posState) === 0) {
|
||||
state = state < 7 ? 9 : 11
|
||||
out[pos] = out[pos - rep0 - 1]
|
||||
pos++
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
let dist: number
|
||||
if (rc.bit(isRepG1, state) === 0) {
|
||||
dist = rep1
|
||||
} else {
|
||||
if (rc.bit(isRepG2, state) === 0) {
|
||||
dist = rep2
|
||||
} else {
|
||||
dist = rep3
|
||||
rep3 = rep2
|
||||
}
|
||||
rep2 = rep1
|
||||
}
|
||||
rep1 = rep0
|
||||
rep0 = dist
|
||||
}
|
||||
len = repLenDecoder.decode(rc, posState)
|
||||
state = state < 7 ? 8 : 11
|
||||
} else {
|
||||
rep3 = rep2
|
||||
rep2 = rep1
|
||||
rep1 = rep0
|
||||
len = lenDecoder.decode(rc, posState)
|
||||
state = state < 7 ? 7 : 10
|
||||
rep0 = distance(len)
|
||||
// The end marker: allowed only where the output is complete, which the loop already ended at.
|
||||
if (rep0 === 0xffffffff) throw new LzmaError('LZMA: end marker before the end')
|
||||
if (rep0 >= dictSize || rep0 >= pos) throw new LzmaError('LZMA: distance out of range')
|
||||
}
|
||||
len += MATCH_MIN_LEN
|
||||
if (len > size - pos) throw new LzmaError('LZMA: match past the end')
|
||||
for (let i = 0; i < len; i++, pos++) out[pos] = out[pos - rep0 - 1]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
import { lzmaDecode } from '#lib/sevenzip/lzma'
|
||||
|
||||
/**
|
||||
* What a 7z archive's header says (its listing): read from the archive's end, where 7z keeps it, so the last
|
||||
* segment of the last volume is enough (the `7z-tail` probe; REBUILD.md, phase 2). The header there is mostly
|
||||
* an encoded one: a pointer to the real header, LZMA-packed just before it, or AES-encrypted (`7z -mhe`), which
|
||||
* hides the names and means a password.
|
||||
*/
|
||||
export interface SevenZipListing {
|
||||
/** Files, not directories, with their unpacked size (0 for an empty one). */
|
||||
files: { name: string; size: number }[]
|
||||
/** Entries or the header encrypted (AES). */
|
||||
encrypted: boolean
|
||||
/** The header itself encrypted: no names known. */
|
||||
headerEncrypted: boolean
|
||||
}
|
||||
|
||||
const AES = '06f10701'
|
||||
const LZMA = '030101'
|
||||
/** A header bigger than this isn't unpacked (headers list names; 64 MiB is ~a million files). */
|
||||
const MAX_HEADER = 64 * 1024 * 1024
|
||||
|
||||
const ID = {
|
||||
end: 0x00,
|
||||
header: 0x01,
|
||||
archiveProperties: 0x02,
|
||||
additionalStreamsInfo: 0x03,
|
||||
mainStreamsInfo: 0x04,
|
||||
filesInfo: 0x05,
|
||||
packInfo: 0x06,
|
||||
unpackInfo: 0x07,
|
||||
subStreamsInfo: 0x08,
|
||||
size: 0x09,
|
||||
crc: 0x0a,
|
||||
folder: 0x0b,
|
||||
codersUnpackSize: 0x0c,
|
||||
numUnpackStream: 0x0d,
|
||||
emptyStream: 0x0e,
|
||||
emptyFile: 0x0f,
|
||||
name: 0x11,
|
||||
encodedHeader: 0x17,
|
||||
dummy: 0x19,
|
||||
}
|
||||
|
||||
class Malformed extends Error {}
|
||||
|
||||
interface Coder {
|
||||
id: string
|
||||
properties: Uint8Array
|
||||
inStreams: number
|
||||
outStreams: number
|
||||
}
|
||||
interface Folder {
|
||||
coders: Coder[]
|
||||
bindOut: Set<number>
|
||||
unpackSizes: number[]
|
||||
crc?: number
|
||||
}
|
||||
interface StreamsInfo {
|
||||
packPos: number
|
||||
packSizes: number[]
|
||||
folders: Folder[]
|
||||
/** Per folder, how many files' streams it holds, and every stream's size in order. */
|
||||
streamsPerFolder: number[]
|
||||
streamSizes: number[]
|
||||
}
|
||||
|
||||
class Reader {
|
||||
constructor(
|
||||
private buf: Uint8Array,
|
||||
public pos = 0
|
||||
) {}
|
||||
|
||||
get done() {
|
||||
return this.pos >= this.buf.length
|
||||
}
|
||||
|
||||
byte(): number {
|
||||
if (this.pos >= this.buf.length) throw new Malformed('7z: header ended')
|
||||
return this.buf[this.pos++]
|
||||
}
|
||||
|
||||
bytes(n: number): Uint8Array {
|
||||
if (n < 0 || this.pos + n > this.buf.length) throw new Malformed('7z: header ended')
|
||||
const out = this.buf.subarray(this.pos, this.pos + n)
|
||||
this.pos += n
|
||||
return out
|
||||
}
|
||||
|
||||
/** 7z's variable-length number: the first byte's high bits say how many bytes follow. */
|
||||
number(): number {
|
||||
const first = this.byte()
|
||||
let mask = 0x80
|
||||
let value = 0
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * 2 ** (8 * i)
|
||||
break
|
||||
}
|
||||
value += this.byte() * 2 ** (8 * i)
|
||||
mask >>>= 1
|
||||
}
|
||||
if (value > Number.MAX_SAFE_INTEGER) throw new Malformed('7z: number too large')
|
||||
return value
|
||||
}
|
||||
|
||||
uint32(): number {
|
||||
const b = this.bytes(4)
|
||||
return (b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)) >>> 0
|
||||
}
|
||||
|
||||
bits(n: number): boolean[] {
|
||||
const out: boolean[] = []
|
||||
let byte = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (i % 8 === 0) byte = this.byte()
|
||||
out.push((byte & (0x80 >>> i % 8)) !== 0)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** "All defined" or a bit vector, then a CRC per defined item. */
|
||||
digests(n: number): (number | undefined)[] {
|
||||
const allDefined = this.byte() !== 0
|
||||
const defined = allDefined ? Array(n).fill(true) : this.bits(n)
|
||||
return defined.map((d) => (d ? this.uint32() : undefined))
|
||||
}
|
||||
|
||||
expect(id: number) {
|
||||
if (this.byte() !== id) throw new Malformed(`7z: expected property ${id}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Counts that come from the data: a garbage position makes huge ones, which shouldn't allocate. */
|
||||
function count(n: number, max = 1_000_000): number {
|
||||
if (n > max) throw new Malformed('7z: implausible count')
|
||||
return n
|
||||
}
|
||||
|
||||
function readFolder(r: Reader): Folder {
|
||||
const coders: Coder[] = []
|
||||
const numCoders = count(r.number(), 64)
|
||||
if (numCoders === 0) throw new Malformed('7z: folder without coders')
|
||||
for (let i = 0; i < numCoders; i++) {
|
||||
const flags = r.byte()
|
||||
if (flags & 0x80) throw new Malformed('7z: alternative coder methods')
|
||||
const id = Buffer.from(r.bytes(flags & 0x0f)).toString('hex')
|
||||
const complex = (flags & 0x10) !== 0
|
||||
const inStreams = complex ? count(r.number(), 64) : 1
|
||||
const outStreams = complex ? count(r.number(), 64) : 1
|
||||
const properties = flags & 0x20 ? r.bytes(count(r.number(), 1 << 16)) : new Uint8Array(0)
|
||||
coders.push({ id, properties, inStreams, outStreams })
|
||||
}
|
||||
const totalOut = coders.reduce((n, c) => n + c.outStreams, 0)
|
||||
const totalIn = coders.reduce((n, c) => n + c.inStreams, 0)
|
||||
const bindOut = new Set<number>()
|
||||
for (let i = 0; i < totalOut - 1; i++) {
|
||||
r.number()
|
||||
bindOut.add(r.number())
|
||||
}
|
||||
const packed = totalIn - (totalOut - 1)
|
||||
if (packed < 1) throw new Malformed('7z: folder without packed streams')
|
||||
if (packed > 1) for (let i = 0; i < packed; i++) r.number()
|
||||
return { coders, bindOut, unpackSizes: [] }
|
||||
}
|
||||
|
||||
/** The folder's output: the one out stream no bind pair consumes. */
|
||||
function unpackSize(folder: Folder): number {
|
||||
for (let i = folder.unpackSizes.length - 1; i >= 0; i--) if (!folder.bindOut.has(i)) return folder.unpackSizes[i]
|
||||
return 0
|
||||
}
|
||||
|
||||
function readStreamsInfo(r: Reader): StreamsInfo {
|
||||
const info: StreamsInfo = { packPos: 0, packSizes: [], folders: [], streamsPerFolder: [], streamSizes: [] }
|
||||
let id = r.byte()
|
||||
if (id === ID.packInfo) {
|
||||
info.packPos = r.number()
|
||||
const n = count(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.size) for (let i = 0; i < n; i++) info.packSizes.push(r.number())
|
||||
else if (id === ID.crc) r.digests(n)
|
||||
else throw new Malformed('7z: unexpected property in pack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.unpackInfo) {
|
||||
r.expect(ID.folder)
|
||||
const n = count(r.number())
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external folders')
|
||||
for (let i = 0; i < n; i++) info.folders.push(readFolder(r))
|
||||
r.expect(ID.codersUnpackSize)
|
||||
for (const f of info.folders) for (const c of f.coders) for (let i = 0; i < c.outStreams; i++) f.unpackSizes.push(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.crc) r.digests(n).forEach((crc, i) => (info.folders[i].crc = crc))
|
||||
else throw new Malformed('7z: unexpected property in unpack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
info.streamsPerFolder = info.folders.map(() => 1)
|
||||
if (id === ID.subStreamsInfo) {
|
||||
id = r.byte()
|
||||
if (id === ID.numUnpackStream) {
|
||||
info.streamsPerFolder = info.folders.map(() => count(r.number()))
|
||||
id = r.byte()
|
||||
}
|
||||
const hasSizes = id === ID.size
|
||||
info.folders.forEach((f, i) => {
|
||||
const n = info.streamsPerFolder[i]
|
||||
if (n === 0) return
|
||||
let sum = 0
|
||||
for (let j = 0; j < n - 1 && hasSizes; j++) {
|
||||
const size = r.number()
|
||||
info.streamSizes.push(size)
|
||||
sum += size
|
||||
}
|
||||
info.streamSizes.push(unpackSize(f) - sum)
|
||||
})
|
||||
if (hasSizes) id = r.byte()
|
||||
while (id !== ID.end) {
|
||||
if (id === ID.crc) {
|
||||
// CRCs for the streams whose folder had none (a folder of one stream with a CRC has its own).
|
||||
const unknown = info.folders.reduce((n, f, i) => n + (info.streamsPerFolder[i] === 1 && f.crc !== undefined ? 0 : info.streamsPerFolder[i]), 0)
|
||||
r.digests(unknown)
|
||||
} else {
|
||||
throw new Malformed('7z: unexpected property in substreams info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
id = r.byte()
|
||||
} else {
|
||||
info.streamSizes = info.folders.map(unpackSize)
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: streams info not ended')
|
||||
return info
|
||||
}
|
||||
|
||||
function readFilesInfo(r: Reader, streams: StreamsInfo | null): { name: string; size: number }[] {
|
||||
const n = count(r.number())
|
||||
let emptyStream: boolean[] = Array(n).fill(false)
|
||||
let emptyFile: boolean[] = []
|
||||
let names: string[] = []
|
||||
for (let id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
const size = r.number()
|
||||
const end = r.pos + size
|
||||
if (id === ID.emptyStream) {
|
||||
emptyStream = r.bits(n)
|
||||
} else if (id === ID.emptyFile) {
|
||||
emptyFile = r.bits(emptyStream.filter(Boolean).length)
|
||||
} else if (id === ID.name) {
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external names')
|
||||
const text = Buffer.from(r.bytes(size - 1)).toString('utf16le')
|
||||
names = text.split('\0').slice(0, n)
|
||||
} else if (id !== ID.dummy && id > 0x19) {
|
||||
throw new Malformed('7z: unknown file property')
|
||||
}
|
||||
r.pos = end
|
||||
}
|
||||
if (names.length !== n) throw new Malformed('7z: names missing')
|
||||
const out: { name: string; size: number }[] = []
|
||||
let stream = 0
|
||||
let empty = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (!emptyStream[i]) {
|
||||
out.push({ name: names[i], size: streams?.streamSizes[stream++] ?? 0 })
|
||||
continue
|
||||
}
|
||||
// An empty stream is a directory unless it's marked an empty file.
|
||||
if (emptyFile[empty++]) out.push({ name: names[i], size: 0 })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function readHeader(r: Reader): SevenZipListing {
|
||||
let id = r.byte()
|
||||
if (id === ID.archiveProperties) {
|
||||
for (let type = r.byte(); type !== 0; type = r.byte()) r.bytes(r.number())
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.additionalStreamsInfo) {
|
||||
readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let streams: StreamsInfo | null = null
|
||||
if (id === ID.mainStreamsInfo) {
|
||||
streams = readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let files: { name: string; size: number }[] = []
|
||||
if (id === ID.filesInfo) {
|
||||
files = readFilesInfo(r, streams)
|
||||
id = r.byte()
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: header not ended')
|
||||
const encrypted = (streams?.folders ?? []).some((f) => f.coders.some((c) => c.id === AES))
|
||||
return { files, encrypted, headerEncrypted: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* The header that ends exactly at `end` in `data`, starting at `start`: a plain one, or an encoded one whose packed
|
||||
* data lies just before it (where 7z writes it). Throws when it isn't one.
|
||||
*/
|
||||
function headerAt(data: Uint8Array, start: number, end: number): SevenZipListing {
|
||||
const r = new Reader(data.subarray(0, end), start)
|
||||
const id = r.byte()
|
||||
if (id === ID.header) {
|
||||
const listing = readHeader(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the header')
|
||||
return listing
|
||||
}
|
||||
if (id !== ID.encodedHeader) throw new Malformed('7z: not a header')
|
||||
const info = readStreamsInfo(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the encoded header')
|
||||
if (info.folders.length !== 1 || info.packSizes.length < 1) throw new Malformed('7z: unexpected encoded header')
|
||||
const folder = info.folders[0]
|
||||
if (folder.coders.some((c) => c.id === AES)) return { files: [], encrypted: true, headerEncrypted: true }
|
||||
if (folder.coders.length !== 1 || folder.coders[0].id !== LZMA) throw new Malformed(`7z: header coder ${folder.coders[0].id} not supported`)
|
||||
const packSize = info.packSizes[0]
|
||||
const size = unpackSize(folder)
|
||||
if (packSize > start || size > MAX_HEADER) throw new Malformed('7z: packed header not in the data')
|
||||
const header = lzmaDecode(folder.coders[0].properties, data.subarray(start - packSize, start), size)
|
||||
if (folder.crc !== undefined && crc32(header) >>> 0 !== folder.crc) throw new Malformed('7z: header CRC mismatch')
|
||||
return headerAt(header, 0, header.length)
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing of the 7z archive that `data` is the end of (the last segment of its last volume, or a whole small
|
||||
* archive); null when no header ends there.
|
||||
*/
|
||||
export function parseSevenZipTail(data: Uint8Array): SevenZipListing | null {
|
||||
const end = data.length
|
||||
// An encoded header is a few dozen bytes; a plain one (7z -mhc=off) can be large.
|
||||
for (let start = end - 2; start >= Math.max(0, end - 1024); start--) {
|
||||
if (data[start] !== ID.encodedHeader || data[start + 1] !== ID.packInfo) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
for (let start = end - 2; start >= 0; start--) {
|
||||
if (data[start] !== ID.header || ![ID.archiveProperties, ID.additionalStreamsInfo, ID.mainStreamsInfo, ID.filesInfo, ID.end].includes(data[start + 1])) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { parseZipStart, type ZipStart } from '#lib/zip/parse'
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* The entries a ZIP's start shows: its local file headers, one before each entry's data, as far as the data
|
||||
* holds them (the first segment of a .zip; REBUILD.md, phase 2). The full list is in the central directory at
|
||||
* the end; the start names the first entries and says whether they're encrypted, as a RAR's first volume does.
|
||||
*/
|
||||
export interface ZipStart {
|
||||
files: { name: string; size: number }[]
|
||||
encrypted: boolean
|
||||
}
|
||||
|
||||
const LOCAL = 0x04034b50
|
||||
/** Traditional PKWARE encryption (bit 0 of the flags), or WinZip AES (method 99). */
|
||||
const ENCRYPTED_FLAG = 0x1
|
||||
const DATA_DESCRIPTOR_FLAG = 0x8
|
||||
const AES_METHOD = 99
|
||||
const MAX_ENTRIES = 10_000
|
||||
|
||||
export function parseZipStart(data: Buffer): ZipStart | null {
|
||||
if (data.length < 30 || data.readUInt32LE(0) !== LOCAL) return null
|
||||
const result: ZipStart = { files: [], encrypted: false }
|
||||
let pos = 0
|
||||
while (pos + 30 <= data.length && data.readUInt32LE(pos) === LOCAL && result.files.length < MAX_ENTRIES) {
|
||||
const flags = data.readUInt16LE(pos + 6)
|
||||
const method = data.readUInt16LE(pos + 8)
|
||||
let compressed = data.readUInt32LE(pos + 18)
|
||||
let size = data.readUInt32LE(pos + 22)
|
||||
const nameLength = data.readUInt16LE(pos + 26)
|
||||
const extraLength = data.readUInt16LE(pos + 28)
|
||||
const nameEnd = pos + 30 + nameLength
|
||||
if (nameEnd > data.length) break
|
||||
// Bit 11: UTF-8 names; others are mostly CP437, read as latin1.
|
||||
const name = data.toString(flags & 0x800 ? 'utf8' : 'latin1', pos + 30, nameEnd)
|
||||
// Zip64: the real sizes in extra field 0x0001.
|
||||
for (let e = nameEnd; e + 4 <= Math.min(nameEnd + extraLength, data.length); ) {
|
||||
const id = data.readUInt16LE(e)
|
||||
const length = data.readUInt16LE(e + 2)
|
||||
if (id === 0x0001 && e + 4 + length <= data.length && length >= 16) {
|
||||
size = Number(data.readBigUInt64LE(e + 4))
|
||||
compressed = Number(data.readBigUInt64LE(e + 12))
|
||||
}
|
||||
e += 4 + length
|
||||
}
|
||||
if (flags & ENCRYPTED_FLAG || method === AES_METHOD) result.encrypted = true
|
||||
if (!name.endsWith('/')) result.files.push({ name, size })
|
||||
// Sizes after the data (a streamed zip): where the next header starts isn't known.
|
||||
if (flags & DATA_DESCRIPTOR_FLAG) break
|
||||
pos = nameEnd + extraLength + compressed
|
||||
}
|
||||
return result.files.length > 0 || result.encrypted ? result : null
|
||||
}
|
||||
+21
-6
@@ -17,12 +17,21 @@ const SOURCE_LABELS = {
|
||||
subject: 'Subject',
|
||||
post: 'Post id',
|
||||
}
|
||||
/** Where a known password came from (release_passwords.source). */
|
||||
const PASSWORD_SOURCES = {
|
||||
nzb: "from the spot's NZB",
|
||||
'spot-title': "from the spot's title",
|
||||
'spot-text': "from the spot's text",
|
||||
}
|
||||
/** Newznab categories (lib/newznab/categories.ts). */
|
||||
const CATEGORIES = [
|
||||
[1000, 'Console', []],
|
||||
[2000, 'Movies', [[2030, 'SD'], [2040, 'HD'], [2045, 'UHD']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3040, 'Lossless']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD']]],
|
||||
[7000, 'Books', [[7020, 'Ebook']]],
|
||||
[3000, 'Audio', [[3010, 'MP3'], [3030, 'Audiobook'], [3040, 'Lossless']]],
|
||||
[4000, 'PC', [[4050, 'Games']]],
|
||||
[5000, 'TV', [[5030, 'SD'], [5040, 'HD'], [5045, 'UHD'], [5070, 'Anime']]],
|
||||
[6000, 'XXX', [[6040, 'x264'], [6045, 'UHD'], [6060, 'ImgSet'], [6070, 'Other']]],
|
||||
[7000, 'Books', [[7020, 'Ebook'], [7030, 'Comics']]],
|
||||
[8000, 'Other', [[8010, 'Misc']]],
|
||||
]
|
||||
function categoryLabel(id) {
|
||||
@@ -220,9 +229,10 @@ function releaseRow(release) {
|
||||
if (release.complete) badges.append(badge(files, 'good', 'Complete'))
|
||||
else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
|
||||
else badges.append(badge(files))
|
||||
if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
|
||||
if (release.passworded && release.passwordKnown) badges.append(badge('Password known', 'warn', 'Its archive is encrypted; the password is in the NZB'))
|
||||
else if (release.passworded) badges.append(badge('Password', 'bad', 'Its archive is encrypted: a password is needed'))
|
||||
if (!release.settled) badges.append(badge('Collecting', 'accent', 'Recently seen: more files may still arrive'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name)'))
|
||||
if (release.category && release.category !== 8010) badges.append(badge(categoryLabel(release.category), '', 'Category (from the name, the inner name, a spot or the groups)'))
|
||||
if (release.source && release.source !== 'stem' && release.source !== 'subject') {
|
||||
badges.append(badge(SOURCE_LABELS[release.source] ?? release.source, 'accent', 'How this release was recognised'))
|
||||
}
|
||||
@@ -347,7 +357,12 @@ function renderDetail(release) {
|
||||
if (!release.settledAt) {
|
||||
notices.push(el('div', { class: 'notice notice-warn', text: 'Recently seen: more files of this release may still arrive. The NZB has what is indexed so far.' }))
|
||||
}
|
||||
if (release.passworded) {
|
||||
const passwords = release.passwords ?? []
|
||||
if (passwords.length) {
|
||||
notices.push(el('div', { class: 'notice notice-warn' },
|
||||
el('span', { text: `${passwords.length === 1 ? 'Password' : 'Passwords'} (the first is in the NZB): ` }),
|
||||
...passwords.map((p, i) => el('span', {}, i ? ', ' : '', el('code', { text: p.password }), ` (${PASSWORD_SOURCES[p.source] ?? p.source})`))))
|
||||
} else if (release.passworded) {
|
||||
notices.push(el('div', { class: 'notice notice-bad', text: 'Passworded: its archive has encrypted entries or headers. The password is not in the index.' }))
|
||||
}
|
||||
if (!release.readable && !release.innerName) {
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
Vendored
+4017
File diff suppressed because it is too large.
Load diff
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,67 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import { decidePar2Completeness, par2Complete } from '#services/par2_completeness'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||
import type { Overview } from '#lib/nntp/index'
|
||||
|
||||
const GROUP = 'alt.binaries.test'
|
||||
|
||||
/** A file posted without an [i/N] counter: the release gets no file_total. */
|
||||
function file(name: string, parts: number, at: number): Overview[] {
|
||||
return Array.from({ length: parts }, (_, i) => [at + i, {
|
||||
'subject': `"${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
||||
'message-id': `<${name}-${i + 1}@test>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at + i}`,
|
||||
}] as Overview)
|
||||
}
|
||||
|
||||
async function store(start: number, overviews: Overview[]) {
|
||||
const g = (await db.from('groups').where('name', GROUP).first()) ?? (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('*'))[0]
|
||||
const [n] = await db.table('nodes').insert({ name: `n${start}` }).returning('id')
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: start + 999, next_id: start, status: 'leased', node_id: n.id }).returning('id')
|
||||
const indexer = new RangeIndexer({ group: GROUP, start, end: start + 999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||
indexer.add(overviews, start + 1000)
|
||||
indexer.finish()
|
||||
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
||||
}
|
||||
|
||||
const setFile = (name: string, length: number, hash16k = name.padEnd(32, '0').slice(0, 32)) => ({ name, length, hash16k })
|
||||
|
||||
test.group('par2 completeness', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('every described file by name or by hash; a renamed upload by its files and bytes', ({ assert }) => {
|
||||
const set = [setFile('a.mkv', 1000), setFile('a.nfo', 10)]
|
||||
const files = [{ name: 'a.mkv', size: 1100 }, { name: 'a.nfo', size: 12 }, { name: 'a.par2', size: 50 }]
|
||||
assert.isTrue(par2Complete(set, files, []))
|
||||
assert.isFalse(par2Complete(set, files.slice(0, 1), []))
|
||||
// The nfo renamed, but its probe hashed it.
|
||||
assert.isTrue(par2Complete(set, [files[0], { name: 'x.nfo', size: 12 }], [{ hash16k: set[1].hash16k, size: 10 }]))
|
||||
// Renamed after the par2 was made (random names): two data files and enough bytes; one missing, or too small.
|
||||
assert.isTrue(par2Complete(set, [{ name: 'Qx81.bin', size: 1100 }, { name: 'Zr02.bin', size: 12 }, { name: 'Qx.par2', size: 50 }], []))
|
||||
assert.isFalse(par2Complete(set, [{ name: 'Qx81.bin', size: 1100 }, { name: 'Qx.par2', size: 50 }], []))
|
||||
assert.isFalse(par2Complete(set, [{ name: 'Qx81.bin', size: 500 }, { name: 'Zr02.bin', size: 12 }], []))
|
||||
})
|
||||
|
||||
test('decided once settled with a par2 set, and kept when more files come', async ({ assert }) => {
|
||||
await store(1, [...file('Some.Upload.mkv', 3, 10), ...file('Some.Upload.par2', 1, 20)])
|
||||
const release = await db.from('releases').firstOrFail()
|
||||
assert.isNull(release.file_total)
|
||||
await db.table('par2_files').multiInsert([setFile('Some.Upload.mkv', 2_000_000), setFile('Some.Upload.nfo', 100)].map((f, i) => ({ set_id: 's'.repeat(32), file_id: String(i).repeat(32), ...f })))
|
||||
await db.table('par2_sets').insert({ set_id: 's'.repeat(32), release_id: release.id, files: 2 })
|
||||
// Not settled yet: left alone.
|
||||
assert.equal(await decidePar2Completeness(db, [release.id]), 0)
|
||||
await db.from('releases').where('id', release.id).update({ settled_at: new Date() })
|
||||
assert.equal(await decidePar2Completeness(db, [release.id]), 1)
|
||||
assert.isFalse((await db.from('releases').where('id', release.id).firstOrFail()).complete)
|
||||
// The nfo comes in a later range: complete, and the batch store's own update keeps it.
|
||||
await store(1001, file('Some.Upload.nfo', 1, 1010))
|
||||
await decidePar2Completeness(db, [release.id])
|
||||
await store(2001, file('Some.Upload.sfv', 1, 2010))
|
||||
const after = await db.from('releases').where('id', release.id).firstOrFail()
|
||||
assert.equal(after.files, 4)
|
||||
assert.isTrue(after.complete)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
|
||||
const categoryOf = async (name: string) => Number((await db.rawQuery(`SELECT release_category(?) AS c`, [name])).rows[0].c)
|
||||
|
||||
test.group('Release categories', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('by name: the sources and codes the first function lost or missed, XXX, audio, books, games', async ({ assert }) => {
|
||||
const expected: [string, number][] = [
|
||||
// Lost to knex's bindings in migration 1200 (`blu-$1ray`, `web-$2dl`, `h\.$326[45]`).
|
||||
['Some.Movie.2020.BluRay.DTS-X', 2040],
|
||||
['Some.Movie.2020.WEB-DL.DDP5', 2030],
|
||||
['Some.Movie.2020.H.264', 2030],
|
||||
['Some.Movie.2020.2160p.WEB-DL', 2045],
|
||||
// TV codes it missed.
|
||||
['Rizzoli & Isles S01e07[Mux XviD', 5030],
|
||||
['Rush S04E01E02 WS PDTVXviD', 5030],
|
||||
['Show.S01E02.1080p.WEB', 5040],
|
||||
['Show.1x02.HDTV', 5030],
|
||||
['Resolution 1920x1080 wallpapers', 8010],
|
||||
['Studio.24.10.22.Some.Name.XXX.1080p.MP4-GRP', 6040],
|
||||
['Studio.Some.Name.XXX.2160p', 6045],
|
||||
['Some.Name.XXX.IMAGESET-GRP', 6060],
|
||||
['Artist-Album-WEB-2026-GRP', 3010],
|
||||
['Artist - Album (2020) FLAC', 3040],
|
||||
['Author - Title (Audiobook)', 3030],
|
||||
['Book.Title.EPUB', 7020],
|
||||
['Game.Title-CODEX', 4050],
|
||||
['Game.Title.NSW-VENOM', 1000],
|
||||
['TsIgNBewuFbuwWy', 8010],
|
||||
]
|
||||
const got: [string, number][] = []
|
||||
for (const [name] of expected) got.push([name, await categoryOf(name)])
|
||||
assert.deepEqual(got, expected)
|
||||
})
|
||||
|
||||
test("the name's, else the inner name's, else a given one (a spot's), else the groups'", async ({ assert }) => {
|
||||
const row = (key: string, extra: Record<string, unknown>) => ({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), source: 'stem', files: 1, groups: '["alt.binaries.misc"]', ...extra })
|
||||
await db.table('releases').multiInsert([
|
||||
row('a', { name: 'Show.S01E02.1080p.WEB' }),
|
||||
row('b', { name: 'TsIgNBewuFbuwWy' }),
|
||||
row('c', { name: 'TsIgNBewuFbuwWy', groups: '["alt.binaries.pictures.erotica.puffies"]' }),
|
||||
// A spot's release of its own, inserted with the spot's category.
|
||||
row('d', { name: 'Some spot title', category: 2040 }),
|
||||
])
|
||||
const byName = async () => Object.fromEntries((await db.from('releases').select('release_key', 'category')).map((r) => [r.release_key[0], r.category]))
|
||||
assert.deepEqual(await byName(), { a: 5040, b: 8010, c: 6060, d: 2040 })
|
||||
// The prober finds an inner name; a spot links a release (the spot inspector sets a Misc one's category).
|
||||
await db.from('releases').where('release_key', 'b'.padEnd(40, '0')).update({ inner_name: 'Some.Movie.2020.1080p.BluRay.x264-GRP.mkv' })
|
||||
await db.from('releases').where('release_key', 'c'.padEnd(40, '0')).update({ category: 5030 })
|
||||
await db.from('releases').where('release_key', 'c'.padEnd(40, '0')).update({ inner_name: 'nothing.bin' })
|
||||
assert.deepEqual(await byName(), { a: 5040, b: 2040, c: 5030, d: 2040 })
|
||||
})
|
||||
|
||||
test("a spot's category in Newznab's", async ({ assert }) => {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spot_category(0, '{"a":[9],"z":[0]}') AS movie_hd, spot_category(0, '{"a":[15],"z":[1]}') AS series_uhd, spot_category(0, '{"a":[0],"z":[3]}') AS erotica,
|
||||
spot_category(0, '{"a":[5]}') AS epub, spot_category(0, '{"z":[4]}') AS picture, spot_category(1, '{"a":[8]}') AS flac, spot_category(1, '{"a":[0],"z":[3]}') AS audiobook,
|
||||
spot_category(2, '{"a":[0]}') AS pc_game, spot_category(2, '{"a":[12]}') AS console, spot_category(3, '{}') AS app`
|
||||
)
|
||||
assert.deepEqual(rows[0], { movie_hd: 2040, series_uhd: 5045, erotica: 6070, epub: 7020, picture: null, flac: 3040, audiobook: 3030, pc_game: 4050, console: 1000, app: 4000 })
|
||||
})
|
||||
})
|
||||
@@ -113,4 +113,27 @@ test.group('Releases API', (group) => {
|
||||
assert.include(feed, '<title>Hidden.Gem.2023</title>')
|
||||
assert.match(feed, /Other Movie 2024[^]*?name="password" value="1"/)
|
||||
})
|
||||
|
||||
test('a known password: in the NZB (the most trusted source first), the details and the list', async ({ client, assert }) => {
|
||||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||
const [n] = await db.table('nodes').insert({ name: 'api-node' }).returning('id')
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id')
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||
indexer.add(file('Locked.Movie.2024.1080p.BluRay.x264.part1.rar', 'Locked.Movie [1/1]', 2, 100), 1001)
|
||||
indexer.finish()
|
||||
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
||||
const movie = await db.from('releases').firstOrFail()
|
||||
await db.from('releases').where('id', movie.id).update({ passworded: true })
|
||||
await db.table('release_passwords').multiInsert([
|
||||
{ release_id: movie.id, password: 'from-text', source: 'spot-text' },
|
||||
{ release_id: movie.id, password: 'from-nzb', source: 'nzb' },
|
||||
])
|
||||
|
||||
const nzb = parseNzb((await client.get(`/api/v1/releases/${movie.id}/nzb`)).text())!
|
||||
assert.equal(nzb.meta.password, 'from-nzb')
|
||||
const detail = (await client.get(`/api/v1/releases/${movie.id}`)).body() as any
|
||||
assert.deepEqual(detail.passwords, [{ password: 'from-nzb', source: 'nzb' }, { password: 'from-text', source: 'spot-text' }])
|
||||
const list = (await client.get('/api/v1/releases?q=locked movie')).body() as { releases: { passwordKnown: boolean }[] }
|
||||
assert.isTrue(list.releases[0].passwordKnown)
|
||||
})
|
||||
})
|
||||
@@ -102,6 +102,8 @@ test.group('Spots', (group) => {
|
||||
assert.isTrue(await inspector.standalone(SPOT))
|
||||
const release = await db.from('releases').where('source', 'spot').firstOrFail()
|
||||
assert.equal(release.name, 'Lonely File')
|
||||
// Its title says nothing: the spot's category (01, a09: a movie, x264).
|
||||
assert.equal(release.category, 2040)
|
||||
assert.equal(release.files, 1)
|
||||
assert.isNotNull(release.poster)
|
||||
assert.isNotNull(await db.from('posters').where('poster', release.poster).first())
|
||||
|
||||
@@ -55,6 +55,20 @@ test.group('edge / RangeIndexer', () => {
|
||||
assert.equal(batch.next, 1_000_001)
|
||||
})
|
||||
|
||||
test("archive candidates: a split 7z's start and its highest volume's end, an unsplit 7z's start and end, a ZIP's start", ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
|
||||
indexer.add([
|
||||
...post(100, 'Split.Seven.7z.001', 2, { counter: 'Split.Seven [1/3]' }),
|
||||
...post(110, 'Split.Seven.7z.003', 2, { counter: 'Split.Seven [3/3]' }),
|
||||
...post(120, 'Split.Seven.7z.002', 2, { counter: 'Split.Seven [2/3]' }),
|
||||
...post(200, 'Whole.Seven.7z', 3, { counter: 'Whole.Seven [1/1]' }),
|
||||
...post(300, 'Some.Zip.zip', 2, { counter: 'Some.Zip [1/1]' }),
|
||||
], 200_000)
|
||||
indexer.finish()
|
||||
const got = indexer.take().candidates.map((c) => `${c.kind} ${c.file}`).sort()
|
||||
assert.deepEqual(got, ['7z-tail Split.Seven.7z.003', '7z-tail Whole.Seven.7z', 'archive Some.Zip.zip', 'archive Split.Seven.7z.001', 'archive Whole.Seven.7z'])
|
||||
})
|
||||
|
||||
test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
|
||||
indexer.add([
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { withPassword } from '#services/nzb_service'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
|
||||
const NZB = (head: string) =>
|
||||
`<?xml version="1.0" encoding="UTF-8"?><nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${head}<file poster="p" date="1" subject=""a.rar" yEnc (1/1)"><groups><group>a.b.x</group></groups><segments><segment bytes="1" number="1">a@x</segment></segments></file></nzb>`
|
||||
|
||||
test.group('nzb / a read NZB with the password', () => {
|
||||
test('added into its head, or a head made; one it has already is kept', ({ assert }) => {
|
||||
assert.equal(parseNzb(withPassword(NZB('<head><meta type="name">x</meta></head>'), 'p<&>w'))!.meta.password, 'p<&>w')
|
||||
assert.equal(parseNzb(withPassword(NZB(''), 'pw'))!.meta.password, 'pw')
|
||||
assert.equal(parseNzb(withPassword(NZB('<head><meta type="password">own</meta></head>'), 'pw'))!.meta.password, 'own')
|
||||
assert.equal(withPassword(NZB(''), undefined), NZB(''))
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import yencode from 'yencode'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
@@ -148,4 +149,45 @@ test.group('probe / innerName', () => {
|
||||
// Nor does packaging: a lone par2 volume.
|
||||
assert.isNull(innerName([{ kind: 'lone', result: { yencName: 'a1b2.vol007+08.par2', size: 1, type: 'par2' } }], 'x9y8'))
|
||||
})
|
||||
|
||||
test("a 7z-tail: the listing from the last segment of the last volume; one that isn't the last segment: none", ({ assert }) => {
|
||||
const tail = readFileSync(new URL('../../fixtures/sevenzip/split-last-volume.tail', import.meta.url))
|
||||
// The volume's last 8,000 bytes as its third part, of a volume 1,000,000 bytes long.
|
||||
const last = tail.subarray(tail.length - 8000)
|
||||
const part = (data: Buffer, begin: number) =>
|
||||
Buffer.from(
|
||||
[`=ybegin part=3 total=3 line=128 size=1000000 name=x.7z.004`, `=ypart begin=${begin} end=${begin + data.length - 1}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} part=3 pcrc32=${crc32Hex(data)}`].join('\r\n'),
|
||||
'latin1'
|
||||
)
|
||||
const result = inspect(part(last, 1_000_000 - 8000 + 1), '7z-tail')
|
||||
assert.equal(result.entries, 4)
|
||||
assert.deepEqual(result.names?.[0], 'Some.Movie.2020.1080p.BluRay.x264-GRP/some.movie.2020.1080p.bluray.x264-grp.mkv')
|
||||
assert.isFalse(result.passworded)
|
||||
assert.isUndefined(inspect(part(last, 1), '7z-tail').names)
|
||||
const encrypted = readFileSync(new URL('../../fixtures/sevenzip/header-encrypted.tail', import.meta.url))
|
||||
const hidden = inspect(part(encrypted.subarray(encrypted.length - 8000), 1_000_000 - 8000 + 1), '7z-tail')
|
||||
assert.isTrue(hidden.passworded)
|
||||
assert.equal(hidden.entries, 0)
|
||||
assert.isUndefined(hidden.names)
|
||||
// A small 7z whole in one article, probed as an archive start.
|
||||
assert.deepEqual(inspect(article(readFileSync(new URL('../../fixtures/sevenzip/tiny.7z', import.meta.url)), 'x.7z'), 'archive').names, ['b.nfo', 'a.txt'])
|
||||
})
|
||||
|
||||
test('a ZIP start: the names of its first entries, and whether they are encrypted', ({ assert }) => {
|
||||
const local = (name: string, data: Buffer, flags = 0) => {
|
||||
const header = Buffer.alloc(30)
|
||||
header.writeUInt32LE(0x04034b50, 0)
|
||||
header.writeUInt16LE(flags, 6)
|
||||
header.writeUInt32LE(data.length, 18)
|
||||
header.writeUInt32LE(data.length, 22)
|
||||
header.writeUInt16LE(Buffer.byteLength(name), 26)
|
||||
return Buffer.concat([header, Buffer.from(name), data])
|
||||
}
|
||||
const zip = Buffer.concat([local('Album/', Buffer.alloc(0)), local('Album/01 - Track.flac', Buffer.alloc(3000, 1)), local('Album/cover.jpg', Buffer.alloc(500, 2))])
|
||||
const plain = inspect(article(zip, 'album.zip'))
|
||||
assert.equal(plain.type, 'zip')
|
||||
assert.deepEqual(plain.names, ['Album/01 - Track.flac', 'Album/cover.jpg'])
|
||||
assert.isFalse(plain.passworded)
|
||||
assert.isTrue(inspect(article(local('secret.mkv', Buffer.alloc(100), 0x1), 'x.zip')).passworded)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,20 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { lzmaDecode, LzmaError } from '#lib/sevenzip/lzma'
|
||||
|
||||
const fixture = (name: string) => readFileSync(new URL(`../../fixtures/sevenzip/${name}`, import.meta.url))
|
||||
|
||||
test.group('probe / LZMA', () => {
|
||||
test('decodes a stream Python made (.lzma: 5 property bytes, the size, the raw stream)', ({ assert }) => {
|
||||
const alone = fixture('sample.lzma')
|
||||
const expected = fixture('sample.bin')
|
||||
const decoded = lzmaDecode(alone.subarray(0, 5), alone.subarray(13), expected.length)
|
||||
assert.isTrue(decoded.equals(expected))
|
||||
})
|
||||
|
||||
test('a cut-off or damaged stream throws LzmaError', ({ assert }) => {
|
||||
const alone = fixture('sample.lzma')
|
||||
const size = fixture('sample.bin').length
|
||||
assert.throws(() => lzmaDecode(alone.subarray(0, 5), alone.subarray(13, 200), size), LzmaError)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,45 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||
|
||||
const fixture = (name: string) => readFileSync(new URL(`../../fixtures/sevenzip/${name}`, import.meta.url))
|
||||
const DIR = 'Some.Movie.2020.1080p.BluRay.x264-GRP'
|
||||
|
||||
test.group('probe / 7z listing from the archive end', () => {
|
||||
test('an LZMA-packed header (7z default): files with sizes, directories left out', ({ assert }) => {
|
||||
const listing = parseSevenZipTail(fixture('plain.tail'))
|
||||
assert.deepEqual(listing, {
|
||||
files: [
|
||||
{ name: `${DIR}/empty.txt`, size: 0 },
|
||||
{ name: `${DIR}/Sample/sample.mkv`, size: 20000 },
|
||||
{ name: `${DIR}/grp.nfo`, size: 4 },
|
||||
{ name: `${DIR}/some.movie.2020.1080p.bluray.x264-grp.mkv`, size: 300000 },
|
||||
],
|
||||
encrypted: false,
|
||||
headerEncrypted: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('an uncompressed header (7z -mhc=off) and the last volume of a split archive', ({ assert }) => {
|
||||
for (const name of ['uncompressed-header.tail', 'split-last-volume.tail']) {
|
||||
const listing = parseSevenZipTail(fixture(name))!
|
||||
assert.lengthOf(listing.files, 4, name)
|
||||
assert.isFalse(listing.encrypted, name)
|
||||
}
|
||||
})
|
||||
|
||||
test('encrypted entries: names listed, encrypted; an encrypted header (7z -mhe): no names, encrypted', ({ assert }) => {
|
||||
const files = parseSevenZipTail(fixture('files-encrypted.tail'))!
|
||||
assert.lengthOf(files.files, 4)
|
||||
assert.isTrue(files.encrypted)
|
||||
assert.isFalse(files.headerEncrypted)
|
||||
assert.deepEqual(parseSevenZipTail(fixture('header-encrypted.tail')), { files: [], encrypted: true, headerEncrypted: true })
|
||||
})
|
||||
|
||||
test('a whole small archive; not the end of an archive: null', ({ assert }) => {
|
||||
assert.deepEqual(parseSevenZipTail(fixture('tiny.7z'))?.files.map((f) => f.name), ['a.txt', 'b.nfo'])
|
||||
// The end cut off, or data that isn't 7z at all.
|
||||
assert.isNull(parseSevenZipTail(fixture('plain.tail').subarray(0, 30000)))
|
||||
assert.isNull(parseSevenZipTail(fixture('sample.bin')))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user