The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog
pending_ladder.ts claims the oldest waiting pending releases' next target (par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each claim; a usable name (the largest named file of a par2 set or a listing, or a lone file's yEnc name, neither packaging nor random) hands the release to the namer, no name the next target or the end of the climb; climbs count on pending_signatures per (tool, group, poster) and (tool, group), and a signature with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are discarded without a probe, the rules' sample excepted. pending_namer.ts reads a named release's articles again from the corpus or by XOVER and writes it to the catalog under the pending key (source weak-stem, the name as inner name) with its chunk and candidates. Migration 6400; config PENDING_*. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
9cf85e09d8
commit
e135076fcd
14 files changed
+1045
-50
No files matched your search
@@ -0,0 +1,208 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import PendingLadder, { PENDING, byEffort, type PendingProbe } from '#services/pending_ladder'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
|
||||
let groupId = 0
|
||||
let n = 0
|
||||
|
||||
async function pending(
|
||||
name: string,
|
||||
candidates: { kind: string; messageId: string; file: string }[],
|
||||
extra: Record<string, unknown> = {}
|
||||
) {
|
||||
n++
|
||||
const key = releaseKey(name, groupId, n)
|
||||
await db.table('pending_releases').insert({
|
||||
release_key: key,
|
||||
identity_key: identityKey(name),
|
||||
group_id: groupId,
|
||||
name,
|
||||
poster: 'Bob <bob@home.me>',
|
||||
first_article: n,
|
||||
last_article: n,
|
||||
files: 1,
|
||||
segments: 10,
|
||||
bytes: 1000,
|
||||
tool: 'other',
|
||||
candidates: JSON.stringify(candidates),
|
||||
...extra,
|
||||
})
|
||||
return key.toString('hex')
|
||||
}
|
||||
|
||||
const inspection = (yencName: string, names?: string[]) => ({
|
||||
yencName,
|
||||
size: 1000,
|
||||
type: null,
|
||||
names,
|
||||
entries: names?.length,
|
||||
})
|
||||
|
||||
test.group('Pending ladder', (group) => {
|
||||
group.each.setup(async () => {
|
||||
const rollback = await testUtils.db().withGlobalTransaction()
|
||||
groupId = Number(
|
||||
(
|
||||
await db
|
||||
.table('groups')
|
||||
.insert({ name: 'alt.binaries.pending', priority: 1 })
|
||||
.returning('id')
|
||||
)[0].id
|
||||
)
|
||||
n = 0
|
||||
return rollback
|
||||
})
|
||||
|
||||
test('targets are climbed by effort: a par2 before an archive before a lone file', ({
|
||||
assert,
|
||||
}) => {
|
||||
assert.deepEqual(
|
||||
byEffort([
|
||||
{ kind: 'lone', messageId: 'l', file: 'l' },
|
||||
{ kind: '7z-tail', messageId: 't', file: 't' },
|
||||
{ kind: 'archive', messageId: 'a', file: 'a' },
|
||||
{ kind: 'par2', messageId: 'p', file: 'p' },
|
||||
]).map((t) => t.kind),
|
||||
['par2', 'archive', 'lone', '7z-tail']
|
||||
)
|
||||
})
|
||||
|
||||
test('a claim takes the oldest waiting releases, their cheapest target first; outcomes move them up or to the end', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const twoTargets = await pending('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', [
|
||||
{ kind: 'archive', messageId: '<a@x>', file: 'x.rar' },
|
||||
{ kind: 'par2', messageId: '<p@x>', file: 'x.par2' },
|
||||
])
|
||||
const lone = await pending('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3', [
|
||||
{ kind: 'lone', messageId: '<l@x>', file: 'f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3' },
|
||||
])
|
||||
const none = await pending('00000000000000000000000000000001', [])
|
||||
const provisional = await pending(
|
||||
'00000000000000000000000000000002',
|
||||
[{ kind: 'lone', messageId: '<v@x>', file: 'v' }],
|
||||
{ finished: false }
|
||||
)
|
||||
|
||||
const probes = await PendingLadder.claim(10)
|
||||
assert.deepEqual(
|
||||
probes.map((p) => [p.releaseKey, p.kind, p.tries, p.targets]),
|
||||
[
|
||||
[twoTargets, 'par2', 0, 2],
|
||||
[lone, 'lone', 0, 1],
|
||||
]
|
||||
)
|
||||
const statusOf = async (key: string) =>
|
||||
(await db
|
||||
.from('pending_releases')
|
||||
.where('release_key', Buffer.from(key, 'hex'))
|
||||
.firstOrFail()) as any
|
||||
assert.equal((await statusOf(twoTargets)).status, PENDING.taken)
|
||||
assert.equal((await statusOf(none)).status, PENDING.waiting)
|
||||
assert.equal((await statusOf(provisional)).status, PENDING.waiting)
|
||||
|
||||
// The par2 names only random volumes: the next target. The lone file's yEnc name is its random name: exhausted.
|
||||
await PendingLadder.record([
|
||||
{
|
||||
probe: probes[0],
|
||||
status: 'done',
|
||||
inspection: inspection('x.par2', ['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4.part01.rar']),
|
||||
},
|
||||
{
|
||||
probe: probes[1],
|
||||
status: 'done',
|
||||
inspection: inspection('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3'),
|
||||
},
|
||||
])
|
||||
const first = await statusOf(twoTargets)
|
||||
assert.include(first, { status: PENDING.waiting, tries: 1, named: null })
|
||||
assert.lengthOf(first.inspections, 1)
|
||||
assert.include(await statusOf(lone), { status: PENDING.exhausted, tries: 1 })
|
||||
|
||||
// The archive's listing names it: for the namer.
|
||||
const [second] = await PendingLadder.claim(10)
|
||||
assert.deepEqual([second.releaseKey, second.kind, second.tries], [twoTargets, 'archive', 1])
|
||||
await PendingLadder.record([
|
||||
{
|
||||
probe: second,
|
||||
status: 'done',
|
||||
inspection: inspection('x.rar', ['Some.Movie.2016.1080p.mkv']),
|
||||
},
|
||||
])
|
||||
assert.include(await statusOf(twoTargets), {
|
||||
status: PENDING.naming,
|
||||
named: 'Some.Movie.2016.1080p.mkv',
|
||||
tries: 0,
|
||||
})
|
||||
|
||||
// The signature counted both ways: the poster's own and the tool's in the group.
|
||||
const signatures = await db.from('pending_signatures').orderBy('poster')
|
||||
assert.deepEqual(
|
||||
signatures.map((s) => [s.tool, s.poster, s.probed, s.named, s.dropped]),
|
||||
[
|
||||
['other', '', 2, 1, false],
|
||||
['other', 'Bob <bob@home.me>', 2, 1, false],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('a signature that names nothing is dropped, and its waiting releases discarded without a probe', async ({
|
||||
assert,
|
||||
}) => {
|
||||
// 100 climbs of the tool in the group, none named.
|
||||
await db
|
||||
.table('pending_signatures')
|
||||
.insert({ tool: 'other', group_id: groupId, poster: '', probed: 99, named: 0 })
|
||||
const last = await pending('00000000000000000000000000000009', [
|
||||
{ kind: 'lone', messageId: '<z@x>', file: 'z' },
|
||||
])
|
||||
const [probe] = await PendingLadder.claim(1)
|
||||
assert.equal(probe.releaseKey, last)
|
||||
await PendingLadder.record([{ probe, status: 'missing' }])
|
||||
const signature = await db
|
||||
.from('pending_signatures')
|
||||
.where({ tool: 'other', group_id: groupId, poster: '' })
|
||||
.firstOrFail()
|
||||
assert.include(signature, { probed: 100, named: 0, dropped: true })
|
||||
|
||||
const waiting = await pending('00000000000000000000000000000010', [
|
||||
{ kind: 'lone', messageId: '<w@x>', file: 'w' },
|
||||
])
|
||||
const other = await pending(
|
||||
'00000000000000000000000000000011',
|
||||
[{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
|
||||
{ tool: 'nyuu' }
|
||||
)
|
||||
// samplePercent 0 in the test rules: no explored sample.
|
||||
await db.table('rule_sets').insert({
|
||||
version: 99,
|
||||
rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
|
||||
note: 'test',
|
||||
})
|
||||
assert.equal(await PendingLadder.discard(), 1)
|
||||
const statusOf = async (key: string) =>
|
||||
(
|
||||
await db
|
||||
.from('pending_releases')
|
||||
.where('release_key', Buffer.from(key, 'hex'))
|
||||
.firstOrFail()
|
||||
).status
|
||||
assert.equal(await statusOf(waiting), PENDING.discarded)
|
||||
assert.equal(await statusOf(other), PENDING.waiting)
|
||||
})
|
||||
|
||||
test('a probe given back goes to the queue again at the same step', async ({ assert }) => {
|
||||
const key = await pending('00000000000000000000000000000020', [
|
||||
{ kind: 'lone', messageId: '<g@x>', file: 'g' },
|
||||
])
|
||||
const [probe] = await PendingLadder.claim(1)
|
||||
await PendingLadder.record([{ probe: probe as PendingProbe, status: 'waiting' }])
|
||||
assert.include(
|
||||
await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
|
||||
{ status: PENDING.waiting, tries: 0 }
|
||||
)
|
||||
assert.lengthOf(await db.from('pending_signatures'), 0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,143 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { PendingNamer } from '#services/pending_namer'
|
||||
import { PENDING } from '#services/pending_ladder'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
|
||||
const GROUP = 'alt.binaries.namer'
|
||||
|
||||
/** One overview line as XOVER gives it (the corpus stores the reply's text). */
|
||||
function line(n: number, subject: string, minutes: number) {
|
||||
const date = new Date(Date.UTC(2016, 0, 4, 10, minutes)).toUTCString()
|
||||
return `${n}\t${subject}\tp <p@x>\t${date}\t<u${n}@test>\t\t1000\t10\tXref: news.example ${GROUP}:${n}`
|
||||
}
|
||||
|
||||
test.group('Pending namer', (group) => {
|
||||
let dir = ''
|
||||
group.each.setup(async () => {
|
||||
const rollback = await testUtils.db().withGlobalTransaction()
|
||||
dir = await mkdtemp(join(tmpdir(), 'namer-'))
|
||||
return rollback
|
||||
})
|
||||
group.each.teardown(() => rm(dir, { recursive: true, force: true }))
|
||||
|
||||
test('a named pending release is read again from the corpus and written to the catalog with its chunk and candidates', async ({
|
||||
assert,
|
||||
}) => {
|
||||
const groupId = Number(
|
||||
(await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id'))[0].id
|
||||
)
|
||||
// The range stored in the corpus: one chunk with a random-named upload (4 parts and its par2) among others.
|
||||
const lines = [
|
||||
line(5, 'Some.Show [1/1] - "Some.Show.mkv" yEnc (1/1)', 0),
|
||||
...[1, 2, 3, 4].map((i) =>
|
||||
line(9 + i, `[1/2] - "q3B9OrpKiCxUnzgtUGnp.part01.rar" yEnc (${i}/4)`, i)
|
||||
),
|
||||
line(14, '[2/2] - "q3B9OrpKiCxUnzgtUGnp.par2" yEnc (1/1)', 5),
|
||||
line(20, 'Other.Thing [1/1] - "Other.Thing.mkv" yEnc (1/1)', 6),
|
||||
]
|
||||
await mkdir(join(dir, GROUP, '1-200000'), { recursive: true })
|
||||
await writeFile(
|
||||
join(dir, GROUP, '1-200000', '0000-1-100000.gz'),
|
||||
deflateSync(Buffer.from(lines.join('\r\n') + '\r\n', 'latin1'))
|
||||
)
|
||||
await db.table('ranges').insert({
|
||||
group_id: groupId,
|
||||
start_id: 1,
|
||||
end_id: 200_000,
|
||||
next_id: 200_001,
|
||||
status: 'done',
|
||||
stored_at: new Date(),
|
||||
capture: true,
|
||||
})
|
||||
|
||||
// The key the edge gave it: the same identity rules, run here over the same lines.
|
||||
const ix = new RangeIndexer({
|
||||
group: GROUP,
|
||||
start: 1,
|
||||
end: 200_000,
|
||||
rules: { ...DEFAULT_RULES, droppedSignatures: [], priorities: { [GROUP]: 1 } },
|
||||
})
|
||||
ix.addText(lines.join('\r\n'), 200_001)
|
||||
ix.finish()
|
||||
const edge = ix.take().releases.find((r) => !r.readable)!
|
||||
assert.equal(edge.firstArticle, 10)
|
||||
const key = releaseKey(edge.key, groupId, 10)
|
||||
await db.table('pending_releases').insert({
|
||||
release_key: key,
|
||||
identity_key: identityKey(edge.key),
|
||||
group_id: groupId,
|
||||
name: 'q3B9OrpKiCxUnzgtUGnp',
|
||||
poster: 'p <p@x>',
|
||||
first_article: 10,
|
||||
last_article: 14,
|
||||
files: 2,
|
||||
segments: 5,
|
||||
bytes: 5000,
|
||||
file_total: 2,
|
||||
tool: 'other',
|
||||
candidates: JSON.stringify([
|
||||
{ kind: 'archive', messageId: 'u10@test', file: 'q3B9OrpKiCxUnzgtUGnp.part01.rar' },
|
||||
]),
|
||||
status: PENDING.naming,
|
||||
named: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
|
||||
})
|
||||
|
||||
const namer = new PendingNamer(dir)
|
||||
const [row] = await namer.take(5)
|
||||
assert.equal(row.named, 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv')
|
||||
await namer.move(row)
|
||||
|
||||
const release = await db.from('releases').where('release_key', key).firstOrFail()
|
||||
assert.include(release, {
|
||||
name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
|
||||
inner_name: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv',
|
||||
source: 7,
|
||||
readable: true,
|
||||
files: 2,
|
||||
segments: '5',
|
||||
file_total: 2,
|
||||
complete: true,
|
||||
first_article: '10',
|
||||
last_article: '14',
|
||||
})
|
||||
assert.isNotNull(release.settled_at)
|
||||
const chunk = await db.from('release_chunks').where('release_id', release.id).firstOrFail()
|
||||
assert.deepEqual(
|
||||
decodeChunk(chunk.payload)
|
||||
.map((f) => [f.name, f.segments.length])
|
||||
.sort(),
|
||||
[
|
||||
['q3B9OrpKiCxUnzgtUGnp.par2', 1],
|
||||
['q3B9OrpKiCxUnzgtUGnp.part01.rar', 4],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(
|
||||
(await db.from('probe_candidates').where('release_id', release.id).orderBy('kind')).map(
|
||||
(c) => [c.kind, c.ready, c.unreadable]
|
||||
),
|
||||
[
|
||||
['archive', true, false],
|
||||
['par2', true, false],
|
||||
]
|
||||
)
|
||||
assert.deepEqual(
|
||||
await db.from('release_texts').where('release_id', release.id).select('kind', 'text'),
|
||||
[{ kind: 1, text: 'Some.Movie.2016.1080p.BluRay.x264-GRP.mkv' }]
|
||||
)
|
||||
assert.equal(
|
||||
(await db.from('pending_releases').where('release_key', key).firstOrFail()).status,
|
||||
PENDING.named
|
||||
)
|
||||
// Nothing else of the chunk got in.
|
||||
assert.equal(Number((await db.from('releases').count('* as n').first())!.n), 1)
|
||||
})
|
||||
})
|
||||
@@ -4,7 +4,7 @@ import { readFileSync } from 'node:fs'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import yencode from 'yencode'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
import { inspect, innerName, MAX_NAMES } from '#lib/probe/index'
|
||||
import { inspect, innerName, MAX_NAMES, usableName } from '#lib/probe/index'
|
||||
|
||||
/** A single-part yEnc article of `data`. */
|
||||
function article(data: Buffer, name: string) {
|
||||
@@ -76,6 +76,19 @@ function rar5File(name: string, size: number, opts: { packed: number; headerFlag
|
||||
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
||||
}
|
||||
|
||||
test.group('probe / usable name (the pending ladder)', () => {
|
||||
test('a par2 or listing that names a real file names the release; random volumes and packaging do not', ({ assert }) => {
|
||||
const base = { size: 1000, type: null }
|
||||
assert.equal(usableName({ ...base, yencName: 'x.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar', 'Some.Movie.2016.mkv'] }, 'par2'), 'Some.Movie.2016.mkv')
|
||||
assert.isNull(usableName({ ...base, yencName: 'q3B9OrpKiCxUnzgtUGnp.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar'] }, 'par2'))
|
||||
assert.equal(usableName({ ...base, yencName: 'x.rar', names: ['Folder/Real.Name.S01E01.mkv'] }, 'archive'), 'Real.Name.S01E01.mkv')
|
||||
assert.isNull(usableName({ ...base, yencName: 'x.rar', names: [], passworded: true }, 'archive'))
|
||||
assert.equal(usableName({ ...base, yencName: 'brothalovers.20.02.16.hazel.moore.mp4' }, 'lone'), 'brothalovers.20.02.16.hazel.moore.mp4')
|
||||
assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63' }, 'lone'))
|
||||
assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63.part01.rar' }, 'lone'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('probe / inspect', () => {
|
||||
test('a par2 index: its files, largest first', ({ assert }) => {
|
||||
const setId = Buffer.alloc(16, 3)
|
||||
|
||||
Reference in new issue
Block a user