diff --git a/REBUILD.md b/REBUILD.md index 570fc8a..cd7796d 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -260,7 +260,7 @@ with 68,032 releases with a password): | Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported | | SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) | | Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only | -| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed | +| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed; one release keeps only the tails of its highest volume (the live tail's pieces each queued one; `seven_zip_tails`) | | Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again | | Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet | diff --git a/app/services/batch_store.ts b/app/services/batch_store.ts index f4ebd8a..cb490f0 100644 --- a/app/services/batch_store.ts +++ b/app/services/batch_store.ts @@ -1,4 +1,5 @@ import db from '@adonisjs/lucid/services/db' +import { pruneSevenZipTails } from '#services/seven_zip_tails' import type { TransactionClientContract } from '@adonisjs/lucid/types/database' import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index' import ChunkPartitions from '#services/chunk_partitions' @@ -324,6 +325,9 @@ class BatchStore { .ignore() .returning('id') out.candidates += added.length + // A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts). + const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id)) + if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails]) // Files added to a settled release (a later range of its upload): the settler made its candidates claimable // already and won't again (93,642 waited since 2026-10-13; LOG.md 2026-10-20 (87)). if (added.length > 0) { diff --git a/app/services/par2_merger.ts b/app/services/par2_merger.ts index b1bfa83..097a701 100644 --- a/app/services/par2_merger.ts +++ b/app/services/par2_merger.ts @@ -4,6 +4,7 @@ import { par2ReleaseName } from '#lib/release/index' import { looksRandom } from '#lib/release/random' import { SidecarFolder } from '#services/sidecar_folder' import { Par2Pruner } from '#services/par2_pruner' +import { pruneSevenZipTails } from '#services/seven_zip_tails' /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */ const MAX_SPREAD_MS = 48 * 3600 * 1000 @@ -341,6 +342,8 @@ export class Par2Merger { WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`, [target.id] ) + // The members' 7z tails together: those below the highest volume go. + await pruneSevenZipTails(trx, [target.id]) // A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every // round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)). await trx.rawQuery( diff --git a/app/services/seven_zip_tails.ts b/app/services/seven_zip_tails.ts new file mode 100644 index 0000000..504f514 --- /dev/null +++ b/app/services/seven_zip_tails.ts @@ -0,0 +1,22 @@ +/** + * 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last + * segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and + * its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while + * it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for + * .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases' + * waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them: + * a piece that saw only part of it queued a segment that isn't its last. + */ +export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise { + if (releaseIds.length === 0) return 0 + const result = await trx.rawQuery( + `DELETE FROM probe_candidates pc USING ( + SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest + FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume + FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t + ) s + WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`, + [releaseIds.map(String)] + ) + return result.rowCount ?? 0 +} diff --git a/tests/functional/batch_store.spec.ts b/tests/functional/batch_store.spec.ts index 3b7bf46..10fe71c 100644 --- a/tests/functional/batch_store.spec.ts +++ b/tests/functional/batch_store.spec.ts @@ -132,6 +132,28 @@ test.group('BatchStore', (group) => { assert.lengthOf(await releasesOf(), 1) }) + test('a split 7z read in pieces keeps the tail of its highest volume only; RAR gets none', async ({ assert }) => { + // As the live tail reads it while posted: each piece queues a tail for the last volume it saw so far. + await storeIn(GROUP, 1, [ + ...upload(10, 'TsIgNBewuFbuwWy.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'), + ...upload(20, 'TsIgNBewuFbuwWy.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'), + ]) + await storeIn(GROUP, 100_001, upload(100_010, 'TsIgNBewuFbuwWy.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z')) + // A lower volume posted late: its tail goes at once. + await storeIn(GROUP, 200_001, upload(200_010, 'TsIgNBewuFbuwWy.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z')) + await storeIn(GROUP, 300_001, upload(300_010, 'Some.Show.S01E03.part01.rar', 2, 'Some.Show.S01E03 [1/2]', '2026-10-23T09:00:00Z')) + await storeIn(GROUP, 400_001, upload(400_010, 'Some.Show.S01E03.part02.rar', 2, 'Some.Show.S01E03 [2/2]', '2026-10-23T09:02:00Z')) + const candidates = await db.from('probe_candidates').orderBy('file').select('kind', 'file') + assert.deepEqual( + candidates.map((c) => [c.kind, c.file]), + [ + ['archive', 'Some.Show.S01E03.part01.rar'], + ['archive', 'TsIgNBewuFbuwWy.7z.001'], + ['7z-tail', 'TsIgNBewuFbuwWy.7z.004'], + ] + ) + }) + test('the same upload in another group (multi-posted) is a release of its own', async ({ assert }) => { await storeIn(GROUP, 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:00:00Z')) await storeIn('alt.binaries.other', 1, upload(10, 'Some.Show.S01E01.1080p.mkv', 2, 'Some.Show.S01E01.1080p [1/1]', '2026-10-05T10:01:00Z', 'alt.binaries.other')) diff --git a/tests/functional/seven_zip_tails.spec.ts b/tests/functional/seven_zip_tails.spec.ts new file mode 100644 index 0000000..f0944a9 --- /dev/null +++ b/tests/functional/seven_zip_tails.spec.ts @@ -0,0 +1,30 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import { pruneSevenZipTails } from '#services/seven_zip_tails' + +test.group('7z tails', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test("of a release's waiting tails, those below its highest volume go; the highest volume's all stay", async ({ assert }) => { + const row = (key: string) => ({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', files: 4 }) + const [merged, other] = (await db.table('releases').multiInsert([row('a'), row('b')]).returning('id')).map((r) => r.id) + const tail = (releaseId: number, volume: string, mid: string, status = 'waiting') => ({ release_id: releaseId, kind: '7z-tail', message_id: mid, file: `x.7z.${volume}`, status }) + await db.table('probe_candidates').multiInsert([ + // Two merged members' tails, one of the highest volume probed already. + tail(merged, '002', '<1@x>'), + tail(merged, '005', '<2@x>'), + tail(merged, '003', '<3@x>', 'done'), + tail(merged, '009', '<4@x>', 'taken'), + // Two pieces that each saw part of volume 009. + tail(merged, '009', '<5@x>'), + { release_id: merged, kind: 'archive', message_id: '<6@x>', file: 'x.7z.001' }, + // Another release: not asked for. + tail(other, '002', '<7@x>'), + tail(other, '004', '<8@x>'), + ]) + assert.equal(await pruneSevenZipTails(db, [merged]), 2) + const left = await db.from('probe_candidates').orderBy('message_id').select('message_id') + assert.deepEqual(left.map((c) => c.message_id), ['<3@x>', '<4@x>', '<5@x>', '<6@x>', '<7@x>', '<8@x>']) + }) +})