Keep only the 7z tails of a release's highest volume
Test and deploy / test (push) Successful in 48s
Test and deploy / edge (push) Successful in 1m47s
Test and deploy / deploy (push) Successful in 1m49s

The live tail reads an upload in pieces while it is posted, and each
piece queued a 7z-tail candidate for the last volume it had seen so far
(release 306905984: seven, for .002-.008). Only the last volume's end
holds the 7z listing. When candidates are stored, and when releases are
merged, waiting tails below the release's highest volume are deleted.
All tails of the highest volume stay: a piece that saw only part of it
queued a segment that is not its last. RAR is unaffected: it lists its
files in the first volume's header and gets no tail candidate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-23 11:16:25 +02:00
1 parent b60d716db0
commit 3b61215f6b
6 files changed
+82 -1

No files matched your search

+4
View File
@@ -1,4 +1,5 @@
import db from '@adonisjs/lucid/services/db'
import { pruneSevenZipTails } from '#services/seven_zip_tails'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
import ChunkPartitions from '#services/chunk_partitions'
@@ -324,6 +325,9 @@ class BatchStore {
.ignore()
.returning('id')
out.candidates += added.length
// A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
// Files added to a settled release (a later range of its upload): the settler made its candidates claimable
// already and won't again (93,642 waited since 2026-10-13; LOG.md 2026-10-20 (87)).
if (added.length > 0) {
+3
View File
@@ -4,6 +4,7 @@ import { par2ReleaseName } from '#lib/release/index'
import { looksRandom } from '#lib/release/random'
import { SidecarFolder } from '#services/sidecar_folder'
import { Par2Pruner } from '#services/par2_pruner'
import { pruneSevenZipTails } from '#services/seven_zip_tails'
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
const MAX_SPREAD_MS = 48 * 3600 * 1000
@@ -341,6 +342,8 @@ export class Par2Merger {
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
[target.id]
)
// The members' 7z tails together: those below the highest volume go.
await pruneSevenZipTails(trx, [target.id])
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
await trx.rawQuery(
+22
View File
@@ -0,0 +1,22 @@
/**
* 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
* segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
* its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
* it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
* .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
* waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
* a piece that saw only part of it queued a segment that isn't its last.
*/
export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
if (releaseIds.length === 0) return 0
const result = await trx.rawQuery(
`DELETE FROM probe_candidates pc USING (
SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
) s
WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
[releaseIds.map(String)]
)
return result.rowCount ?? 0
}