Daan MeijerandClaude Fable 5.1
2580f20a43
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
90 lines
4.5 KiB
TypeScript
90 lines
4.5 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
|
|
|
/** Candidates looked at per round. */
|
|
const BATCH = 2000
|
|
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
|
|
const WINDOW_MINUTES = 10
|
|
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
|
|
export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
|
|
|
/** The words of a name worth comparing: 4+ letters or digits. */
|
|
function words(text: string): Set<string> {
|
|
return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
|
|
}
|
|
|
|
export interface Sibling {
|
|
id: number
|
|
name: string
|
|
files: number
|
|
file_total: number | null
|
|
}
|
|
|
|
/**
|
|
* Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
|
|
* ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
|
|
* under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
|
|
* isn't one release.
|
|
*/
|
|
export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
|
|
const said = words(files.map((f) => f.subject).join(' '))
|
|
const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
|
|
if (named.length === 1) return named[0]
|
|
if (named.length > 1) return null
|
|
const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
|
|
return counted.length === 1 ? counted[0] : null
|
|
}
|
|
|
|
export interface FoldRound {
|
|
looked: number
|
|
sidecars: number
|
|
folded: number
|
|
}
|
|
|
|
/**
|
|
* The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
|
|
* counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
|
|
* settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
|
|
* the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
|
|
* 'sidecar'). The rest leave the queue untouched.
|
|
*/
|
|
export class SidecarFolder {
|
|
constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
|
|
|
|
async round(): Promise<FoldRound> {
|
|
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
|
|
// A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
|
|
// queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
|
|
const { rows: queued } = await db.rawQuery(
|
|
`SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
|
|
FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
|
[BATCH]
|
|
)
|
|
const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
|
|
for (const c of queued) {
|
|
if (c.id === null) continue
|
|
round.looked++
|
|
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
|
|
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
|
|
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
|
|
round.sidecars++
|
|
if (!c.posted_at) continue
|
|
const { rows: siblings } = await db.rawQuery(
|
|
`SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
|
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
|
|
AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
|
|
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
|
|
)
|
|
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
|
|
if (!sibling) continue
|
|
const full = siblings.find((s: any) => Number(s.id) === sibling.id)
|
|
const merged = await db.transaction((trx) =>
|
|
this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
|
|
)
|
|
if (merged > 0) round.folded++
|
|
}
|
|
if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
|
|
return round
|
|
}
|
|
}
|