Rebuild: the fragment merger
Files cut by a range's edge are put together once the ranges around them are done (coordinator:run, every minute): a complete file is judged by the rules and written as a release chunk with probe candidates; one still missing parts while both neighbouring ranges are done is dropped and counted (incomplete/edge). A neighbour not planned yet counts as not done, except before the group's oldest article. BatchStore.writeReleases is the shared write path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2c14315f94
commit
b862201991
7 files changed
+365
-65
No files matched your search
@@ -0,0 +1,35 @@
|
||||
import type { AggregatedFile } from '../files/index.js'
|
||||
import type { Fragment } from './range_indexer.js'
|
||||
import type { EdgeSegment, FileMeta } from './scan.js'
|
||||
|
||||
/**
|
||||
* The fragments of one file from neighbouring ranges, put together
|
||||
* (REBUILD.md, fragments): the complete file when every part from 1 to its
|
||||
* total is there (the first article seen of a part wins), else null.
|
||||
*/
|
||||
export function mergeFragments(fragments: Fragment[]): AggregatedFile<EdgeSegment> | null {
|
||||
if (fragments.length === 0) return null
|
||||
const first = fragments[0]
|
||||
const meta: FileMeta = {
|
||||
fileKey: first.fileKey,
|
||||
name: first.name,
|
||||
poster: first.poster,
|
||||
subject: first.subject,
|
||||
totalParts: first.totalParts,
|
||||
groups: first.groups,
|
||||
release: first.release ? { ...first.release, source: first.release.source as any } : null,
|
||||
}
|
||||
const parts = new Map<number, EdgeSegment>()
|
||||
for (const fragment of fragments) {
|
||||
const postedAt = fragment.postedAt ?? Number.NaN
|
||||
for (const [number, messageId, bytes, articleNumber] of fragment.segments) {
|
||||
const known = parts.get(number)
|
||||
if (!known || articleNumber < known.articleNumber) {
|
||||
parts.set(number, { fileKey: meta.fileKey, number, totalParts: meta.totalParts, messageId, bytes, articleNumber, postedAt, file: meta })
|
||||
}
|
||||
}
|
||||
}
|
||||
if (parts.size < meta.totalParts) return null
|
||||
for (let n = 1; n <= meta.totalParts; n++) if (!parts.has(n)) return null
|
||||
return { fileKey: meta.fileKey, segments: [...parts.values()].sort((a, b) => a.number - b.number) }
|
||||
}
|
||||
Reference in new issue
Block a user