seed:node splits a batch the seed API finds too large
Dense recent a.b.ath ranges produced batches over the 64 MB body limit (413). The node now posts such a batch in halves; only the second moves the range's resume point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f0ff8e200f
commit
7893876435
1 file changed
+24
-1
+24
-1
@@ -301,6 +301,8 @@ export default class SeedNode extends BaseCommand {
|
||||
const started = Date.now()
|
||||
let headers = 0
|
||||
let chain: Promise<void> = Promise.resolve()
|
||||
// Where the range resumes after what was stored: a batch split in two reports it with its first half.
|
||||
let reported = lease.start
|
||||
const posting = new Set<Promise<void>>()
|
||||
// Throws once a post has failed, so the range stops fetching.
|
||||
const failedSoFar = () => Promise.race([chain, Promise.resolve()])
|
||||
@@ -321,7 +323,8 @@ export default class SeedNode extends BaseCommand {
|
||||
if (this.stopping) throw new Stopping()
|
||||
while (posting.size >= 2) await Promise.race(posting)
|
||||
const post = chain.then(async () => {
|
||||
await api.post(`/api/v1/seed/ranges/${lease.id}/batches`, batch)
|
||||
await this.postBatch(api, lease.id, batch, reported)
|
||||
reported = batch.next
|
||||
headers += batch.headers
|
||||
})
|
||||
chain = post
|
||||
@@ -354,6 +357,26 @@ export default class SeedNode extends BaseCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Posts a batch; one the seed API finds too large (413: dense recent ranges) is
|
||||
* posted in two halves, at most three splits deep. Only the second half moves
|
||||
* the range's resume point, so a failure in between loses nothing.
|
||||
*/
|
||||
private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise<void> {
|
||||
try {
|
||||
await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch)
|
||||
} catch (error: any) {
|
||||
const parts = batch.files.length + batch.fragments.length
|
||||
if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error
|
||||
const files = Math.ceil(batch.files.length / 2)
|
||||
const fragments = Math.ceil(batch.fragments.length / 2)
|
||||
const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) }
|
||||
const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) }
|
||||
await this.postBatch(api, rangeId, first, reported, depth + 1)
|
||||
await this.postBatch(api, rangeId, second, reported, depth + 1)
|
||||
}
|
||||
}
|
||||
|
||||
/** Sleeps, but wakes up early when the node is stopping. */
|
||||
private async pause(ms: number) {
|
||||
for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
|
||||
|
||||
Reference in new issue
Block a user