seed:node splits a batch the seed API finds too large
Dense recent a.b.ath ranges produced batches over the 64 MB body limit (413). The node now posts such a batch in halves; only the second moves the range's resume point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f0ff8e200f
commit
7893876435
1 file changed
+24
-1
+24
-1
@@ -301,6 +301,8 @@ export default class SeedNode extends BaseCommand {
|
|||||||
const started = Date.now()
|
const started = Date.now()
|
||||||
let headers = 0
|
let headers = 0
|
||||||
let chain: Promise<void> = Promise.resolve()
|
let chain: Promise<void> = Promise.resolve()
|
||||||
|
// Where the range resumes after what was stored: a batch split in two reports it with its first half.
|
||||||
|
let reported = lease.start
|
||||||
const posting = new Set<Promise<void>>()
|
const posting = new Set<Promise<void>>()
|
||||||
// Throws once a post has failed, so the range stops fetching.
|
// Throws once a post has failed, so the range stops fetching.
|
||||||
const failedSoFar = () => Promise.race([chain, Promise.resolve()])
|
const failedSoFar = () => Promise.race([chain, Promise.resolve()])
|
||||||
@@ -321,7 +323,8 @@ export default class SeedNode extends BaseCommand {
|
|||||||
if (this.stopping) throw new Stopping()
|
if (this.stopping) throw new Stopping()
|
||||||
while (posting.size >= 2) await Promise.race(posting)
|
while (posting.size >= 2) await Promise.race(posting)
|
||||||
const post = chain.then(async () => {
|
const post = chain.then(async () => {
|
||||||
await api.post(`/api/v1/seed/ranges/${lease.id}/batches`, batch)
|
await this.postBatch(api, lease.id, batch, reported)
|
||||||
|
reported = batch.next
|
||||||
headers += batch.headers
|
headers += batch.headers
|
||||||
})
|
})
|
||||||
chain = post
|
chain = post
|
||||||
@@ -354,6 +357,26 @@ export default class SeedNode extends BaseCommand {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Posts a batch; one the seed API finds too large (413: dense recent ranges) is
|
||||||
|
* posted in two halves, at most three splits deep. Only the second half moves
|
||||||
|
* the range's resume point, so a failure in between loses nothing.
|
||||||
|
*/
|
||||||
|
private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise<void> {
|
||||||
|
try {
|
||||||
|
await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch)
|
||||||
|
} catch (error: any) {
|
||||||
|
const parts = batch.files.length + batch.fragments.length
|
||||||
|
if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error
|
||||||
|
const files = Math.ceil(batch.files.length / 2)
|
||||||
|
const fragments = Math.ceil(batch.fragments.length / 2)
|
||||||
|
const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) }
|
||||||
|
const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) }
|
||||||
|
await this.postBatch(api, rangeId, first, reported, depth + 1)
|
||||||
|
await this.postBatch(api, rangeId, second, reported, depth + 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Sleeps, but wakes up early when the node is stopping. */
|
/** Sleeps, but wakes up early when the node is stopping. */
|
||||||
private async pause(ms: number) {
|
private async pause(ms: number) {
|
||||||
for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
|
for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
|
||||||
|
|||||||
Reference in new issue
Block a user