diff --git a/commands/SeedNode.ts b/commands/SeedNode.ts index 025134b..73aac73 100644 --- a/commands/SeedNode.ts +++ b/commands/SeedNode.ts @@ -301,6 +301,8 @@ export default class SeedNode extends BaseCommand { const started = Date.now() let headers = 0 let chain: Promise = Promise.resolve() + // Where the range resumes after what was stored: a batch split in two reports it with its first half. + let reported = lease.start const posting = new Set>() // Throws once a post has failed, so the range stops fetching. const failedSoFar = () => Promise.race([chain, Promise.resolve()]) @@ -321,7 +323,8 @@ export default class SeedNode extends BaseCommand { if (this.stopping) throw new Stopping() while (posting.size >= 2) await Promise.race(posting) const post = chain.then(async () => { - await api.post(`/api/v1/seed/ranges/${lease.id}/batches`, batch) + await this.postBatch(api, lease.id, batch, reported) + reported = batch.next headers += batch.headers }) chain = post @@ -354,6 +357,26 @@ export default class SeedNode extends BaseCommand { } } + /** + * Posts a batch; one the seed API finds too large (413: dense recent ranges) is + * posted in two halves, at most three splits deep. Only the second half moves + * the range's resume point, so a failure in between loses nothing. + */ + private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise { + try { + await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch) + } catch (error: any) { + const parts = batch.files.length + batch.fragments.length + if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error + const files = Math.ceil(batch.files.length / 2) + const fragments = Math.ceil(batch.fragments.length / 2) + const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) } + const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) } + await this.postBatch(api, rangeId, first, reported, depth + 1) + await this.postBatch(api, rangeId, second, reported, depth + 1) + } + } + /** Sleeps, but wakes up early when the node is stopping. */ private async pause(ms: number) { for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)