Rebuild: a lease never waits for prefetching; one-shot hashes
A lease was only granted for a range with spooled chunks, and the prefetcher kept ranges ahead per active lease: once the spool ran dry a worker got none and slept 30 s, fewer ranges were leased, fewer prefetched. Three workers held one lease between them (~50k headers/s). A lease now takes a spooled range first, else the next planned one (leased ranges are fetched first; the stream waits). crypto.hash instead of createHash().update().digest() for file and release keys: same digests, ~2.5x faster (~4.6% of a worker's time). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
168605148f
commit
9dc0a27bb4
5 files changed
+30
-25
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { hash } from 'node:crypto'
|
||||
import type { Overview } from '../nntp/index.js'
|
||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||
import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
|
||||
@@ -281,7 +281,7 @@ export class RangeIndexer {
|
||||
|
||||
/** The release key; a file without a recognisable release becomes a release of its own. */
|
||||
export function releaseKeyOf(s: EdgeSegment): string {
|
||||
return s.file.release?.key ?? createHash('sha1').update(`file\n${s.fileKey}`).digest('hex')
|
||||
return s.file.release?.key ?? hash('sha1', `file\n${s.fileKey}`)
|
||||
}
|
||||
|
||||
function postedOf(segments: EdgeSegment[]): number | null {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { hash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Identity of one uploaded file: every segment of it has the same poster, the
|
||||
@@ -7,7 +7,7 @@ import { createHash } from 'node:crypto'
|
||||
* See CONCEPT.md (D12).
|
||||
*/
|
||||
export function fileKey(poster: string, base: string, totalSegments: number): string {
|
||||
return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex')
|
||||
return hash('sha1', `${poster}\n${base}\n${totalSegments}`)
|
||||
}
|
||||
|
||||
/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */
|
||||
@@ -26,7 +26,7 @@ export function looksLikeFilename(name: string): boolean {
|
||||
*/
|
||||
export function weakFileKey(name: string, size: number, totalParts: number): string {
|
||||
const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
|
||||
return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
|
||||
return hash('sha1', `weak\n${identity}\n${size}\n${totalParts}`)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,7 +41,5 @@ export function counterFileKey(segment: { poster: string; subject: string; total
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
|
||||
if (!counter || segment.totalParts < 2) return null
|
||||
const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
|
||||
return createHash('sha1')
|
||||
.update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
|
||||
.digest('hex')
|
||||
return hash('sha1', `counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { hash as digest } from 'node:crypto'
|
||||
import { looksLikeFilename } from '../headers/file_key.js'
|
||||
|
||||
export interface ReleaseIdentity {
|
||||
@@ -103,7 +103,7 @@ export function namesRelease(remainder: string): boolean {
|
||||
}
|
||||
|
||||
function hash(...parts: (string | number | undefined)[]) {
|
||||
return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
|
||||
return digest('sha1', parts.map((p) => p ?? '').join('\n'))
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { hash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Upload signatures (CONCEPT.md D42): which tool posted an article, read from
|
||||
@@ -48,5 +48,5 @@ export function signatureKey(tool: string, group: string) {
|
||||
*/
|
||||
export function explored(key: string, percent: number): boolean {
|
||||
if (percent <= 0) return false
|
||||
return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100
|
||||
return hash('md5', key, 'buffer').readUInt32BE(0) % 10_000 < percent * 100
|
||||
}
|
||||
Reference in new issue
Block a user