Complete archive listings by walking RAR headers, at low priority
Test and deploy / test (push) Successful in 1m24s
Test and deploy / deploy (push) Successful in 1m58s

Inspection lists only the entries whose headers are in the first segment
of a first volume. A walk hops from header to header, fetching one segment
per entry and volume, in its own queue (one job per collection worker) on
NNTP connections inspection isn't waiting for. content:walk-archives queues
releases inspected earlier (CONCEPT D40).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-05 23:32:52 +02:00
1 parent b7ae8267c6
commit f4ded062df
17 files changed
+597 -17

No files matched your search

+12 -4
View File
@@ -7,6 +7,8 @@ class NntpService {
private allConnections: Set<any>;
private idleConnections: any[];
private waiters: ((conn: any) => void)[];
/** Background work (archive walks): served only when no one else waits. */
private lowWaiters: ((conn: any) => void)[] = [];
private createdCount: number;
constructor(poolSize = 10) {
@@ -34,7 +36,8 @@ class NntpService {
return conn;
}
public async acquire() {
/** A connection from the pool; with `low`, after every waiting normal caller. */
public async acquire(low = false) {
if (this.idleConnections.length > 0) {
return this.idleConnections.pop();
}
@@ -45,7 +48,7 @@ class NntpService {
return this._createConnection();
}
return new Promise(resolve => this.waiters.push(resolve));
return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
}
/**
@@ -58,7 +61,7 @@ class NntpService {
this.discard(conn);
return;
}
const resolve = this.waiters.shift();
const resolve = this.nextWaiter();
if (resolve) {
resolve(conn);
} else {
@@ -73,13 +76,17 @@ class NntpService {
this._destroy(conn);
// Someone is waiting for a slot: open a fresh connection for them.
const resolve = this.waiters.shift();
const resolve = this.nextWaiter();
if (resolve) {
this.createdCount++;
this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
}
}
private nextWaiter() {
return this.waiters.shift() ?? this.lowWaiters.shift();
}
private _destroy(conn: any) {
try {
conn.sock?.destroy();
@@ -126,6 +133,7 @@ class NntpService {
this.allConnections.clear();
this.idleConnections.length = 0;
this.waiters.length = 0;
this.lowWaiters.length = 0;
this.createdCount = 0;
}
}