Complete archive listings by walking RAR headers, at low priority
Inspection lists only the entries whose headers are in the first segment of a first volume. A walk hops from header to header, fetching one segment per entry and volume, in its own queue (one job per collection worker) on NNTP connections inspection isn't waiting for. content:walk-archives queues releases inspected earlier (CONCEPT D40). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b7ae8267c6
commit
f4ded062df
17 files changed
+597
-17
No files matched your search
@@ -7,6 +7,8 @@ class NntpService {
|
||||
private allConnections: Set<any>;
|
||||
private idleConnections: any[];
|
||||
private waiters: ((conn: any) => void)[];
|
||||
/** Background work (archive walks): served only when no one else waits. */
|
||||
private lowWaiters: ((conn: any) => void)[] = [];
|
||||
private createdCount: number;
|
||||
|
||||
constructor(poolSize = 10) {
|
||||
@@ -34,7 +36,8 @@ class NntpService {
|
||||
return conn;
|
||||
}
|
||||
|
||||
public async acquire() {
|
||||
/** A connection from the pool; with `low`, after every waiting normal caller. */
|
||||
public async acquire(low = false) {
|
||||
if (this.idleConnections.length > 0) {
|
||||
return this.idleConnections.pop();
|
||||
}
|
||||
@@ -45,7 +48,7 @@ class NntpService {
|
||||
return this._createConnection();
|
||||
}
|
||||
|
||||
return new Promise(resolve => this.waiters.push(resolve));
|
||||
return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -58,7 +61,7 @@ class NntpService {
|
||||
this.discard(conn);
|
||||
return;
|
||||
}
|
||||
const resolve = this.waiters.shift();
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
resolve(conn);
|
||||
} else {
|
||||
@@ -73,13 +76,17 @@ class NntpService {
|
||||
this._destroy(conn);
|
||||
|
||||
// Someone is waiting for a slot: open a fresh connection for them.
|
||||
const resolve = this.waiters.shift();
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
this.createdCount++;
|
||||
this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
private nextWaiter() {
|
||||
return this.waiters.shift() ?? this.lowWaiters.shift();
|
||||
}
|
||||
|
||||
private _destroy(conn: any) {
|
||||
try {
|
||||
conn.sock?.destroy();
|
||||
@@ -126,6 +133,7 @@ class NntpService {
|
||||
this.allConnections.clear();
|
||||
this.idleConnections.length = 0;
|
||||
this.waiters.length = 0;
|
||||
this.lowWaiters.length = 0;
|
||||
this.createdCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user