Prober: pipelined bodies, merge-first order, a probe cache across resets
- XoverClient.bodies: PROBE_PIPELINE (8) BODY commands per connection at once, replies read in order (one at a time it ran 3-10 articles/s on 6 connections against a queue of 1M). - Claims in tiers: lone files first, then par2 sets and archives of releases without a readable name (what the par2 merge needs), then those of readable releases. - probe_cache: final answers by message-id (status, inspection with the 16 KiB hash and par2 file list), PROBE_VERSION 2. A reset empties the probe queue, not the cache; recreated candidates of probed articles are answered from it without the news server. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6d1f2cc2a1
commit
0727afe968
9 files changed
+160
-34
No files matched your search
@@ -0,0 +1,25 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* What a probed article showed, by message-id (app/services/prober.ts): an
|
||||
* article's inspection doesn't depend on the grouping rules, so a catalog
|
||||
* reset (which empties the probe queue) needn't read it from the news server
|
||||
* again. `version` is PROBE_VERSION when it was inspected: a change to the
|
||||
* inspection makes older rows stale. Not truncated by a reset.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.raw(`
|
||||
CREATE TABLE probe_cache (
|
||||
message_id text PRIMARY KEY,
|
||||
status varchar(8) NOT NULL,
|
||||
result jsonb,
|
||||
version integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
)`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.raw(`DROP TABLE IF EXISTS probe_cache`)
|
||||
}
|
||||
}
|
||||
@@ -153,6 +153,21 @@ export class Par2SetSchema extends BaseModel {
|
||||
declare setId: string
|
||||
}
|
||||
|
||||
export class ProbeCacheSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'messageId', 'result', 'status', 'version'] as const
|
||||
$columns = ProbeCacheSchema.$columns
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
@column({ isPrimary: true })
|
||||
declare messageId: string
|
||||
@column()
|
||||
declare result: any | null
|
||||
@column()
|
||||
declare status: string
|
||||
@column()
|
||||
declare version: number
|
||||
}
|
||||
|
||||
export class ProbeCandidateSchema extends BaseModel {
|
||||
static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const
|
||||
$columns = ProbeCandidateSchema.$columns
|
||||
|
||||
Reference in new issue
Block a user