Compare commits
2
Commits
6ef7ddca92
...
eadae82b0c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eadae82b0c | ||
|
|
e02a7c89b4 |
No files matched your search
+7
-4
@@ -717,10 +717,13 @@ releases; the inspector matches them again), `nntp_shares`, `nodes`,
|
||||
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
|
||||
`settled_at IS NOT NULL`), the settler, the fragment merger,
|
||||
`releases_category_newest_idx`.
|
||||
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
|
||||
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
|
||||
rule read all of `release_inspections` every round, 17 min on the HDD;
|
||||
the judge must read it by release or keep what it needs on the NVMe.
|
||||
- `release_inspections` and `probe_candidates` were on `CHUNK_TABLESPACE`
|
||||
(the HDD) with their indexes; both are back on the NVMe (migrations 4800
|
||||
and 7300). Note LOG.md 2026-10-19 (81): the judge's fake rule read all of
|
||||
`release_inspections` every round, 17 min on the HDD; and 2026-11-03
|
||||
(136): the prober's catalog claim updates `probe_candidates` from random
|
||||
points every minute, 2–3 min on the HDD beside the pending ladder's sweep.
|
||||
What a worker reads or updates by random key every minute stays on the NVMe.
|
||||
|
||||
### Batch store (built 2026-10-25)
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Everything the prober reads or writes in the catalog's tablespace (default_tablespace: nvme on homelab), out of
|
||||
* CHUNK_TABLESPACE (the HDD): pending_releases with its two HDD indexes (migration 6300), release_inspections
|
||||
* (6000), probe_cache with its toast. The edge's batch upserts into pending_releases took minutes each on the HDD and
|
||||
* the prober's flushes waited on them (LOG.md 2026-11-03 (137)); and with these on the NVMe the prober can run
|
||||
* through the night without the HDD. Homelab moved them by hand (edge, node-api and the workers stopped; ~66 GB);
|
||||
* a no-op where they're there already, or without a default tablespace (tests).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.defer(async (db) => {
|
||||
const { rows: setting } = await db.rawQuery(
|
||||
`SELECT current_setting('default_tablespace') AS target`
|
||||
)
|
||||
const target = setting[0].target
|
||||
if (!target) return
|
||||
for (const table of ['pending_releases', 'release_inspections', 'probe_cache']) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||
WHERE (c.relname = ? OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = ?::regclass))
|
||||
AND c.relkind IN ('r', 'i') AND coalesce(t.spcname, 'pg_default') <> ?
|
||||
ORDER BY c.relkind DESC, c.relname`,
|
||||
[table, table, target]
|
||||
)
|
||||
for (const { relname, relkind } of rows) {
|
||||
await db.rawQuery(
|
||||
`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async down() {}
|
||||
}
|
||||
+3
-2
@@ -47,8 +47,9 @@ echo "== recreating app services"
|
||||
docker compose stop edge
|
||||
if [ -e "$CACHE/paused" ]; then
|
||||
# Paused for the night (~/bin/usenet-rebuild-pause.sh): the workers stay stopped, the resume starts the new build.
|
||||
echo "== paused: coordinator, prober, merger, judge, spotter, spot-inspector and edge left stopped"
|
||||
docker compose up -d --no-deps node-api web
|
||||
# The prober runs through the night (2026-11-03, LOG.md (137)), so it gets the new build now.
|
||||
echo "== paused: coordinator, merger, judge, spotter, spot-inspector and edge left stopped"
|
||||
docker compose up -d --no-deps node-api web prober
|
||||
else
|
||||
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
||||
if [ -e "$CACHE/no-edge" ]; then
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Installed as ~/bin/usenet-rebuild-pause.sh (by hand). Nightly (homelab crontab, 23:45): voyager 2's processing paused for the night; usenet-rebuild-resume.sh (07:45)
|
||||
# starts it again. Postgres, web and node-api stay up. Takes the deploy lock, so it never runs during a CI deploy, and
|
||||
# leaves .cache/paused, so a deploy in the night doesn't start the workers again.
|
||||
# starts it again. Postgres, web, node-api and the prober stay up: since 2026-11-03 (LOG.md (137)) everything the
|
||||
# prober touches is on the NVMe, so it probes through the night without the HDD. Takes the deploy lock, so it never
|
||||
# runs during a CI deploy, and leaves .cache/paused, so a deploy in the night doesn't start the workers again.
|
||||
cd /mnt/bulk/usenet-rebuild || exit 1
|
||||
exec 9>.cache/deploy.lock
|
||||
flock 9
|
||||
touch .cache/paused
|
||||
date "+%F %T pausing"
|
||||
docker compose stop edge
|
||||
docker compose stop coordinator prober merger judge spotter spot-inspector
|
||||
docker compose stop coordinator merger judge spotter spot-inspector
|
||||
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||
Reference in new issue
Block a user