Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7880633467 | ||
|
|
7c43abd230 |
No files matched your search
@@ -1,9 +1,8 @@
|
||||
# Tests every push; a push to master that passes is deployed to homelab.
|
||||
#
|
||||
# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
|
||||
# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
|
||||
# which fetches the commit from this repo, builds the image, runs the
|
||||
# migrations and recreates the app services (never postgres or redis).
|
||||
# Tests every push. Voyager 1 (this branch) is retired since 2026-10-11: its
|
||||
# database was deleted on 2026-10-09, and a push to master no longer deploys.
|
||||
# Voyager 2 (branch rebuild) has its own workflow and deploy
|
||||
# (deploy/README.md there); homelab's ~/bin/deploy-usenet-indexer refuses
|
||||
# voyager 1 deploys too.
|
||||
name: Test and deploy
|
||||
|
||||
on:
|
||||
@@ -63,19 +62,3 @@ jobs:
|
||||
- name: Test
|
||||
run: node ace test
|
||||
|
||||
deploy:
|
||||
needs: test
|
||||
if: github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy to homelab
|
||||
env:
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
install -m 700 -d ~/.ssh
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||
ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
|
||||
daan@10.206.0.175 "deploy ${{ github.sha }}"
|
||||
@@ -2,6 +2,8 @@ import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
const BATCH = 50_000
|
||||
|
||||
/**
|
||||
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
||||
* listings still on `files` into file_details, a partition at a time.
|
||||
@@ -23,13 +25,31 @@ export default class FilesMoveDetails extends BaseCommand {
|
||||
let total = 0
|
||||
for (const { relname } of partitions) {
|
||||
const started = Date.now()
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_details (file_id, content_type, archive)
|
||||
SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
|
||||
ON CONFLICT (file_id) DO NOTHING`
|
||||
)
|
||||
total += result.rowCount ?? 0
|
||||
this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
|
||||
if (rows[0].lo === null) continue
|
||||
let copied = 0
|
||||
// Short statements in id order: one statement over a whole partition took 30 minutes, and the
|
||||
// garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
|
||||
for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_details (file_id, content_type, archive)
|
||||
SELECT id, content_type, archive FROM ${relname}
|
||||
WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
|
||||
ON CONFLICT (file_id) DO NOTHING`,
|
||||
[lo, lo + BATCH]
|
||||
)
|
||||
copied += result.rowCount ?? 0
|
||||
break
|
||||
} catch (error: any) {
|
||||
// A file deleted while this batch ran: the batch again, without it.
|
||||
if (error.code !== '23503' || attempt >= 5) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
total += copied
|
||||
this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
this.logger.info(`${total} files' details copied.`)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user