Compare commits
2
Commits
monetization
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7880633467 | ||
|
|
7c43abd230 |
No files matched your search
@@ -1,9 +1,8 @@
|
|||||||
# Tests every push; a push to master that passes is deployed to homelab.
|
# Tests every push. Voyager 1 (this branch) is retired since 2026-10-11: its
|
||||||
#
|
# database was deleted on 2026-10-09, and a push to master no longer deploys.
|
||||||
# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
|
# Voyager 2 (branch rebuild) has its own workflow and deploy
|
||||||
# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
|
# (deploy/README.md there); homelab's ~/bin/deploy-usenet-indexer refuses
|
||||||
# which fetches the commit from this repo, builds the image, runs the
|
# voyager 1 deploys too.
|
||||||
# migrations and recreates the app services (never postgres or redis).
|
|
||||||
name: Test and deploy
|
name: Test and deploy
|
||||||
|
|
||||||
on:
|
on:
|
||||||
@@ -63,19 +62,3 @@ jobs:
|
|||||||
- name: Test
|
- name: Test
|
||||||
run: node ace test
|
run: node ace test
|
||||||
|
|
||||||
deploy:
|
|
||||||
needs: test
|
|
||||||
if: github.ref == 'refs/heads/master'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Deploy to homelab
|
|
||||||
env:
|
|
||||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
||||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
|
||||||
run: |
|
|
||||||
install -m 700 -d ~/.ssh
|
|
||||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
|
|
||||||
chmod 600 ~/.ssh/deploy_key
|
|
||||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
|
||||||
ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
|
|
||||||
daan@10.206.0.175 "deploy ${{ github.sha }}"
|
|
||||||
@@ -2,6 +2,8 @@ import { BaseCommand } from '@adonisjs/core/ace'
|
|||||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||||
import db from '@adonisjs/lucid/services/db'
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
|
||||||
|
const BATCH = 50_000
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
||||||
* listings still on `files` into file_details, a partition at a time.
|
* listings still on `files` into file_details, a partition at a time.
|
||||||
@@ -23,13 +25,31 @@ export default class FilesMoveDetails extends BaseCommand {
|
|||||||
let total = 0
|
let total = 0
|
||||||
for (const { relname } of partitions) {
|
for (const { relname } of partitions) {
|
||||||
const started = Date.now()
|
const started = Date.now()
|
||||||
const result = await db.rawQuery(
|
const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
|
||||||
`INSERT INTO file_details (file_id, content_type, archive)
|
if (rows[0].lo === null) continue
|
||||||
SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
|
let copied = 0
|
||||||
ON CONFLICT (file_id) DO NOTHING`
|
// Short statements in id order: one statement over a whole partition took 30 minutes, and the
|
||||||
)
|
// garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
|
||||||
total += result.rowCount ?? 0
|
for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
|
||||||
this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
for (let attempt = 1; ; attempt++) {
|
||||||
|
try {
|
||||||
|
const result = await db.rawQuery(
|
||||||
|
`INSERT INTO file_details (file_id, content_type, archive)
|
||||||
|
SELECT id, content_type, archive FROM ${relname}
|
||||||
|
WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
|
||||||
|
ON CONFLICT (file_id) DO NOTHING`,
|
||||||
|
[lo, lo + BATCH]
|
||||||
|
)
|
||||||
|
copied += result.rowCount ?? 0
|
||||||
|
break
|
||||||
|
} catch (error: any) {
|
||||||
|
// A file deleted while this batch ran: the batch again, without it.
|
||||||
|
if (error.code !== '23503' || attempt >= 5) throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total += copied
|
||||||
|
this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||||
}
|
}
|
||||||
this.logger.info(`${total} files' details copied.`)
|
this.logger.info(`${total} files' details copied.`)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user