diff --git a/app/services/pending_ladder.ts b/app/services/pending_ladder.ts index 05c4cf8..3416c16 100644 --- a/app/services/pending_ladder.ts +++ b/app/services/pending_ladder.ts @@ -438,8 +438,9 @@ class PendingLadder { for (const s of signatures) { let rows: any[] do { - // The poster's rows by the signature index, a tool's and group's by the ladder's: never a test on the rows, - // which was a scan of the whole table per signature on the HDD (LOG.md (134)). + // The poster's rows by the signature index (on the poster's md5: a poster can be longer than a btree + // key), a tool's and group's by the ladder's: never a test on the rows, which was a scan of the whole + // table per signature on the HDD (LOG.md (134)). ;({ rows } = s.poster === '' ? await db.rawQuery( @@ -449,8 +450,8 @@ class PendingLadder { ) : await db.rawQuery( `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases - WHERE status = ? AND group_id = ? AND tool = ? AND poster = ? LIMIT ?`, - [PENDING.waiting, s.group_id, s.tool, s.poster, limit] + WHERE status = ? AND group_id = ? AND tool = ? AND md5(poster) = md5(?::text) AND poster = ? LIMIT ?`, + [PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit] )) discarded += await this.discardKeys(rows, samplePercent) } while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent))) diff --git a/database/migrations/1770000007200_pending_signature_index.ts b/database/migrations/1770000007200_pending_signature_index.ts index 59b38a2..77fc623 100644 --- a/database/migrations/1770000007200_pending_signature_index.ts +++ b/database/migrations/1770000007200_pending_signature_index.ts @@ -3,14 +3,17 @@ import { BaseSchema } from '@adonisjs/lucid/schema' /** * The discard sweep of a dropped signature (tool, group, poster) read the whole pending_releases table per * signature: the planner took a parallel seq scan over the poster test, four minutes on the HDD that starved the - * lanes' claims (LOG.md (134)). The waiting rows by signature make it an index range. + * lanes' claims (LOG.md (134)). The waiting rows by signature make it an index range. By the poster's md5: a + * poster of 2.7 kB broke the first build (a btree key is at most a third of a page), which left an invalid index + * behind that IF NOT EXISTS would have taken for the real one; the drop clears it. */ export default class extends BaseSchema { static disableTransactions = true async up() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_signature_idx`) this.schema.raw( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_signature_idx ON pending_releases (group_id, tool, poster) WHERE status = 0` + `CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_signature_idx ON pending_releases (group_id, tool, md5(poster)) WHERE status = 0` ) }