A complete provisional release quiet for six hours is settled
A release the window hadn't closed on at a lease's end is written provisionally for its continuation to finish; the first run's range leases left 109k such rows at their blocks' end edges, 606 of them complete, shown as still collecting (release 10697). The coordinator now settles a provisional release that is complete and quiet for six hours as the finish would: settled_at set, its waiting probe candidates made claimable. Incomplete ones keep waiting. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
1810abd7d8
commit
db96642a72
4 files changed
+163
No files matched your search
@@ -464,6 +464,13 @@ releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
|||||||
merger: the carry does the joining. The settler goes: `settled_at` is
|
merger: the carry does the joining. The settler goes: `settled_at` is
|
||||||
the finish. The **tail** needs no path of its own: its lease ends leave
|
the finish. The **tail** needs no path of its own: its lease ends leave
|
||||||
such provisional releases, the next lease's continuation finishes them.
|
such provisional releases, the next lease's continuation finishes them.
|
||||||
|
One exception (2026-10-27): a provisional release that is complete
|
||||||
|
(every file its counter announced found) and quiet for six hours is
|
||||||
|
settled by the coordinator (`Settler.settleComplete`, every five
|
||||||
|
minutes), as the finish would: no counted file can still come, and the
|
||||||
|
continuation may be hours away or never (the first run's range leases
|
||||||
|
left 109k provisional rows at their blocks' end edges, 606 of them
|
||||||
|
complete, shown as still collecting; the user, release 10697).
|
||||||
|
|
||||||
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
||||||
only by a number sequence, or share a directory-like prefix, posted
|
only by a number sequence, or share a directory-like prefix, posted
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
|
||||||
|
/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
|
||||||
|
export const SETTLE_COMPLETE_HOURS = 6
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
|
||||||
|
* lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
|
||||||
|
* never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
|
||||||
|
* hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
|
||||||
|
* announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
|
||||||
|
* settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
|
||||||
|
* candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
|
||||||
|
*/
|
||||||
|
class Settler {
|
||||||
|
async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
|
||||||
|
return db.transaction(async (trx) => {
|
||||||
|
const { rows } = await trx.rawQuery(
|
||||||
|
`UPDATE releases SET settled_at = now()
|
||||||
|
WHERE settled_at IS NULL AND complete
|
||||||
|
AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
|
||||||
|
RETURNING id`,
|
||||||
|
[quietHours, quietHours]
|
||||||
|
)
|
||||||
|
if (rows.length > 0)
|
||||||
|
await trx.rawQuery(
|
||||||
|
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||||
|
WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||||
|
[rows.map((r: { id: string }) => Number(r.id))]
|
||||||
|
)
|
||||||
|
return rows.length
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default new Settler()
|
||||||
@@ -4,6 +4,7 @@ import { setTimeout as sleep } from 'node:timers/promises'
|
|||||||
import { Prefetcher } from '#services/prefetcher'
|
import { Prefetcher } from '#services/prefetcher'
|
||||||
import Planner from '#services/planner'
|
import Planner from '#services/planner'
|
||||||
import Leases from '#services/leases'
|
import Leases from '#services/leases'
|
||||||
|
import Settler from '#services/settler'
|
||||||
import NntpPool from '#services/nntp_pool'
|
import NntpPool from '#services/nntp_pool'
|
||||||
import { NntpShare } from '#services/nntp_shares'
|
import { NntpShare } from '#services/nntp_shares'
|
||||||
import coordinatorConfig from '#config/coordinator'
|
import coordinatorConfig from '#config/coordinator'
|
||||||
@@ -101,6 +102,13 @@ export default class CoordinatorRun extends BaseCommand {
|
|||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
|
this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
|
const settled = await Settler.settleComplete()
|
||||||
|
if (settled > 0)
|
||||||
|
this.logger.info(`Settled ${settled} complete provisional releases quiet for hours.`)
|
||||||
|
} catch (error: any) {
|
||||||
|
this.logger.error(`Settling complete releases failed: ${error.message}`)
|
||||||
|
}
|
||||||
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
|
for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
|
||||||
await sleep(TICK_MS)
|
await sleep(TICK_MS)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import Settler from '#services/settler'
|
||||||
|
import { insertRelease } from '#tests/functional/helpers/releases'
|
||||||
|
|
||||||
|
const hoursAgo = (h: number) => new Date(Date.now() - h * 3600_000)
|
||||||
|
|
||||||
|
test.group('Settler', (group) => {
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
|
||||||
|
test('a complete provisional release quiet for six hours is settled, its waiting candidates made claimable', async ({
|
||||||
|
assert,
|
||||||
|
}) => {
|
||||||
|
const base = {
|
||||||
|
groups: ['alt.binaries.test'],
|
||||||
|
poster: 'p@x',
|
||||||
|
first_article: 1,
|
||||||
|
posted_at: hoursAgo(30),
|
||||||
|
last_posted_at: hoursAgo(30),
|
||||||
|
}
|
||||||
|
// Complete and old: settled.
|
||||||
|
const old = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'old',
|
||||||
|
files: 3,
|
||||||
|
file_total: 3,
|
||||||
|
complete: true,
|
||||||
|
readable: true,
|
||||||
|
first_seen_at: hoursAgo(7),
|
||||||
|
})
|
||||||
|
// Complete but seen an hour ago: its window may still be carried.
|
||||||
|
const recent = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'recent',
|
||||||
|
files: 3,
|
||||||
|
file_total: 3,
|
||||||
|
complete: true,
|
||||||
|
first_seen_at: hoursAgo(1),
|
||||||
|
})
|
||||||
|
// Complete, old, but its last part posted an hour ago: still arriving.
|
||||||
|
const posting = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'posting',
|
||||||
|
files: 3,
|
||||||
|
file_total: 3,
|
||||||
|
complete: true,
|
||||||
|
first_seen_at: hoursAgo(7),
|
||||||
|
last_posted_at: hoursAgo(1),
|
||||||
|
})
|
||||||
|
// Incomplete, and without a counter: the continuation's.
|
||||||
|
const partial = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'partial',
|
||||||
|
files: 2,
|
||||||
|
file_total: 3,
|
||||||
|
complete: false,
|
||||||
|
first_seen_at: hoursAgo(7),
|
||||||
|
})
|
||||||
|
const uncounted = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'uncounted',
|
||||||
|
files: 2,
|
||||||
|
file_total: null,
|
||||||
|
complete: null,
|
||||||
|
first_seen_at: hoursAgo(7),
|
||||||
|
})
|
||||||
|
// Settled already: left alone.
|
||||||
|
const settled = await insertRelease({
|
||||||
|
...base,
|
||||||
|
name: 'settled',
|
||||||
|
files: 3,
|
||||||
|
file_total: 3,
|
||||||
|
complete: true,
|
||||||
|
first_seen_at: hoursAgo(7),
|
||||||
|
settled_at: hoursAgo(6),
|
||||||
|
})
|
||||||
|
await db.table('probe_candidates').multiInsert([
|
||||||
|
{ release_id: old, kind: 'par2', message_id: '<old@x>', file: 'x.par2', status: 'waiting' },
|
||||||
|
{
|
||||||
|
release_id: partial,
|
||||||
|
kind: 'par2',
|
||||||
|
message_id: '<partial@x>',
|
||||||
|
file: 'x.par2',
|
||||||
|
status: 'waiting',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
assert.equal(await Settler.settleComplete(), 1)
|
||||||
|
const rows = await db
|
||||||
|
.from('releases')
|
||||||
|
.whereIn('id', [old, recent, posting, partial, uncounted, settled])
|
||||||
|
.select('id', 'settled_at')
|
||||||
|
const settledIds = rows
|
||||||
|
.filter((r) => r.settled_at !== null)
|
||||||
|
.map((r) => Number(r.id))
|
||||||
|
.sort()
|
||||||
|
assert.deepEqual(settledIds, [old, settled].sort())
|
||||||
|
const candidates = await db
|
||||||
|
.from('probe_candidates')
|
||||||
|
.whereIn('release_id', [old, partial])
|
||||||
|
.select('release_id', 'ready', 'unreadable')
|
||||||
|
assert.deepEqual(
|
||||||
|
candidates.map((c) => [Number(c.release_id), c.ready, c.unreadable]).sort(),
|
||||||
|
[
|
||||||
|
[old, true, false],
|
||||||
|
[partial, false, false],
|
||||||
|
].sort()
|
||||||
|
)
|
||||||
|
assert.equal(await Settler.settleComplete(), 0)
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in new issue
Block a user