implemented a lot of features

database/migrations/1768620764770_create_fetch_ranges_table.ts

@@ -0,0 +1,37 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Article ranges handed to worker:fetch, so a range only counts as indexed
+ * once it was fetched successfully (CONCEPT.md D18).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('groups', (table) => {
+      // How far the scheduler has queued; last_indexed_id becomes the
+      // watermark below which every article was fetched.
+      table.bigInteger('last_queued_id').notNullable().defaultTo(0)
+    })
+    this.defer(async (db) => {
+      await db.rawQuery('UPDATE groups SET last_queued_id = last_indexed_id')
+    })
+
+    this.schema.createTable('fetch_ranges', (table) => {
+      table.bigIncrements('id')
+      table.integer('group_id').notNullable().references('groups.id').onDelete('CASCADE')
+      table.bigInteger('start_id').notNullable()
+      table.bigInteger('end_id').notNullable()
+      // queued | done | failed
+      table.string('status', 8).notNullable().defaultTo('queued')
+      table.integer('requeues').notNullable().defaultTo(0)
+      table.text('error')
+      table.timestamp('created_at', { useTz: true }).notNullable()
+      table.timestamp('updated_at', { useTz: true }).notNullable()
+      table.index(['group_id', 'status'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable('fetch_ranges')
+    this.schema.alterTable('groups', (table) => table.dropColumn('last_queued_id'))
+  }
+}