usenet-indexer/app/services/ArchiveWalkService.ts

import db from '@adonisjs/lucid/services/db'
import env from '#start/env'
import NntpService from '#services/NntpService'
import QueueService from '#services/QueueService'
import { segmentsOf } from '#services/FileStore'
import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
import { rarVolume } from '#lib/content/index'
import type { RarEntry, RarInfo } from '#lib/rar/index'
import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'

/** Stored per release, as for a first-volume listing (ArchiveService). */
const MAX_ENTRIES = 500

/** Listed sizes below this share of the release's size: the first segment listed only part of it. */
const COVERED = 0.7

/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */
const WALK_DELAY_MS = 30 * 60 * 1000

export interface WalkOutcome {
  releaseId: number
  /** Nothing downloaded: why. */
  skipped?: 'missing' | 'covered' | 'volumes'
  entries?: number
  before?: number
  complete?: boolean
  stopped?: string
  segments?: number
  renamed?: string
}

/**
 * Complete archive listings by walking RAR headers (CONCEPT.md D40). The
 * first segment of a first volume lists the entries that start in it: for
 * a set of many small files (an audiobook's MP3s, a music album) that is
 * the first one or two. The walk reads one segment per further entry and
 * volume, at low priority: its own queue, one job at a time per collection
 * worker, and NNTP connections only when inspection isn't waiting for one.
 */
class ArchiveWalkService {
  /** Segments one walk may download (~0.7 MB each). */
  readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400)

  /** A listing that leaves most of the release unaccounted for. */
  worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean {
    if (entries === 0 || entries >= MAX_ENTRIES) return false
    return listedBytes < COVERED * releaseSize
  }

  /** After inspection listed a first RAR volume whose listing ends inside the set. */
  async queueAfterInspection(releaseId: number | null, info: RarInfo) {
    if (!releaseId || info.format === '7z' || info.firstVolume === false) return
    if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return
    await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
  }

  /**
   * The database half before a walk (D41): the release's volumes, or the
   * outcome when there's nothing to walk.
   */
  async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
    const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
    if (!release) return { outcome: { releaseId, skipped: 'missing' } }
    const before = Array.isArray(release.contents) ? release.contents : []
    const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
    if (!this.worthWalking(before.length, listed, Number(release.size))) {
      return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
    }
    const volumes = await this.volumes(releaseId)
    if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
    return {
      task: {
        kind: 'walk',
        releaseId,
        volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
        maxSegments: this.maxSegments,
      },
    }
  }

  /** The database half after a walk: the fuller listing, the password flag and a better name. */
  async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
    const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
    if (!release) return { releaseId, skipped: 'missing' }
    const before = Array.isArray(release.contents) ? release.contents : []
    const outcome: WalkOutcome = {
      releaseId,
      before: before.length,
      entries: walked.entries.length,
      complete: walked.complete,
      stopped: walked.stopped,
      segments: walked.segments,
    }
    if (walked.entries.length > before.length) {
      const contents = walked.entries.slice(0, MAX_ENTRIES)
      const update: Record<string, unknown> = {
        contents: JSON.stringify(contents),
        encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
      }
      const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
      // The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
      if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
        update.name = better
        outcome.renamed = better
      }
      await db.from('releases').where('id', releaseId).update(update)
    }
    return this.done(releaseId, outcome)
  }

  /** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
  async walk(releaseId: number): Promise<WalkOutcome> {
    const prepared = await this.prepare(releaseId)
    if ('outcome' in prepared) return prepared.outcome
    return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
  }

  private async done(releaseId: number, outcome: WalkOutcome) {
    await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
    return outcome
  }

  /**
   * The release's RAR volumes in order, from volume 1 up to the first gap.
   * Null without a first volume (e.g. volumes with random names).
   */
  private async volumes(releaseId: number): Promise<any[] | null> {
    const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments')
    const sets = new Map<string, Map<number, any>>()
    for (const file of files) {
      const volume = rarVolume(String(file.real_name ?? file.filename))
      if (!volume || segmentsOf(file).length === 0) continue
      const set = sets.get(volume.base) ?? new Map()
      if (!set.has(volume.number)) set.set(volume.number, file)
      sets.set(volume.base, set)
    }
    // A release holds one set, now and then a second (a sample, subs): the biggest.
    const set = [...sets.values()].sort((a, b) => b.size - a.size)[0]
    if (!set?.has(1)) return null
    const ordered: any[] = []
    for (let n = 1; set.has(n); n++) ordered.push(set.get(n))
    return ordered
  }
}

/** One article body on a pooled connection, behind inspection's waiting jobs. */
async function fetchLow(messageId: string): Promise<Buffer> {
  const conn = await NntpService.acquire(true)
  let failed = false
  try {
    return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY')
  } catch (error) {
    failed = true
    throw error
  } finally {
    NntpService.release(conn, failed)
  }
}

export default new ArchiveWalkService()