usenet-indexer/app/services/sidecar_folder.ts

import db from '@adonisjs/lucid/services/db'
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'

/** Candidates looked at per round. */
const BATCH = 2000
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
const WINDOW_MINUTES = 10
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i

/** The words of a name worth comparing: 4+ letters or digits. */
function words(text: string): Set<string> {
  return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
}

export interface Sibling {
  id: number
  name: string
  files: number
  file_total: number | null
}

/**
 * Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
 * ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
 * under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
 * isn't one release.
 */
export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
  const said = words(files.map((f) => f.subject).join(' '))
  const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
  if (named.length === 1) return named[0]
  if (named.length > 1) return null
  const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
  return counted.length === 1 ? counted[0] : null
}

export interface FoldRound {
  looked: number
  sidecars: number
  folded: number
}

/**
 * The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
 * counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
 * settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
 * the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
 * 'sidecar'). The rest leave the queue untouched.
 */
export class SidecarFolder {
  constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}

  async round(): Promise<FoldRound> {
    const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
    const { rows: queued } = await db.rawQuery(
      `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
       FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
      [BATCH]
    )
    const { rows: stale } = await db.rawQuery(
      `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
      [BATCH]
    )
    const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
    for (const c of queued) {
      round.looked++
      done.push(Number(c.id))
      const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
      const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
      if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
      round.sidecars++
      if (!c.posted_at) continue
      const { rows: siblings } = await db.rawQuery(
        `SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
         WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
           AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
        [c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
      )
      const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
      if (!sibling) continue
      const full = siblings.find((s: any) => Number(s.id) === sibling.id)
      const merged = await db.transaction((trx) =>
        this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
      )
      if (merged > 0) round.folded++
    }
    if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
    return round
  }
}