usenet-indexer/public/app.js

// Usenet Indexer web interface. No framework; all Usenet-derived text is
// rendered with textContent, never as HTML.

const $ = (selector) => document.querySelector(selector)
const FILTERS = ['readable', 'complete', 'hide_encrypted', 'par2_only']
const SOURCE_LABELS = {
  stem: 'Filename',
  subject: 'Subject',
  post: 'Post id',
  par2: 'par2 contents',
  rar: 'RAR headers',
  'weak-stem': 'Obfuscated filename',
  nzb: 'Posted NZB',
  spot: 'Spotnet',
}

const state = {
  q: '',
  sort: 'newest',
  source: '',
  poster: '',
  readable: false,
  complete: false,
  hide_encrypted: false,
  par2_only: false,
  page: 1,
  release: null,
}
let requestId = 0
let searchAbort = null

// --- Formatting ------------------------------------------------------------

function formatSize(bytes) {
  if (!bytes) return '–'
  const units = ['B', 'KB', 'MB', 'GB', 'TB']
  let value = bytes
  let unit = 0
  while (value >= 1000 && unit < units.length - 1) {
    value /= 1000
    unit++
  }
  return `${value >= 100 || unit === 0 ? Math.round(value) : value.toFixed(1)} ${units[unit]}`
}

const number = new Intl.NumberFormat()

function formatAge(date) {
  if (!date) return ''
  const seconds = (Date.now() - new Date(date).getTime()) / 1000
  if (seconds < 90) return 'just now'
  const steps = [
    [60, 'minute'],
    [3600, 'hour'],
    [86400, 'day'],
    [2592000, 'month'],
    [31536000, 'year'],
  ]
  let [size, name] = steps[0]
  for (const step of steps) if (seconds >= step[0]) [size, name] = step
  const n = Math.floor(seconds / size)
  return `${n} ${name}${n === 1 ? '' : 's'} ago`
}

function el(tag, props = {}, ...children) {
  const node = document.createElement(tag)
  for (const [key, value] of Object.entries(props)) {
    if (value === undefined || value === null || value === false) continue
    if (key === 'class') node.className = value
    else if (key === 'text') node.textContent = value
    else if (key.startsWith('on')) node.addEventListener(key.slice(2), value)
    else node.setAttribute(key, value === true ? '' : value)
  }
  for (const child of children.flat()) {
    if (child === null || child === undefined || child === false) continue
    node.append(child instanceof Node ? child : document.createTextNode(String(child)))
  }
  return node
}

function badge(text, tone = '', title) {
  return el('span', { class: `badge${tone ? ` badge-${tone}` : ''}`, text, title })
}

// --- URL state -------------------------------------------------------------

function readUrl() {
  const params = new URLSearchParams(location.search)
  state.q = params.get('q') ?? ''
  state.sort = params.get('sort') ?? 'newest'
  state.source = params.get('source') ?? ''
  state.poster = params.get('poster') ?? ''
  for (const f of FILTERS) state[f] = params.get(f) === '1'
  state.release = params.get('release') ? Number(params.get('release')) : null
}

function writeUrl(push = false) {
  const params = new URLSearchParams()
  if (state.q) params.set('q', state.q)
  if (state.sort !== 'newest') params.set('sort', state.sort)
  if (state.source) params.set('source', state.source)
  if (state.poster) params.set('poster', state.poster)
  for (const f of FILTERS) if (state[f]) params.set(f, '1')
  if (state.release) params.set('release', String(state.release))
  const url = `${location.pathname}${params.size ? `?${params}` : ''}`
  history[push ? 'pushState' : 'replaceState'](null, '', url)
}

function syncControls() {
  $('#q').value = state.q
  $('#sort').value = state.sort
  $('#source').value = state.source
  for (const f of FILTERS) {
    document.querySelector(`[data-filter="${f}"]`).setAttribute('aria-pressed', String(state[f]))
  }
  $('#poster-filter').hidden = !state.poster
  $('#poster-filter-label').textContent = `Poster: ${state.poster}`
}

// --- API -------------------------------------------------------------------

async function api(path, { signal } = {}) {
  const response = await fetch(path, { headers: { accept: 'application/json' }, signal })
  if (!response.ok) throw new Error(`${response.status} ${response.statusText}`)
  return response.json()
}

function searchParams(page) {
  const params = new URLSearchParams({ sort: state.sort, page: String(page), per_page: '50' })
  if (state.q) params.set('q', state.q)
  if (state.source) params.set('source', state.source)
  if (state.poster) params.set('poster', state.poster)
  for (const f of FILTERS) if (state[f]) params.set(f, '1')
  return params
}

// --- Stats -----------------------------------------------------------------

async function loadStats() {
  try {
    const stats = await api('/api/v1/stats')
    const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0
    // Estimated from Postgres statistics and a sample (CONCEPT.md D33).
    const about = stats.approximate ? '≈ ' : ''
    const items = [
      ['Releases', about + number.format(stats.releases)],
      ['Readable', `${about}${readable}%`],
      ['Files', about + number.format(stats.files)],
      ['Size', about + formatSize(stats.size)],
      ['Indexed', formatAge(stats.lastIndexedAt) || '–'],
    ]
    $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd }))))
  } catch {
    // Stats are decoration; the page works without them.
  }
}

// --- Results ---------------------------------------------------------------

function releaseRow(release) {
  const row = $('#row-template').content.firstElementChild.cloneNode(true)
  row.dataset.id = release.id
  if (release.id === state.release) row.classList.add('selected')

  const name = row.querySelector('.release-name')
  name.textContent = release.name
  if (!release.readable) name.classList.add('obfuscated')

  const meta = [
    formatAge(release.postedAt),
    release.poster ? release.poster.replace(/\s*<.*>$/, '') : 'obfuscated poster',
    release.groups?.length ? release.groups[0] + (release.groups.length > 1 ? ` +${release.groups.length - 1}` : '') : '',
  ].filter(Boolean)
  row.querySelector('.release-meta').textContent = meta.join(' · ')

  const badges = row.querySelector('.badges')
  if (release.par2Only) badges.append(badge('par2 only', 'warn', 'Only the par2 files arrived; the data they describe is missing'))
  if (release.container === 'nzbe') badges.append(badge('Encrypted NZB', 'bad', 'An encrypted NZB container: unusable without its key'))
  else if (release.container === 'rclone') badges.append(badge('Encrypted (rclone)', 'bad', 'rclone crypt: names and contents encrypted, unusable without the key'))
  else if (release.container === 'yencbin') badges.append(badge('Encrypted (.bin)', 'bad', 'An encrypted .bin container (yEncBin): unusable without its key'))
  else if (release.passwordState === 'missing') badges.append(badge('Password needed', 'bad', 'Password-protected archive; the password is not known'))
  else if (release.passwordState === 'known') {
    badges.append(badge('Password known', 'good', 'The password is in the NZB; SABnzbd and NZBGet use it to unpack'))
  }
  const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}`
  if (release.listingOnly) badges.append(badge('Spot', 'accent', 'A Spotnet spot: its NZB is fetched from Usenet when you download it'))
  else if (release.complete) badges.append(badge(files, 'good', 'Complete'))
  else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
  else if (release.listedFiles) {
    badges.append(badge(`${release.listedPresent}/${release.listedFiles} listed`, 'warn', 'Files the par2 set lists: present / total'))
  } else badges.append(badge(files))
  if (release.source && release.source !== 'stem' && release.source !== 'subject') {
    badges.append(badge(SOURCE_LABELS[release.source] ?? release.source, 'accent', 'How this release was recognised'))
  }

  row.querySelector('.release-size').textContent = formatSize(release.size)
  const nzb = row.querySelector('.nzb')
  nzb.href = `/api/v1/releases/${release.id}/nzb`
  row.querySelector('.release-main').addEventListener('click', () => openRelease(release.id, true))
  return row
}

async function loadResults({ append = false } = {}) {
  const id = ++requestId
  if (!append) state.page = 1
  // A newer search makes the running one pointless: cancel it.
  searchAbort?.abort()
  const abort = (searchAbort = new AbortController())
  $('#more').hidden = true
  if (!append) {
    $('#summary').textContent = 'Searching…'
    $('#empty').hidden = true
    // The old rows no longer match the search; grey them out until the new ones arrive.
    $('#list').classList.add('stale')
    $('.results').setAttribute('aria-busy', 'true')
  }

  try {
    const result = await api(`/api/v1/releases?${searchParams(state.page)}`, { signal: abort.signal })
    if (id !== requestId) return // a newer search superseded this one
    $('#list').classList.remove('stale')
    $('.results').removeAttribute('aria-busy')
    const rows = result.releases.map(releaseRow)
    if (append) $('#list').append(...rows)
    else $('#list').replaceChildren(...rows)

    const shown = $('#list').children.length
    $('#summary').textContent =
      result.total === 0 ? '' : `${number.format(result.total)}${result.totalCapped ? '+' : ''} ${result.total === 1 ? 'release' : 'releases'}` +
        (shown < result.total ? ` · showing ${number.format(shown)}` : '')
    $('#empty').hidden = result.total !== 0
    $('#more').hidden = shown >= result.total && !result.totalCapped
  } catch (error) {
    if (id !== requestId) return
    $('#list').classList.remove('stale')
    $('.results').removeAttribute('aria-busy')
    $('#summary').textContent = `Search failed: ${error.message}`
  }
}

let debounce
function scheduleSearch() {
  clearTimeout(debounce)
  debounce = setTimeout(() => {
    writeUrl()
    loadResults()
  }, 220)
}

// --- Detail drawer ---------------------------------------------------------

function fact(label, value) {
  if (value === null || value === undefined || value === '') return null
  return el('div', {}, el('dt', { text: label }), el('dd', {}, value))
}

function section(title, note, content) {
  return [el('h3', { class: 'section-title' }, title, note ? el('small', { text: note }) : null), content]
}

/**
 * Spotnet descriptions use UBB markup: [br], [b], [i], [u], [url=…]…[/url].
 * Built as DOM nodes (never HTML); links only when they are http(s).
 */
function ubb(text) {
  const root = el('div', { class: 'spot-description' })
  const stack = [root]
  const top = () => stack[stack.length - 1]
  // Old (key 1) spots use HTML instead: line breaks and simple tags become UBB, the rest is dropped.
  text = text
    .replace(/<br\s*\/?>/gi, '[br]')
    .replace(/<(\/?)(b|i|u)>/gi, '[$1$2]')
    .replace(/<\/?[a-z][^>]*>/gi, '')
  for (const token of text.split(/(\[\/?[a-z]+(?:=[^\]]*)?\])/i)) {
    const tag = /^\[(\/?)([a-z]+)(?:=([^\]]*))?\]$/i.exec(token)
    if (!tag) {
      if (token) top().append(token)
      continue
    }
    const [, closing, rawName, arg] = tag
    const name = rawName.toLowerCase()
    if (name === 'br') {
      top().append(el('br'))
    } else if (!closing && ['b', 'i', 'u', 'url'].includes(name)) {
      const url = name === 'url' ? (arg ?? '').trim() : ''
      const node = name === 'url'
        ? /^https?:\/\//i.test(url) ? el('a', { href: url, target: '_blank', rel: 'noopener noreferrer nofollow' }) : el('span')
        : el({ b: 'strong', i: 'em', u: 'u' }[name])
      node.dataset.ubb = name
      top().append(node)
      stack.push(node)
    } else if (closing && stack.length > 1 && top().dataset.ubb === name) {
      stack.pop()
    }
    // Other tags ([img], [color=…], …) are dropped.
  }
  return root
}

function spotSection(release) {
  const spot = release.spot
  if (!spot) return []
  const image = spot.image?.url
    ? el('img', {
        class: 'spot-image',
        src: spot.image.url,
        alt: '',
        loading: 'lazy',
        width: spot.image.width ?? undefined,
        height: spot.image.height ?? undefined,
        onerror: (event) => event.target.remove(),
      })
    : null
  const categories = spot.categories?.head
    ? el('dl', { class: 'facts' },
        fact('Category', spot.categories.head),
        ...spot.categories.subcats.map((sub) => fact(sub.type, sub.values.join(', '))))
    : null
  const links = el('div', { class: 'spot-links' },
    spot.website && /^https?:\/\//i.test(spot.website)
      ? el('a', { href: spot.website, target: '_blank', rel: 'noopener noreferrer nofollow', text: spot.website })
      : null,
    spot.image?.external
      ? el('a', { href: spot.image.external, target: '_blank', rel: 'noopener noreferrer nofollow', text: 'Image (external site)' })
      : null)
  const note = [spot.poster && `by ${spot.poster}`, spot.tag && spot.tag !== spot.poster && `tag ${spot.tag}`].filter(Boolean).join(' · ')
  return section('Spot', note, el('div', { class: 'spot' },
    image,
    el('div', { class: 'spot-body' },
      categories,
      spot.description ? ubb(spot.description) : el('p', { class: 'empty', text: 'No description.' }),
      links.childElementCount ? links : null)))
}

function renderDetail(release) {
  $('#detail-name').textContent = release.name
  $('#detail-source').textContent = `Recognised by ${SOURCE_LABELS[release.source] ?? release.source ?? 'filename'}`

  const actions = el(
    'div',
    { class: 'actions' },
    el('a', { class: 'button button-primary', href: `/api/v1/releases/${release.id}/nzb` },
      el('span', { text: 'Download NZB' })),
    el('button', {
      class: 'button',
      type: 'button',
      text: 'Copy name',
      onclick: async (event) => {
        await navigator.clipboard?.writeText(release.name)
        event.target.textContent = 'Copied'
        setTimeout(() => (event.target.textContent = 'Copy name'), 1200)
      },
    })
  )

  const poster = release.poster
    ? el('button', {
        class: 'link-button',
        type: 'button',
        text: release.poster.replace(/\s*<.*>$/, ''),
        title: 'Show releases by this poster',
        onclick: () => {
          state.poster = release.poster
          closeRelease()
          syncControls()
          writeUrl(true)
          loadResults()
        },
      })
    : 'Random per article (obfuscated)'

  const listedPresent = release.par2Files.filter((f) => f.present).length
  const facts = el(
    'dl',
    { class: 'facts' },
    fact('Size', formatSize(release.size)),
    fact('Files', release.fileTotal ? `${release.fileCount} of ${release.fileTotal}` : `${release.fileCount} posted`),
    fact('par2 lists', release.par2Files.length ? `${listedPresent} of ${release.par2Files.length} present` : null),
    fact('Posted', release.postedAt ? `${new Date(release.postedAt).toLocaleString()} (${formatAge(release.postedAt)})` : null),
    fact('Poster', poster),
    fact('Groups', release.groups?.join(', '))
  )

  const notices = []
  if (release.container === 'nzbe') {
    notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted NZB container ("nzbe"): an encrypted download index for others, unreadable without its key.' }))
  } else if (release.container === 'yencbin') {
    notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted .bin container (from the yEncBin uploader): the data is encrypted with a key that is not part of the post.' }))
  } else if (release.container === 'rclone') {
    notices.push(el('div', { class: 'notice notice-bad', text: 'rclone crypt: file names and contents are encrypted with a password and salt that are not part of the post.' }))
  } else if (release.passwordState === 'missing') {
    notices.push(el('div', { class: 'notice notice-bad', text: 'Password needed: the archive is password-protected, and neither the post nor its spot says the password.' }))
  }
  if (release.password) {
    notices.push(el('div', { class: 'notice notice-good' },
      el('div', {}, 'Password ', el('code', { class: 'password', text: release.password }),
        release.passwordSource === 'spot' ? ' (from the spot\'s description)' : '',
        ` is included in the NZB; SABnzbd and NZBGet use it to unpack.${release.encrypted == null ? ' Whether the archive needs it is not known yet.' : ''}`)))
  }
  if (release.listingOnly) {
    notices.push(el('div', { class: 'notice' },
      `A Spotnet spot${release.spot?.poster ? ` by ${release.spot.poster}` : ''}. Its files are read from the spot's NZB on Usenet when you download it; that takes a few seconds the first time.`))
  }
  if (release.pending.binaries > 0) {
    const pct = release.pending.partsTotal ? Math.round((100 * release.pending.partsHave) / release.pending.partsTotal) : 0
    notices.push(
      el('div', { class: 'notice notice-warn' },
        el('div', { style: 'flex:1' },
          `${release.pending.binaries} more ${release.pending.binaries === 1 ? 'file is' : 'files are'} still arriving (${pct}% of their parts seen).`,
          el('div', { class: 'progress' }, el('span', { style: `width:${pct}%` }))))
    )
  }

  const files = el('table', { class: 'file-table' },
    el('tbody', {}, release.files.map((f) =>
      el('tr', {},
        el('td', { class: 'name' }, f.name,
          f.realName && f.realName !== f.postedName ? el('span', { class: 'posted', text: `posted as ${f.postedName}` }) : null),
        el('td', { class: 'num', text: formatSize(f.size) }),
        el('td', { class: 'num', text: `${f.parts} ${f.parts === 1 ? 'part' : 'parts'}` })))))

  const par2 = release.par2Files.length
    ? el('table', { class: 'file-table' },
        el('tbody', {}, release.par2Files.map((f) =>
          el('tr', {},
            el('td', { class: `state ${f.present ? 'ok' : 'missing'}`, text: f.present ? '✓' : '·', title: f.present ? 'Present' : 'Not seen yet' }),
            el('td', { class: 'name', text: f.name }),
            el('td', { class: 'num', text: formatSize(f.size) })))))
    : null

  const contents = release.contents.length
    ? el('table', { class: 'file-table' },
        el('tbody', {}, release.contents.map((entry) =>
          el('tr', {},
            el('td', { class: 'name', text: entry.name + (entry.directory ? '/' : '') }),
            el('td', { class: 'num', text: entry.size === null ? '' : formatSize(entry.size) }),
            el('td', { class: 'state', text: entry.encrypted ? '🔒' : '', title: entry.encrypted ? 'Encrypted' : undefined })))))
    : null

  $('#detail').replaceChildren(
    actions,
    ...notices,
    facts,
    ...spotSection(release),
    ...section('Files', `${release.files.length} posted`, files),
    ...(par2 ? section('par2 file list', `${listedPresent} of ${release.par2Files.length} present`, par2) : []),
    ...(contents ? section('Archive contents', `${release.contents.length} entries`, contents) : [])
  )
}

async function openRelease(id, push = false) {
  state.release = id
  writeUrl(push)
  document.querySelectorAll('.release.selected').forEach((r) => r.classList.remove('selected'))
  document.querySelector(`.release[data-id="${id}"]`)?.classList.add('selected')

  $('#drawer').hidden = false
  $('#scrim').hidden = false
  $('#detail-name').textContent = ''
  $('#detail-source').textContent = ''
  $('#detail').replaceChildren(...Array.from({ length: 6 }, () => el('div', { class: 'skeleton' })))
  $('#close').focus()
  try {
    renderDetail(await api(`/api/v1/releases/${id}`))
  } catch (error) {
    $('#detail').replaceChildren(el('p', { class: 'empty', text: `Could not load this release (${error.message}).` }))
  }
}

function closeRelease(push = false) {
  if (!state.release) return
  const id = state.release
  state.release = null
  $('#drawer').hidden = true
  $('#scrim').hidden = true
  writeUrl(push)
  document.querySelector(`.release[data-id="${id}"] .release-main`)?.focus()
}

// --- Wiring ----------------------------------------------------------------

$('#q').addEventListener('input', (event) => {
  state.q = event.target.value.trim()
  scheduleSearch()
})
$('#sort').addEventListener('change', (event) => {
  state.sort = event.target.value
  writeUrl()
  loadResults()
})
$('#source').addEventListener('change', (event) => {
  state.source = event.target.value
  writeUrl()
  loadResults()
})
for (const button of document.querySelectorAll('[data-filter]')) {
  button.addEventListener('click', () => {
    const f = button.dataset.filter
    state[f] = !state[f]
    button.setAttribute('aria-pressed', String(state[f]))
    writeUrl()
    loadResults()
  })
}
$('#poster-clear').addEventListener('click', () => {
  state.poster = ''
  syncControls()
  writeUrl()
  loadResults()
})
$('#more').addEventListener('click', () => {
  state.page++
  loadResults({ append: true })
})
$('#close').addEventListener('click', () => closeRelease(true))
$('#scrim').addEventListener('click', () => closeRelease(true))

document.addEventListener('keydown', (event) => {
  if (event.key === 'Escape') closeRelease(true)
  if (event.key === '/' && document.activeElement !== $('#q')) {
    event.preventDefault()
    $('#q').focus()
    $('#q').select()
  }
})

window.addEventListener('popstate', () => {
  const previous = { ...state }
  readUrl()
  syncControls()
  const searchChanged = ['q', 'sort', 'source', 'poster', ...FILTERS].some((k) => previous[k] !== state[k])
  if (searchChanged) loadResults()
  if (state.release && state.release !== previous.release) openRelease(state.release)
  if (!state.release && previous.release) {
    $('#drawer').hidden = true
    $('#scrim').hidden = true
  }
})

readUrl()
syncControls()
loadStats()
loadResults()
if (state.release) openRelease(state.release)