Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
129 lines
5.6 KiB
TypeScript
129 lines
5.6 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import { looksRandom } from '#services/ArchiveService'
|
|
|
|
/**
|
|
* A release that is nothing more than one randomly named file (its own
|
|
* filename stem). Such a file moves into the par2 set that names it exactly;
|
|
* real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
|
|
*/
|
|
export const WEAK_RELEASE_SQL = (column: string) =>
|
|
`EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
|
|
AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
|
|
|
|
/**
|
|
* Whether `candidate` (e.g. from a par2 listing) should replace `current`:
|
|
* the current name is random, or a single token without separators
|
|
* (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
|
|
* candidate reads like a release name.
|
|
*/
|
|
export function isBetterName(current: string, candidate: string): boolean {
|
|
if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
|
|
return looksRandom(current) || !/[._ -]/.test(current)
|
|
}
|
|
|
|
/** Smaller files (nfo, sfv) collide too easily on size alone. */
|
|
const MIN_SIZE_ONLY = 64 * 1024
|
|
|
|
/**
|
|
* Gives binaries (and their files) the real filename that a par2 set lists
|
|
* for them. See CONCEPT.md (D21).
|
|
* - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
|
|
* also puts the binary in that par2 set's release.
|
|
* - Size only, when the fingerprint is unknown: within the binary's own
|
|
* release, only when exactly one listed file has that size.
|
|
*/
|
|
class NameService {
|
|
async resolve(binaryIds: number[]): Promise<number> {
|
|
if (binaryIds.length === 0) return 0
|
|
const exact = await db.rawQuery(
|
|
`UPDATE binaries b SET real_name = m.name,
|
|
release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
|
|
FROM (
|
|
SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
|
|
FROM binaries b2
|
|
JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
|
|
WHERE b2.id = ANY(?)
|
|
ORDER BY b2.id, rf.release_id DESC
|
|
) m
|
|
WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
|
|
OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
|
|
[binaryIds]
|
|
)
|
|
const bySize = await db.rawQuery(
|
|
`UPDATE binaries b SET real_name = m.name
|
|
FROM (
|
|
SELECT b2.id, min(rf.name) AS name
|
|
FROM binaries b2
|
|
JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
|
|
WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
|
GROUP BY b2.id
|
|
HAVING count(DISTINCT rf.name) = 1
|
|
) m
|
|
WHERE b.id = m.id`,
|
|
[binaryIds, MIN_SIZE_ONLY]
|
|
)
|
|
await db.rawQuery(
|
|
`UPDATE files f SET real_name = b.real_name,
|
|
release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
|
|
FROM binaries b
|
|
WHERE b.id = ANY(?) AND b.file_id = f.id
|
|
AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
|
|
OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
|
|
[binaryIds]
|
|
)
|
|
return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
|
|
}
|
|
|
|
/**
|
|
* When a par2 set arrives: the binaries it can name, i.e. those matching
|
|
* one of its entries exactly (size + first-16-KiB MD5) and those in its
|
|
* release. Selecting every binary of the listed sizes instead picked up
|
|
* hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
|
|
*/
|
|
async resolveForRelease(releaseId: number): Promise<number> {
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT b.id FROM release_files rf
|
|
JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
|
|
WHERE rf.release_id = ?
|
|
UNION
|
|
SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
|
|
[releaseId, releaseId]
|
|
)
|
|
return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Repair for data from before files moved into the par2 set that names them
|
|
* (used by `release:rename`): regroups exact matches out of random-named
|
|
* single-file releases, then renames par2 releases that still carry a
|
|
* random name after their listing. No downloads.
|
|
*/
|
|
export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT DISTINCT b.id FROM binaries b
|
|
JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
|
|
WHERE b.release_id IS DISTINCT FROM rf.release_id
|
|
AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
|
|
)
|
|
const ids = rows.map((r: { id: string }) => Number(r.id))
|
|
for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
|
|
|
|
const { par2ReleaseName } = await import('#lib/release/index')
|
|
const listings = await db.rawQuery(
|
|
`SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
|
|
JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
|
|
)
|
|
let renamed = 0
|
|
for (const release of listings.rows) {
|
|
const better = par2ReleaseName(release.names)
|
|
if (!better || better === release.name || !isBetterName(release.name, better)) continue
|
|
await db.from('releases').where('id', release.id).update({ name: better })
|
|
renamed++
|
|
}
|
|
return { regrouped: ids.length, renamed }
|
|
}
|
|
|
|
const nameService = new NameService()
|
|
export default nameService
|