diff --git a/app/services/SpotService.ts b/app/services/SpotService.ts index 840443f..f8a1bea 100644 --- a/app/services/SpotService.ts +++ b/app/services/SpotService.ts @@ -297,6 +297,90 @@ class SpotService { return read } + /** + * Takes up to `limit` listed spots whose article and NZB haven't been read, + * newest first, marking them so another reader skips them. + */ + async claimListings(limit: number): Promise { + const { rows } = await db.rawQuery( + `UPDATE spots SET nzb_read_at = now() + WHERE message_id IN ( + SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL + ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED) + RETURNING *`, + [limit] + ) + return rows + } + + /** + * Reads a listed spot's article and NZB without importing its files + * (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's + * own password (`password_source = 'nzb'`, over one from the spot's text), + * else one in the spot's text; a placeholder NZB hides the spot (`dummy`). + * Returns what it found. A connection failure gives the spot back for later. + */ + async readListing(spot: any): Promise { + try { + const head = await this.readHead(spot.message_id) + if (!head) { + await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() }) + return 'unverified' + } + const update: Record = this.details(head.xml) + let outcome = 'nothing' + let nzbXml: string | null = null + try { + nzbXml = await this.readNzb(head) + } catch (error: any) { + if (!/^4\d\d/.test(String(error.message))) throw error + // The NZB's articles are gone: the listing can't be downloaded any more. + update.error = 'NZB articles gone' + outcome = 'nzb-gone' + } + const nzb = nzbXml ? parseNzb(nzbXml) : null + if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) { + update.status = 'dummy' + outcome = 'dummy' + } else if (nzb?.meta.password && spot.release_id) { + await db + .from('releases') + .where('id', spot.release_id) + .where((q) => q.whereNull('password').orWhere('password_source', 'spot')) + .update({ password: nzb.meta.password, password_source: 'nzb' }) + outcome = 'nzb-password' + } + await db.from('spots').where('message_id', spot.message_id).update(update) + if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password' + return outcome + } catch (error: any) { + // The spot's own article is gone (430): nothing to read later either. + if (/^4\d\d/.test(String(error.message))) { + await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() }) + return 'gone' + } + await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null }) + logger.warn(`Spot ${spot.message_id} listing: ${error.message}`) + return 'retry' + } + } + + /** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */ + async readListings(limit: number, concurrency: number): Promise> { + const spots = await this.claimListings(limit) + const counts: Record = {} + let next = 0 + await Promise.all( + Array.from({ length: Math.max(1, concurrency) }, async () => { + while (next < spots.length) { + const outcome = await this.readListing(spots[next++]) + counts[outcome] = (counts[outcome] ?? 0) + 1 + } + }) + ) + return counts + } + /** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */ async image(spot: any): Promise { const segments: string[] = spot.image?.segments ?? [] @@ -315,13 +399,19 @@ class SpotService { return { header, xml: parseSpotXml(decodeSpotText(xml)) } } - /** Downloads the spot's compressed NZB and imports it as a release (D26). */ - private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise> { + /** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */ + private async readNzb(head: { xml: SpotXml | null }): Promise { const segments = head.xml?.nzbSegments ?? [] - if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' } + if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null const bodies = [] for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`)) - const nzbXml = decodeSpotBinary(bodies).toString('utf8') + return decodeSpotBinary(bodies).toString('utf8') + } + + /** Downloads the spot's compressed NZB and imports it as a release (D26). */ + private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise> { + const nzbXml = await this.readNzb(head) + if (!nzbXml) return { status: 'no-nzb' } const nzb = parseNzb(nzbXml) if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' } const { header } = head diff --git a/commands/SpotPostprocess.ts b/commands/SpotPostprocess.ts new file mode 100644 index 0000000..790b4b2 --- /dev/null +++ b/commands/SpotPostprocess.ts @@ -0,0 +1,55 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' +import env from '#start/env' +import Shutdown from '#services/Shutdown' +import NntpService from '#services/NntpService' +import SpotService from '#services/SpotService' + +const BATCH = 500 + +/** + * Reads the article and NZB of listed spots that were listed from their + * overview only (LOG.md 2026-10-08 (10)): description, the NZB's password or + * one in the spot's text, placeholder NZBs. Newest first; can be stopped and + * started again (worker:spots does the same when idle, a batch at a time). + * Connections: NNTP_POOL_SIZE of this process, --concurrency at a time. + */ +export default class SpotPostprocess extends BaseCommand { + public static commandName = 'spot:postprocess' + public static description = "Reads listed spots' articles and NZBs for their passwords and details." + + public static options: CommandOptions = { + startApp: true, + } + + @flags.number({ description: 'Spots read at a time (default: NNTP_POOL_SIZE)' }) + declare concurrency?: number + + @flags.number({ description: 'Stop after this many spots' }) + declare limit?: number + + public async run() { + const concurrency = this.concurrency ?? env.get('NNTP_POOL_SIZE', 4) + const [{ count }] = await db.from('spots').where('status', 'listed').whereNull('nzb_read_at').count('* as count') + this.logger.info(`${count} listed spots to read, ${concurrency} at a time.`) + const totals: Record = {} + const started = Date.now() + let done = 0 + try { + while (!Shutdown.requested && (!this.limit || done < this.limit)) { + const counts = await SpotService.readListings(Math.min(BATCH, this.limit ? this.limit - done : BATCH), concurrency) + const read = Object.values(counts).reduce((sum, n) => sum + n, 0) + if (read === 0) break + for (const [k, v] of Object.entries(counts)) totals[k] = (totals[k] ?? 0) + v + done += read + const rate = done / ((Date.now() - started) / 1000) + this.logger.info(`${done} read (${rate.toFixed(1)}/s): ${Object.entries(totals).map(([k, v]) => `${v} ${k}`).join(', ')}.`) + // Usenet unreachable: everything was given back; don't spin on it. + if (counts.retry === read) await Shutdown.sleep(30_000) + } + } finally { + await NntpService.shutdown() + } + } +} diff --git a/commands/workers/Spots.ts b/commands/workers/Spots.ts index 7738d2c..fb0e66d 100644 --- a/commands/workers/Spots.ts +++ b/commands/workers/Spots.ts @@ -10,10 +10,13 @@ const IDLE_MS = 30_000 const BATCH = 500 /** Imported spots whose details are read per idle round. */ const DETAILS_BATCH = 50 +/** Listed spots whose article and NZB are read per idle round (passwords, dummies). */ +const LISTINGS_BATCH = 50 /** * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one, - * downloads its compressed NZB (a few KB to a few MB) and imports it. + * downloads its compressed NZB (a few KB to a few MB) and imports it. When + * idle, reads what listed spots were listed without: details, passwords. */ export default class SpotsWorker extends BaseCommand { public static commandName = 'worker:spots' @@ -37,8 +40,12 @@ export default class SpotsWorker extends BaseCommand { const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity)) if (batch.length === 0) { if (this.limit) break - // Idle: read the details of imported spots that don't have them yet (newest first). - if ((await SpotService.fillDetails(DETAILS_BATCH)) === 0) await Shutdown.sleep(IDLE_MS) + // Idle: read the details of imported spots that don't have them yet (newest first), then + // the article and NZB of listed spots (their passwords; LOG.md 2026-10-08 (10)). + if ((await SpotService.fillDetails(DETAILS_BATCH)) > 0) continue + const listings = await SpotService.readListings(LISTINGS_BATCH, concurrency) + const read = Object.values(listings).reduce((sum, n) => sum + n, 0) + if (read === 0 || listings.retry === read) await Shutdown.sleep(IDLE_MS) continue } // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read. diff --git a/database/migrations/1768620765280_spot_listing_reads.ts b/database/migrations/1768620765280_spot_listing_reads.ts new file mode 100644 index 0000000..4e1c3fd --- /dev/null +++ b/database/migrations/1768620765280_spot_listing_reads.ts @@ -0,0 +1,23 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Listed spots (D29) were only read from their overview: no description, no + * NZB, so neither a password in the spot's text nor the NZB's own + * `` was known (LOG.md 2026-10-08 (10)). nzb_read_at: + * when the spot's article and NZB were read for that (SpotService.readListing), + * also set while a reader holds it. The index finds the ones still to read, + * newest first. + */ +export default class extends BaseSchema { + async up() { + await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS nzb_read_at timestamptz`) + await this.db.rawQuery( + `CREATE INDEX IF NOT EXISTS spots_listing_unread_idx ON spots (posted_at DESC) WHERE status = 'listed' AND nzb_read_at IS NULL` + ) + } + + async down() { + await this.db.rawQuery(`DROP INDEX IF EXISTS spots_listing_unread_idx`) + await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS nzb_read_at`) + } +} diff --git a/database/schema.ts b/database/schema.ts index ec5c874..703b00c 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -601,7 +601,7 @@ export class SignatureProbeSchema extends BaseModel { } export class SpotSchema extends BaseModel { - static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const + static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'nzbReadAt', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const $columns = SpotSchema.$columns @column() declare articleNumber: bigint | number | null @@ -624,6 +624,8 @@ export class SpotSchema extends BaseModel { @column({ isPrimary: true }) declare messageId: string @column.dateTime() + declare nzbReadAt: DateTime | null + @column.dateTime() declare postedAt: DateTime | null @column() declare poster: string diff --git a/lib/spotnet/password.ts b/lib/spotnet/password.ts index 3f2db89..4247254 100644 --- a/lib/spotnet/password.ts +++ b/lib/spotnet/password.ts @@ -3,14 +3,25 @@ * the NZB's ``: * `Title{{secret}}` (the newznab/SABnzbd convention), * `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`, - * `WW FC0401` (abbreviation without separator: only with a digit in it). + * a few words between keyword and separator (`Het wachtwoord voor de nzb is: dw0067`, + * `ww in nzb: X`, `Wachtwoord inbegrepen: wysiwyg`), + * the value on the next line (`WW:` / `tijdloos`), + * `wachtwoord is arn2`, `… voorzien van een wachtwoord. Deze is: X`, + * `WW FC0401` (abbreviation without separator: with a digit in it, or as the whole line). * The value must be the rest of its line and a single token, so sentences * such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or - * "met een password beschermd" don't count. + * "met een password beschermd" don't count (LOG.md 2026-10-08 (10)). */ -const KEYWORD = String.raw`wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw` -const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])(?:${KEYWORD})\s*(?:is\s*)?[:=]\s*(.*)$`, 'i') +const KEYWORD = String.raw`(?:wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw)(?![a-z])` +const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}(?:\s+[a-zà-ÿ]+){0,3}?\s*(?:is\s*)?[:=]\s*(.*)$`, 'i') +const IS_VALUE = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}\s+is\s+(\S+)\s*$`, 'i') +/** "Deze is voorzien van een wachtwoord. Deze is: X": the value in the next sentence. */ +const NEXT_SENTENCE = /(?:^|[.!?]\s*)(?:deze|dit|het|die|this|it)\s+is\s*[:=]\s*(\S+)\s*$/i +const MENTIONS = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}`, 'i') const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i +/** "WW carrie" as a line of its own; "WW II" is the war. */ +const WHOLE_LINE_ABBREVIATION = /^(?:ww|pw)\s+(\S+)$/i +const NOT_PASSWORD_TOKEN = /^(?:i{1,3}|iv|v|[12])$/i const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i @@ -24,7 +35,11 @@ function lines(description: string): string[] { } function clean(value: string): string | null { - const token = value.trim().replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '') + const token = value + .trim() + // "Wachtwoord : yakowyke !!" + .replace(/^[!*~#]+\s*|\s*[!*~#]+$/g, '') + .replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '') if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null return token } @@ -32,19 +47,37 @@ function clean(value: string): string | null { export function passwordFromSpot(title: string, description?: string | null): string | null { const braces = /\{\{([^{}]{1,64})\}\}/.exec(title) if (braces && clean(braces[1])) return clean(braces[1]) - for (const line of lines(description ?? '')) { + const all = lines(description ?? '') + for (const [i, line] of all.entries()) { if (!line || NO_PASSWORD.test(line)) continue const separated = WITH_SEPARATOR.exec(line) if (separated) { - const value = clean(separated[1]) + const value = separated[1].trim() ? clean(separated[1]) : nextLineValue(all, i) if (value) return value continue } - const abbreviated = ABBREVIATION.exec(line) - if (abbreviated && /\d/.test(abbreviated[1])) { + const is = IS_VALUE.exec(line) + if (is) { + const value = clean(is[1]) + if (value) return value + } + const sentence = MENTIONS.test(line) ? NEXT_SENTENCE.exec(line) : null + if (sentence) { + const value = clean(sentence[1]) + if (value) return value + } + const whole = WHOLE_LINE_ABBREVIATION.exec(line) + const abbreviated = whole ?? ABBREVIATION.exec(line) + if (abbreviated && (whole ? !NOT_PASSWORD_TOKEN.test(abbreviated[1]) : /\d/.test(abbreviated[1]))) { const value = clean(abbreviated[1]) if (value) return value } } return null } + +/** `WW:` with the value alone on the next non-empty line. */ +function nextLineValue(all: string[], i: number): string | null { + const next = all.slice(i + 1).find((line) => line !== '') + return next ? clean(next) : null +} diff --git a/tests/functional/spot_listing.spec.ts b/tests/functional/spot_listing.spec.ts index 339108a..3f9e9c1 100644 --- a/tests/functional/spot_listing.spec.ts +++ b/tests/functional/spot_listing.spec.ts @@ -107,3 +107,77 @@ test.group('SpotService passwords', (group) => { assert.equal(passwordState(null, false), 'none') }) }) + +test.group('SpotService reading listings', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + const nzb = (password: string | null, bytes = 700_000_000) => ` +${password ? `${password}` : ''} +alt.binaries.test +a1@x` + + /** A listed spot, as worker:spots leaves an old one, with its article and NZB as given. */ + async function listedSpot(id: string, read: { description?: string; nzb?: string | null; error?: string }) { + await db.table('spots').insert({ + message_id: id, group_name: 'free.pt', article_number: 1, title: `Listed ${id}`, poster: 'Bart', key_id: 7, size: 700_000_000, + posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: new Date(), status: 'pending', + }) + const [claimed] = await SpotService.claim(10) + await SpotService.list([claimed]) + const service = SpotService as any + service.readHead = async () => { + if (read.error) throw new Error(read.error) + return { header: { size: 700_000_000 }, xml: { description: read.description ?? '', nzbSegments: ['n@x'] } } + } + service.readNzb = async () => read.nzb ?? null + } + + const restore = () => { + delete (SpotService as any).readHead + delete (SpotService as any).readNzb + } + group.each.teardown(restore) + + const passwordOf = async (id: string) => { + const spot = await db.from('spots').where('message_id', id).first() + const release = await db.from('releases').where('id', spot.release_id).first() + return { spot, password: release.password, source: release.password_source } + } + + test("reads the NZB's password and the details, without importing files", async ({ assert }) => { + await listedSpot('l1@spot.net', { description: 'Plot', nzb: nzb('fromnzb1') }) + assert.deepEqual(await SpotService.readListings(10, 2), { 'nzb-password': 1 }) + const { spot, password, source } = await passwordOf('l1@spot.net') + assert.deepEqual([password, source, spot.status, spot.description], ['fromnzb1', 'nzb', 'listed', 'Plot']) + assert.isNotNull(spot.nzb_read_at) + assert.equal((await db.from('releases').where('id', spot.release_id).first()).file_count, 0) + // Read once: nothing left. + assert.deepEqual(await SpotService.readListings(10, 2), {}) + }) + + test("the spot's text when the NZB has none; the NZB's own password wins over it", async ({ assert }) => { + await listedSpot('l2@spot.net', { description: 'Wachtwoord: fromtext2', nzb: nzb(null) }) + assert.deepEqual(await SpotService.readListings(10, 1), { 'spot-password': 1 }) + assert.include(await passwordOf('l2@spot.net'), { password: 'fromtext2', source: 'spot' }) + + await listedSpot('l3@spot.net', { description: 'Wachtwoord: fromtext3', nzb: nzb('fromnzb3') }) + await db.from('releases').where('id', (await db.from('spots').where('message_id', 'l3@spot.net').first()).release_id).update({ password: 'fromtext3', password_source: 'spot' }) + assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-password': 1 }) + assert.include(await passwordOf('l3@spot.net'), { password: 'fromnzb3', source: 'nzb' }) + }) + + test('a placeholder NZB hides the listing; a lost connection gives the spot back, a gone article does not', async ({ assert }) => { + await listedSpot('l4@spot.net', { nzb: nzb(null, 1500) }) + assert.deepEqual(await SpotService.readListings(10, 1), { dummy: 1 }) + assert.equal((await db.from('spots').where('message_id', 'l4@spot.net').first()).status, 'dummy') + + await listedSpot('l5@spot.net', { error: 'Connection reset' }) + assert.deepEqual(await SpotService.readListings(10, 1), { retry: 1 }) + assert.isNull((await db.from('spots').where('message_id', 'l5@spot.net').first()).nzb_read_at) + + await db.from('spots').where('message_id', 'l5@spot.net').delete() + await listedSpot('l6@spot.net', { error: '430 No Such Article' }) + assert.deepEqual(await SpotService.readListings(10, 1), { gone: 1 }) + assert.isNotNull((await db.from('spots').where('message_id', 'l6@spot.net').first()).nzb_read_at) + }) +}) diff --git a/tests/unit/spotnet.spec.ts b/tests/unit/spotnet.spec.ts index 5a32571..5ba991f 100644 --- a/tests/unit/spotnet.spec.ts +++ b/tests/unit/spotnet.spec.ts @@ -146,6 +146,19 @@ test.group('spotnet / passwords in descriptions', () => { assert.equal(passwordFromSpot('T', '
Password = "abc123"
'), 'abc123') }) + test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => { + const { passwordFromSpot } = await import('#lib/spotnet/index') + // LOG.md 2026-10-08 (10) + assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg') + assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren') + assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD') + assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022') + assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg') + assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2') + assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos') + assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie') + }) + test('ignores sentences about passwords', async ({ assert }) => { const { passwordFromSpot } = await import('#lib/spotnet/index') for (const description of [ @@ -158,6 +171,14 @@ test.group('spotnet / passwords in descriptions', () => { 'Wachtwoord: geen', 'WW: zie nzb', 'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!', + 'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen', + 'Bringing History to Life - Greatest Battles of WW II, Sept 2026', + 'WW II', + 'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF', + 'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.', + 'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk', + 'Het ww staat op het plaatje, type het EXACT zo in.', + 'Wachtwoord:[br]Deze download heeft er geen', ]) { assert.isNull(passwordFromSpot('T', description), description) }