Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
47 lines
2.3 KiB
TypeScript
47 lines
2.3 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { randomBytes } from 'node:crypto'
|
|
import { zstdCompressSync, constants } from 'node:zlib'
|
|
import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
|
|
|
|
function tarEntry(name: string, data: Buffer): Buffer {
|
|
const header = Buffer.alloc(512)
|
|
header.write(name, 0, 'utf8')
|
|
header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
|
|
header.write('ustar\0', 257, 'latin1')
|
|
return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
|
|
}
|
|
|
|
test.group('nicovideo captures', () => {
|
|
test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
|
|
const dir = '1790720489_nicovideo_jp_watch_sm27965722'
|
|
const response = {
|
|
data: {
|
|
metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
|
|
response: {
|
|
video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
|
|
owner: { nickname: 'まふまふ' },
|
|
tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
|
|
},
|
|
},
|
|
}
|
|
const tar = Buffer.concat([
|
|
tarEntry(`${dir}/`, Buffer.alloc(0)),
|
|
tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
|
|
tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
|
|
])
|
|
const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
|
|
// Only the first part of the stream, as in the file's first article.
|
|
const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
|
|
assert.deepEqual(info, {
|
|
id: 'sm27965722',
|
|
title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】',
|
|
owner: 'まふまふ',
|
|
registeredAt: '2016-01-08T20:00:00+09:00',
|
|
tags: ['音楽', 'After_the_Rain'],
|
|
})
|
|
assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
|
|
assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
|
|
assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
|
|
})
|
|
})
|