Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
@@ -0,0 +1,333 @@
|
||||
/**
|
||||
* Spotnet's category tree, from Spotweb's SpotCategories (English names).
|
||||
* A spot has a head category (0–3) and subcategories per type letter:
|
||||
* `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
|
||||
* `d` genre, `z` kind (movie, series, book, …).
|
||||
*/
|
||||
export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
|
||||
|
||||
/** What each type letter means per head category. */
|
||||
export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
|
||||
{ a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Genre', z: 'Type' },
|
||||
]
|
||||
|
||||
const NAMES: Record<string, Record<number, string>>[] = [
|
||||
{
|
||||
a: {
|
||||
0: 'DivX',
|
||||
1: 'WMV',
|
||||
2: 'MPG',
|
||||
3: 'DVD5',
|
||||
4: 'HD other',
|
||||
5: 'ePub',
|
||||
6: 'Blu-ray',
|
||||
7: 'HD-DVD',
|
||||
8: 'WMVHD',
|
||||
9: 'x264',
|
||||
10: 'DVD9',
|
||||
11: 'PDF',
|
||||
12: 'Bitmap',
|
||||
13: 'Vector',
|
||||
14: '3D',
|
||||
15: 'UHD',
|
||||
},
|
||||
b: {
|
||||
0: 'CAM',
|
||||
1: '(S)VCD',
|
||||
2: 'Promo',
|
||||
3: 'Retail',
|
||||
4: 'TV',
|
||||
6: 'Satellite',
|
||||
7: 'R5',
|
||||
8: 'Telecine',
|
||||
9: 'Telesync',
|
||||
10: 'Scan',
|
||||
11: 'WEB-DL',
|
||||
12: 'WEBRip',
|
||||
13: 'HDRip',
|
||||
},
|
||||
c: {
|
||||
0: 'No subtitles',
|
||||
1: 'Dutch subtitles (external)',
|
||||
2: 'Dutch subtitles (builtin)',
|
||||
3: 'English subtitles (external)',
|
||||
4: 'English subtitles (builtin)',
|
||||
6: 'Dutch subtitles (available)',
|
||||
7: 'English subtitles (available)',
|
||||
10: 'English audio/written',
|
||||
11: 'Dutch audio/written',
|
||||
12: 'German audio/written',
|
||||
13: 'French audio/written',
|
||||
14: 'Spanish audio/written',
|
||||
15: 'Asian audio/written',
|
||||
},
|
||||
d: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Animation',
|
||||
3: 'Cabaret',
|
||||
4: 'Comedy',
|
||||
5: 'Crime',
|
||||
6: 'Documentary',
|
||||
7: 'Drama',
|
||||
8: 'Family',
|
||||
9: 'Fantasy',
|
||||
10: 'Arthouse',
|
||||
11: 'Television',
|
||||
12: 'Horror',
|
||||
13: 'Music',
|
||||
14: 'Musical',
|
||||
15: 'Mystery',
|
||||
16: 'Romance',
|
||||
17: 'Science Fiction',
|
||||
18: 'Sport',
|
||||
19: 'Short movie',
|
||||
20: 'Thriller',
|
||||
21: 'War',
|
||||
22: 'Western',
|
||||
23: 'Erotica (hetero)',
|
||||
24: 'Erotica (gay male)',
|
||||
25: 'Erotica (gay female)',
|
||||
26: 'Erotica (bi)',
|
||||
28: 'Asian',
|
||||
29: 'Anime',
|
||||
30: 'Cover',
|
||||
31: 'Comicbook',
|
||||
32: 'Cartoons',
|
||||
33: 'Youth',
|
||||
34: 'Business',
|
||||
35: 'Computer',
|
||||
36: 'Hobby',
|
||||
37: 'Cooking',
|
||||
38: 'Handwork',
|
||||
39: 'Craftwork',
|
||||
40: 'Health',
|
||||
41: 'History',
|
||||
42: 'Psychology',
|
||||
43: 'Newspaper',
|
||||
44: 'Magazine',
|
||||
45: 'Science',
|
||||
46: 'Female',
|
||||
47: 'Religion',
|
||||
48: 'Roman',
|
||||
49: 'Biography',
|
||||
50: 'Detective',
|
||||
51: 'Animals',
|
||||
52: 'Humor',
|
||||
53: 'Travel',
|
||||
54: 'True story',
|
||||
55: 'Non-fiction',
|
||||
56: 'Politics',
|
||||
57: 'Poetry',
|
||||
58: 'Fairy tale',
|
||||
59: 'Technical',
|
||||
60: 'Art',
|
||||
72: 'Bi',
|
||||
73: 'Lesbian',
|
||||
74: 'Homo',
|
||||
75: 'Hetero',
|
||||
76: 'Amature',
|
||||
77: 'Group',
|
||||
78: 'POV',
|
||||
79: 'Solo',
|
||||
80: 'Young',
|
||||
81: 'Soft',
|
||||
82: 'Fetish',
|
||||
83: 'Old',
|
||||
84: 'Fat',
|
||||
85: 'SM',
|
||||
86: 'Rough',
|
||||
87: 'Dark',
|
||||
88: 'Hentai',
|
||||
89: 'Outside',
|
||||
},
|
||||
z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'MP3',
|
||||
1: 'WMA',
|
||||
2: 'WAV',
|
||||
3: 'OGG',
|
||||
4: 'EAC',
|
||||
5: 'DTS',
|
||||
6: 'AAC',
|
||||
7: 'APE',
|
||||
8: 'FLAC',
|
||||
},
|
||||
b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
|
||||
c: {
|
||||
0: 'Variable',
|
||||
1: '< 96kbit',
|
||||
2: '96kbit',
|
||||
3: '128kbit',
|
||||
4: '160kbit',
|
||||
5: '192kbit',
|
||||
6: '256kbit',
|
||||
7: '320kbit',
|
||||
8: 'Lossless',
|
||||
9: 'Other',
|
||||
},
|
||||
d: {
|
||||
0: 'Blues',
|
||||
1: 'Compilation',
|
||||
2: 'Cabaret',
|
||||
3: 'Dance',
|
||||
4: 'Diverse',
|
||||
5: 'Hardcore',
|
||||
6: 'World',
|
||||
7: 'Jazz',
|
||||
8: 'Youth',
|
||||
9: 'Classical',
|
||||
10: 'Kleinkunst',
|
||||
11: 'Dutch',
|
||||
12: 'New Age',
|
||||
13: 'Pop',
|
||||
14: 'RnB',
|
||||
15: 'Hiphop',
|
||||
16: 'Reggae',
|
||||
17: 'Religious',
|
||||
18: 'Rock',
|
||||
19: 'Soundtracks',
|
||||
20: 'Other',
|
||||
21: 'Hardstyle',
|
||||
22: 'Asian',
|
||||
23: 'Disco',
|
||||
24: 'Classics',
|
||||
25: 'Metal',
|
||||
26: 'Country',
|
||||
27: 'Dubstep',
|
||||
28: 'Nederhop',
|
||||
29: 'DnB',
|
||||
30: 'Electro',
|
||||
31: 'Folk',
|
||||
32: 'Soul',
|
||||
33: 'Trance',
|
||||
34: 'Balkan',
|
||||
35: 'Techno',
|
||||
36: 'Ambient',
|
||||
37: 'Latin',
|
||||
38: 'Live',
|
||||
},
|
||||
z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'Playstation',
|
||||
4: 'Playstation 2',
|
||||
5: 'PSP',
|
||||
6: 'Xbox',
|
||||
7: 'Xbox 360',
|
||||
8: 'Gameboy Advance',
|
||||
9: 'Gamecube',
|
||||
10: 'Nintendo DS',
|
||||
11: 'Nintento Wii',
|
||||
12: 'Playstation 3',
|
||||
13: 'Windows Phone',
|
||||
14: 'iOS',
|
||||
15: 'Android',
|
||||
16: 'Nintendo 3DS',
|
||||
17: 'Playstation 4',
|
||||
18: 'XBox 1',
|
||||
},
|
||||
b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
|
||||
c: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Strategy',
|
||||
3: 'Roleplaying',
|
||||
4: 'Simulation',
|
||||
5: 'Race',
|
||||
6: 'Flying',
|
||||
7: 'Shooter',
|
||||
8: 'Platform',
|
||||
9: 'Sport',
|
||||
10: 'Child/youth',
|
||||
11: 'Puzzle',
|
||||
12: 'Other',
|
||||
13: 'Boardgame',
|
||||
14: 'Cards',
|
||||
15: 'Education',
|
||||
16: 'Music',
|
||||
17: 'Family',
|
||||
},
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'OS/2',
|
||||
4: 'Windows Phone',
|
||||
5: 'Navigation systems',
|
||||
6: 'iOS',
|
||||
7: 'Android',
|
||||
},
|
||||
b: {
|
||||
0: 'Audio',
|
||||
1: 'Video',
|
||||
2: 'Graphics',
|
||||
3: 'CD/DVD Tools',
|
||||
4: 'Media players',
|
||||
5: 'Rippers & Encoders',
|
||||
6: 'Plugins',
|
||||
7: 'Database tools',
|
||||
8: 'Email software',
|
||||
9: 'Photo',
|
||||
10: 'Screensavers',
|
||||
11: 'Skin software',
|
||||
12: 'Drivers',
|
||||
13: 'Browsers',
|
||||
14: 'Download managers',
|
||||
15: 'Download',
|
||||
16: 'Usenet software',
|
||||
17: 'RSS Readers',
|
||||
18: 'FTP software',
|
||||
19: 'Firewalls',
|
||||
20: 'Antivirus software',
|
||||
21: 'Antispyware software',
|
||||
22: 'Optimization software',
|
||||
23: 'Security software',
|
||||
24: 'System software',
|
||||
25: 'Other',
|
||||
26: 'Educational',
|
||||
27: 'Office',
|
||||
28: 'Internet',
|
||||
29: 'Communication',
|
||||
30: 'Development',
|
||||
31: 'Spotnet',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
export interface SpotCategoryLabels {
|
||||
head: string | null
|
||||
/** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
|
||||
subcats: { type: string; values: string[] }[]
|
||||
}
|
||||
|
||||
/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
|
||||
export function spotCategoryLabels(
|
||||
category: number | null,
|
||||
subcats: Record<string, number[]> | null
|
||||
): SpotCategoryLabels {
|
||||
if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
|
||||
const labels: SpotCategoryLabels['subcats'] = []
|
||||
for (const letter of ['z', 'a', 'b', 'c', 'd']) {
|
||||
const values = [
|
||||
...new Set(
|
||||
(subcats?.[letter] ?? [])
|
||||
.map((n) => NAMES[category][letter]?.[n])
|
||||
.filter((v): v is string => Boolean(v))
|
||||
),
|
||||
]
|
||||
if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
|
||||
}
|
||||
return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
|
||||
}
|
||||
Reference in new issue
Block a user