85 lines
2.7 KiB
JavaScript
85 lines
2.7 KiB
JavaScript
import 'dotenv/config';
|
|
import { getDb } from '../lib/database.js';
|
|
import { NntpPool } from '../lib/NntpPool.js';
|
|
import { YencFile } from '../lib/YencFile.js';
|
|
import fs from 'fs/promises';
|
|
import log4js from '../lib/logger.js';
|
|
|
|
const logger = log4js.getLogger('download');
|
|
|
|
async function downloadAndProcessPart(pool, partNumber, segment, yencFile) {
|
|
let conn;
|
|
try {
|
|
conn = await pool.acquire();
|
|
await conn.group('alt.binaries.test');
|
|
logger.debug(`Downloading part ${partNumber} with message ID: ${segment.id}`);
|
|
const bodyBuffer = (await conn.body(`<${segment.id}>`)).data;
|
|
yencFile.processPart(bodyBuffer);
|
|
} catch (error) {
|
|
if (error.code === 430) {
|
|
logger.error(`Article not found for part ${partNumber} (Message ID: ${segment.id})`);
|
|
} else {
|
|
throw error;
|
|
}
|
|
} finally {
|
|
if (conn) {
|
|
pool.release(conn);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function downloadFile(fileId, numConnections) {
|
|
const pool = new NntpPool(numConnections);
|
|
const yencFile = new YencFile();
|
|
|
|
const db = await getDb();
|
|
const file = await db.get('SELECT * FROM files WHERE id = ?', fileId);
|
|
|
|
if (!file) {
|
|
logger.error(`File with ID ${fileId} not found.`);
|
|
await pool.shutdown();
|
|
return;
|
|
}
|
|
|
|
logger.info(`Downloading file: ${file.filename} with ${numConnections} connections.`);
|
|
|
|
const messageIds = JSON.parse(file.message_ids);
|
|
const sortedParts = Object.entries(messageIds).sort(([a], [b]) => parseInt(a, 10) - parseInt(b, 10));
|
|
|
|
// Sequentially process the first part to initialize the YencFile
|
|
const [firstPartNumber, firstSegment] = sortedParts[0];
|
|
await downloadAndProcessPart(pool, firstPartNumber, firstSegment, yencFile);
|
|
|
|
// Concurrently process the rest of the parts
|
|
const remainingParts = sortedParts.slice(1);
|
|
const downloadPromises = remainingParts.map(([partNumber, segment]) =>
|
|
downloadAndProcessPart(pool, partNumber, segment, yencFile)
|
|
);
|
|
|
|
await Promise.all(downloadPromises);
|
|
|
|
const completeFile = yencFile.getBuffer();
|
|
if (completeFile) {
|
|
await fs.writeFile(file.filename, completeFile);
|
|
logger.info(`File "${file.filename}" downloaded successfully.`);
|
|
} else {
|
|
logger.error('Could not assemble the final file.');
|
|
}
|
|
|
|
await pool.shutdown();
|
|
}
|
|
|
|
const args = process.argv.slice(2);
|
|
const fileIdArg = args.find(arg => !arg.startsWith('--'));
|
|
const connectionsArg = args.find(arg => arg.startsWith('--connections='));
|
|
|
|
const fileId = fileIdArg ? parseInt(fileIdArg, 10) : null;
|
|
const numConnections = connectionsArg ? parseInt(connectionsArg.split('=')[1], 10) : 10;
|
|
|
|
if (!fileId || isNaN(fileId)) {
|
|
logger.error('Please provide a valid file ID as a command-line argument.');
|
|
process.exit(1);
|
|
}
|
|
|
|
downloadFile(fileId, numConnections);
|