make async database integrity check

This commit is contained in:
Zimeng Xiong
2025-11-22 21:59:18 -08:00
parent 888834c8f0
commit bb42187ba8
2 changed files with 45 additions and 16 deletions
+27 -16
View File
@@ -5,6 +5,7 @@ import path from "path";
import fs from "fs"; import fs from "fs";
import { createServer } from "http"; import { createServer } from "http";
import { Server } from "socket.io"; import { Server } from "socket.io";
import { Worker } from "worker_threads";
import multer from "multer"; import multer from "multer";
import archiver from "archiver"; import archiver from "archiver";
import Database from "better-sqlite3"; import Database from "better-sqlite3";
@@ -125,21 +126,31 @@ const respondWithValidationErrors = (
}); });
}; };
const runIntegrityCheck = (filePath: string): boolean => { // Non-blocking CPU check using worker threads
let dbInstance: Database.Database | undefined; const verifyDatabaseIntegrityAsync = (filePath: string): Promise<boolean> => {
try { return new Promise((resolve) => {
dbInstance = new Database(filePath, { const worker = new Worker(
readonly: true, path.resolve(__dirname, "./workers/db-verify.js"),
fileMustExist: true, {
workerData: { filePath },
}
);
worker.on("message", (isValid: boolean) => resolve(isValid));
worker.on("error", (err) => {
console.error("Worker error:", err);
resolve(false);
}); });
const result = dbInstance.prepare("PRAGMA integrity_check;").get(); worker.on("exit", (code) => {
return result?.integrity_check === "ok"; if (code !== 0) resolve(false);
} catch (error) { });
console.error("Integrity check failed:", error);
return false; // Kill worker if it takes too long (DoS protection)
} finally { setTimeout(() => {
dbInstance?.close(); worker.terminate();
} resolve(false);
}, 10000); // 10 second timeout
});
}; };
const removeFileIfExists = (filePath?: string) => { const removeFileIfExists = (filePath?: string) => {
@@ -645,7 +656,7 @@ app.post("/import/sqlite/verify", upload.single("db"), async (req, res) => {
} }
const stagedPath = req.file.path; const stagedPath = req.file.path;
const isValid = runIntegrityCheck(stagedPath); const isValid = await verifyDatabaseIntegrityAsync(stagedPath);
removeFileIfExists(stagedPath); removeFileIfExists(stagedPath);
if (!isValid) { if (!isValid) {
@@ -684,7 +695,7 @@ app.post("/import/sqlite", upload.single("db"), async (req, res) => {
return res.status(500).json({ error: "Failed to stage uploaded file" }); return res.status(500).json({ error: "Failed to stage uploaded file" });
} }
const isValid = runIntegrityCheck(stagedPath); const isValid = await verifyDatabaseIntegrityAsync(stagedPath);
if (!isValid) { if (!isValid) {
removeFileIfExists(stagedPath); removeFileIfExists(stagedPath);
return res return res
+18
View File
@@ -0,0 +1,18 @@
const { parentPort, workerData } = require('worker_threads');
const Database = require('better-sqlite3');
if (!parentPort) throw new Error("Must be run in a worker thread");
try {
const { filePath } = workerData;
const db = new Database(filePath, { readonly: true, fileMustExist: true });
// This is the CPU-heavy operation
const result = db.prepare("PRAGMA integrity_check;").get();
db.close();
parentPort.postMessage(result.integrity_check === "ok");
} catch (error) {
// Any error means invalid or corrupt DB
parentPort.postMessage(false);
}