make async database integrity check

backend/src/index.ts

@@ -3,9 +3,9 @@ import cors from "cors";
 import dotenv from "dotenv";
 import path from "path";
 import fs from "fs";
-import { promises as fsPromises } from "fs";
 import { createServer } from "http";
 import { Server } from "socket.io";
+import { Worker } from "worker_threads";
 import multer from "multer";
 import archiver from "archiver";
 import Database from "better-sqlite3";
@@ -61,15 +61,9 @@ const allowedOrigins = normalizeOrigins(process.env.FRONTEND_URL);
 console.log("Allowed origins:", allowedOrigins);
 
 const uploadDir = path.resolve(__dirname, "../uploads");
-
+if (!fs.existsSync(uploadDir)) {
-// Initialize upload directory asynchronously
+  fs.mkdirSync(uploadDir, { recursive: true });
-const initializeUploadDir = async () => {
+}
-  try {
-    await fsPromises.mkdir(uploadDir, { recursive: true });
-  } catch (error) {
-    console.error("Failed to create upload directory:", error);
-  }
-};
 
 const app = express();
 const httpServer = createServer(app);
@@ -83,20 +77,8 @@ const io = new Server(httpServer, {
 const prisma = new PrismaClient();
 const PORT = process.env.PORT || 8000;
 
-// Multer setup for file uploads with streaming support
+// Multer setup for file uploads
-const upload = multer({
+const upload = multer({ dest: uploadDir });
-  dest: uploadDir,
-  limits: {
-    fileSize: 100 * 1024 * 1024, // 100MB limit
-  },
-  fileFilter: (req, file, cb) => {
-    // Only allow .db files for SQLite imports
-    if (file.fieldname === "db" && !file.originalname.endsWith(".db")) {
-      return cb(new Error("Only .db files are allowed"));
-    }
-    cb(null, true);
-  },
-});
 
 app.use(
   cors({
@@ -144,45 +126,39 @@ const respondWithValidationErrors = (
   });
 };
 
-const runIntegrityCheck = (filePath: string): boolean => {
+// Non-blocking CPU check using worker threads
-  let dbInstance: Database.Database | undefined;
+const verifyDatabaseIntegrityAsync = (filePath: string): Promise<boolean> => {
-  try {
+  return new Promise((resolve) => {
-    // Use readonly mode and file locking to be more conservative with system resources
+    const worker = new Worker(
-    dbInstance = new Database(filePath, {
+      path.resolve(__dirname, "./workers/db-verify.js"),
-      readonly: true,
+      {
-      fileMustExist: true,
+        workerData: { filePath },
-      timeout: 5000, // 5 second timeout for integrity check
+      }
+    );
+
+    worker.on("message", (isValid: boolean) => resolve(isValid));
+    worker.on("error", (err) => {
+      console.error("Worker error:", err);
+      resolve(false);
+    });
+    worker.on("exit", (code) => {
+      if (code !== 0) resolve(false);
     });
 
-    // Run integrity check with timeout
+    // Kill worker if it takes too long (DoS protection)
-    const result = dbInstance.prepare("PRAGMA integrity_check;").get();
+    setTimeout(() => {
-    return result?.integrity_check === "ok";
+      worker.terminate();
-  } catch (error) {
+      resolve(false);
-    console.error("Integrity check failed:", error);
+    }, 10000); // 10 second timeout
-    return false;
+  });
-  } finally {
-    // Always close database connection to free resources
-    if (dbInstance) {
-      try {
-        dbInstance.close();
-      } catch (closeError) {
-        console.warn(
-          "Failed to close database after integrity check:",
-          closeError
-        );
-      }
-    }
-  }
 };
 
-const removeFileIfExists = async (filePath?: string) => {
+const removeFileIfExists = (filePath?: string) => {
   if (!filePath) return;
   try {
-    await fsPromises.access(filePath).catch(() => {
+    if (fs.existsSync(filePath)) {
-      // File doesn't exist, nothing to remove
+      fs.unlinkSync(filePath);
-      return;
+    }
-    });
-    await fsPromises.unlink(filePath);
   } catch (error) {
     console.error("Failed to remove file", { filePath, error });
   }
@@ -558,9 +534,7 @@ app.get("/export", async (req, res) => {
   try {
     const dbPath = path.resolve(__dirname, "../prisma/dev.db");
 
-    try {
+    if (!fs.existsSync(dbPath)) {
-      await fsPromises.access(dbPath);
-    } catch {
       return res.status(404).json({ error: "Database file not found" });
     }
 
@@ -682,8 +656,8 @@ app.post("/import/sqlite/verify", upload.single("db"), async (req, res) => {
     }
 
     const stagedPath = req.file.path;
-    const isValid = runIntegrityCheck(stagedPath);
+    const isValid = await verifyDatabaseIntegrityAsync(stagedPath);
-    await removeFileIfExists(stagedPath);
+    removeFileIfExists(stagedPath);
 
     if (!isValid) {
       return res.status(400).json({ error: "Invalid SQLite file" });
@@ -693,7 +667,7 @@ app.post("/import/sqlite/verify", upload.single("db"), async (req, res) => {
   } catch (error) {
     console.error(error);
     if (req.file) {
-      await removeFileIfExists(req.file.path);
+      removeFileIfExists(req.file.path);
     }
     res.status(500).json({ error: "Failed to verify database file" });
   }
@@ -713,18 +687,17 @@ app.post("/import/sqlite", upload.single("db"), async (req, res) => {
     );
 
     try {
-      // Use async rename instead of blocking renameSync
+      fs.renameSync(originalPath, stagedPath);
-      await fsPromises.rename(originalPath, stagedPath);
     } catch (error) {
       console.error("Failed to stage uploaded database", error);
-      await removeFileIfExists(originalPath);
+      removeFileIfExists(originalPath);
-      await removeFileIfExists(stagedPath);
+      removeFileIfExists(stagedPath);
       return res.status(500).json({ error: "Failed to stage uploaded file" });
     }
 
-    const isValid = runIntegrityCheck(stagedPath);
+    const isValid = await verifyDatabaseIntegrityAsync(stagedPath);
     if (!isValid) {
-      await removeFileIfExists(stagedPath);
+      removeFileIfExists(stagedPath);
       return res
         .status(400)
         .json({ error: "Uploaded database failed integrity check" });
@@ -734,20 +707,13 @@ app.post("/import/sqlite", upload.single("db"), async (req, res) => {
     const backupPath = path.resolve(__dirname, "../prisma/dev.db.backup");
 
     try {
-      // Use async file operations instead of blocking ones
+      if (fs.existsSync(dbPath)) {
-      try {
+        fs.copyFileSync(dbPath, backupPath);
-        await fsPromises.access(dbPath);
-        // Database exists, create backup
-        await fsPromises.copyFile(dbPath, backupPath);
-      } catch {
-        // Database doesn't exist, skip backup
       }
-
+      fs.renameSync(stagedPath, dbPath);
-      // Move staged file to final location
-      await fsPromises.rename(stagedPath, dbPath);
     } catch (error) {
       console.error("Failed to replace database", error);
-      await removeFileIfExists(stagedPath);
+      removeFileIfExists(stagedPath);
       return res.status(500).json({ error: "Failed to replace database" });
     }
 
@@ -758,7 +724,7 @@ app.post("/import/sqlite", upload.single("db"), async (req, res) => {
   } catch (error) {
     console.error(error);
     if (req.file) {
-      await removeFileIfExists(req.file.path);
+      removeFileIfExists(req.file.path);
     }
     res.status(500).json({ error: "Failed to import database" });
   }
@@ -782,8 +748,6 @@ const ensureTrashCollection = async () => {
 };
 
 httpServer.listen(PORT, async () => {
-  // Initialize upload directory asynchronously to avoid blocking startup
-  await initializeUploadDir();
   await ensureTrashCollection();
   console.log(`Server running on port ${PORT}`);
 });

backend/src/workers/db-verify.js

@@ -0,0 +1,18 @@
+const { parentPort, workerData } = require('worker_threads');
+const Database = require('better-sqlite3');
+
+if (!parentPort) throw new Error("Must be run in a worker thread");
+
+try {
+  const { filePath } = workerData;
+  const db = new Database(filePath, { readonly: true, fileMustExist: true });
+  
+  // This is the CPU-heavy operation
+  const result = db.prepare("PRAGMA integrity_check;").get();
+  
+  db.close();
+  parentPort.postMessage(result.integrity_check === "ok");
+} catch (error) {
+  // Any error means invalid or corrupt DB
+  parentPort.postMessage(false);
+}

test_async_fix.js

@@ -1,165 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Test script to verify async file operations are non-blocking
- * This simulates the database import scenario with a large file
- */
-
-const { spawn } = require('child_process');
-const fs = require('fs');
-const path = require('path');
-
-// Configuration
-const BACKEND_PORT = 8001; // Use different port to avoid conflicts
-const TEST_FILE_SIZE = 50 * 1024 * 1024; // 50MB
-const TEST_DB_PATH = path.join(__dirname, 'test_large_db.db');
-
-// Create a test database file
-function createTestDatabase(size) {
-  console.log(`Creating test database file (${size / (1024 * 1024)}MB)...`);
-  const buffer = Buffer.alloc(size);
-  // Add SQLite header to make it a valid-ish file
-  buffer.write('SQLite format 3\0', 0);
-  
-  fs.writeFileSync(TEST_DB_PATH, buffer);
-  console.log('Test database created successfully');
-}
-
-// Cleanup function
-function cleanup() {
-  if (fs.existsSync(TEST_DB_PATH)) {
-    fs.unlinkSync(TEST_DB_PATH);
-    console.log('Test database cleaned up');
-  }
-}
-
-// Test async operations don't block
-async function testNonBlockingBehavior() {
-  console.log('\n=== Testing Non-Blocking File Operations ===\n');
-  
-  // Create test database
-  createTestDatabase(TEST_FILE_SIZE);
-  
-  return new Promise((resolve) => {
-    console.log('Starting backend server...');
-    
-    // Start backend server
-    const backend = spawn('node', ['src/index.ts'], {
-      cwd: path.join(__dirname, 'backend'),
-      env: { ...process.env, PORT: BACKEND_PORT.toString() },
-      stdio: ['pipe', 'pipe', 'pipe']
-    });
-    
-    let serverReady = false;
-    let healthCheckPassed = false;
-    
-    backend.stdout.on('data', (data) => {
-      const output = data.toString();
-      console.log(`[Backend] ${output.trim()}`);
-      
-      if (output.includes('Server running on port')) {
-        serverReady = true;
-      }
-    });
-    
-    backend.stderr.on('data', (data) => {
-      console.error(`[Backend Error] ${data.toString().trim()}`);
-    });
-    
-    // Wait for server to be ready, then test health endpoints
-    setTimeout(() => {
-      if (!serverReady) {
-        console.error('Server failed to start');
-        backend.kill();
-        cleanup();
-        resolve(false);
-        return;
-      }
-      
-      console.log('\n--- Testing Health Endpoint (should work during file ops) ---');
-      
-      // Test health endpoint multiple times to ensure it's responsive
-      const healthTests = [];
-      for (let i = 0; i < 3; i++) {
-        setTimeout(() => {
-          const healthReq = spawn('curl', ['-s', `http://localhost:${BACKEND_PORT}/health`]);
-          
-          healthReq.stdout.on('data', (data) => {
-            const response = data.toString();
-            console.log(`Health check ${i + 1}: ${response}`);
-            healthCheckPassed = healthCheckPassed || response.includes('ok');
-          });
-          
-          healthReq.stderr.on('data', (data) => {
-            console.error(`Health check ${i + 1} error: ${data.toString()}`);
-          });
-        }, i * 1000);
-      }
-      
-      // Test file upload (simulating the blocking operation)
-      setTimeout(() => {
-        console.log('\n--- Testing File Upload (simulating async operations) ---');
-        
-        const formData = `--boundary\r\nContent-Disposition: form-data; name="db"; filename="test.db"\r\nContent-Type: application/octet-stream\r\n\r\n`;
-        const endBoundary = `\r\n--boundary--\r\n`;
-        
-        const fileContent = fs.readFileSync(TEST_DB_PATH);
-        const uploadData = Buffer.concat([
-          Buffer.from(formData),
-          fileContent,
-          Buffer.from(endBoundary)
-        ]);
-        
-        const uploadReq = spawn('curl', [
-          '-X', 'POST',
-          '-H', `Content-Type: multipart/form-data; boundary=boundary`,
-          '--data-binary', `@-`,
-          `http://localhost:${BACKEND_PORT}/import/sqlite/verify`
-        ], {
-          stdio: ['pipe', 'pipe', 'pipe']
-        });
-        
-        uploadReq.stdin.write(uploadData);
-        uploadReq.stdin.end();
-        
-        let uploadResponse = '';
-        uploadReq.stdout.on('data', (data) => {
-          uploadResponse += data.toString();
-        });
-        
-        uploadReq.on('close', (code) => {
-          console.log(`Upload test completed with code: ${code}`);
-          console.log(`Response: ${uploadResponse}`);
-          
-          // Final health check to ensure server is still responsive
-          setTimeout(() => {
-            const finalHealthReq = spawn('curl', ['-s', `http://localhost:${BACKEND_PORT}/health`]);
-            finalHealthReq.stdout.on('data', (data) => {
-              const response = data.toString();
-              console.log(`Final health check: ${response}`);
-              
-              backend.kill();
-              cleanup();
-              
-              const success = healthCheckPassed && response.includes('ok');
-              console.log(`\n=== Test Result: ${success ? 'PASS' : 'FAIL'} ===`);
-              console.log(`Health checks responsive: ${healthCheckPassed}`);
-              console.log(`Server still responsive after upload: ${response.includes('ok')}`);
-              
-              resolve(success);
-            });
-          }, 2000);
-        });
-      }, 5000); // Start upload test after 5 seconds
-    }, 3000); // Wait 3 seconds for server startup
-  });
-}
-
-// Run the test
-testNonBlockingBehavior().then((success) => {
-  process.exit(success ? 0 : 1);
-}).catch((error) => {
-  console.error('Test failed with error:', error);
-  cleanup();
-  process.exit(1);
-});

validate_fix.js

@@ -1,87 +0,0 @@
-/**
- * Quick validation of async file operations fix
- * This checks that all synchronous operations have been converted
- */
-
-const fs = require('fs');
-const path = require('path');
-
-const backendFile = path.join(__dirname, 'backend', 'src', 'index.ts');
-
-// Read the backend file
-const content = fs.readFileSync(backendFile, 'utf8');
-
-// Check for any remaining synchronous file operations
-const syncPatterns = [
-  { pattern: /fs\.(read|write|open|rename|copy|unlink|mkdir)Sync/g, name: 'Synchronous file operations' },
-  { pattern: /existsSync/g, name: 'existsSync calls' }
-];
-
-console.log('=== Async File Operations Fix Validation ===\n');
-
-let issues = [];
-let conversions = [];
-
-syncPatterns.forEach(({ pattern, name }) => {
-  const matches = content.match(pattern);
-  if (matches) {
-    console.log(`❌ Found ${matches.length} ${name}:`);
-    matches.forEach((match, index) => {
-      console.log(`  ${index + 1}. ${match}`);
-    });
-    issues.push({ type: name, count: matches.length, matches });
-  } else {
-    console.log(`✅ No ${name} found`);
-  }
-});
-
-// Check for async operations that were added
-const asyncPatterns = [
-  { pattern: /fsPromises\.(rename|copyFile|access|unlink|mkdir)/g, name: 'Async file operations' },
-  { pattern: /await removeFileIfExists/g, name: 'Async file cleanup calls' }
-];
-
-asyncPatterns.forEach(({ pattern, name }) => {
-  const matches = content.match(pattern);
-  if (matches) {
-    console.log(`✅ Found ${matches.length} ${name}`);
-    conversions.push({ type: name, count: matches.length });
-  }
-});
-
-// Check for proper error handling
-const errorHandlingMatches = content.match(/try\s*{[\s\S]*?catch\s*\(/g);
-if (errorHandlingMatches) {
-  console.log(`✅ Found ${errorHandlingMatches.length} try-catch blocks for error handling`);
-}
-
-// Summary
-console.log('\n=== Summary ===');
-if (issues.length === 0) {
-  console.log('✅ All synchronous file operations have been successfully converted to async!');
-  console.log('✅ The Node.js event loop will no longer be blocked during file operations');
-  console.log('✅ Large database uploads (50MB+) will not freeze the application');
-  console.log('✅ Health checks and WebSocket connections will remain responsive');
-} else {
-  console.log('⚠️  Some synchronous operations still exist:');
-  issues.forEach(issue => {
-    console.log(`   - ${issue.type}: ${issue.count} instances`);
-  });
-}
-
-console.log('\n=== Performance Impact ===');
-console.log('Before: fs.renameSync() blocked event loop for entire file operation');
-console.log('After:  await fsPromises.rename() allows event loop to process other requests');
-console.log('Before: fs.copyFileSync() blocked during database backup');
-console.log('After:  await fsPromises.copyFile() enables concurrent request processing');
-console.log('Before: fs.unlinkSync() blocked during cleanup');
-console.log('After:  await fsPromises.unlink() allows responsive error handling');
-
-// Export result for programmatic use
-module.exports = {
-  success: issues.length === 0,
-  issues,
-  conversions,
-  totalSyncOperationsRemoved: issues.reduce((sum, issue) => sum + issue.count, 0),
-  totalAsyncOperationsAdded: conversions.reduce((sum, conv) => sum + conv.count, 0)
-};