Spaces:
Paused
Paused
Deploy from GitHub Actions 2025-12-15_23-41-14
Browse files
apps/backend/package.json
CHANGED
|
@@ -13,7 +13,12 @@
|
|
| 13 |
"lint": "eslint .",
|
| 14 |
"neural-bridge": "tsx src/mcp/servers/NeuralBridgeServer.ts",
|
| 15 |
"neural-bridge:build": "tsc && node dist/mcp/servers/NeuralBridgeServer.js",
|
| 16 |
-
"ingest-drive": "tsx src/scripts/ingest-drive.ts"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
},
|
| 18 |
"dependencies": {
|
| 19 |
"@anthropic-ai/sdk": "^0.71.0",
|
|
|
|
| 13 |
"lint": "eslint .",
|
| 14 |
"neural-bridge": "tsx src/mcp/servers/NeuralBridgeServer.ts",
|
| 15 |
"neural-bridge:build": "tsc && node dist/mcp/servers/NeuralBridgeServer.js",
|
| 16 |
+
"ingest-drive": "tsx src/scripts/ingest-drive.ts",
|
| 17 |
+
"sync-neo4j": "tsx src/scripts/sync-neo4j-to-cloud.ts",
|
| 18 |
+
"sync:start": "tsx src/scripts/neo4j-auto-sync.ts start",
|
| 19 |
+
"sync:now": "tsx src/scripts/neo4j-auto-sync.ts sync",
|
| 20 |
+
"sync:full": "tsx src/scripts/neo4j-auto-sync.ts full",
|
| 21 |
+
"sync:status": "tsx src/scripts/neo4j-auto-sync.ts status"
|
| 22 |
},
|
| 23 |
"dependencies": {
|
| 24 |
"@anthropic-ai/sdk": "^0.71.0",
|
apps/backend/src/index.ts
CHANGED
|
@@ -615,6 +615,11 @@ async function startServer() {
|
|
| 615 |
app.use('/api/email', emailRoutes);
|
| 616 |
console.log('π§ Email API mounted at /api/email');
|
| 617 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 618 |
// Start KnowledgeCompiler auto-compilation (every 60 seconds)
|
| 619 |
const { knowledgeCompiler } = await import('./services/Knowledge/index.js');
|
| 620 |
knowledgeCompiler.startAutoCompilation(60000);
|
|
|
|
| 615 |
app.use('/api/email', emailRoutes);
|
| 616 |
console.log('π§ Email API mounted at /api/email');
|
| 617 |
|
| 618 |
+
// Neo4j Sync API - Auto-sync between local and cloud
|
| 619 |
+
const neo4jSyncRoutes = (await import('./routes/neo4j-sync.js')).default;
|
| 620 |
+
app.use('/api/neo4j-sync', neo4jSyncRoutes);
|
| 621 |
+
console.log('π Neo4j Sync API mounted at /api/neo4j-sync');
|
| 622 |
+
|
| 623 |
// Start KnowledgeCompiler auto-compilation (every 60 seconds)
|
| 624 |
const { knowledgeCompiler } = await import('./services/Knowledge/index.js');
|
| 625 |
knowledgeCompiler.startAutoCompilation(60000);
|
apps/backend/src/routes/neo4j-sync.ts
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Neo4j Sync API Routes
|
| 3 |
+
*
|
| 4 |
+
* Provides REST endpoints for Neo4j sync status and control.
|
| 5 |
+
*/
|
| 6 |
+
|
| 7 |
+
import { Router, Request, Response } from 'express';
|
| 8 |
+
import { getNeo4jAutoSync } from '../services/Neo4jAutoSync.js';
|
| 9 |
+
|
| 10 |
+
const router = Router();
|
| 11 |
+
|
| 12 |
+
/**
|
| 13 |
+
* GET /api/neo4j-sync/status
|
| 14 |
+
* Get current sync status
|
| 15 |
+
*/
|
| 16 |
+
router.get('/status', (_req: Request, res: Response) => {
|
| 17 |
+
try {
|
| 18 |
+
const syncService = getNeo4jAutoSync();
|
| 19 |
+
const status = syncService.getStatus();
|
| 20 |
+
res.json({
|
| 21 |
+
success: true,
|
| 22 |
+
data: status
|
| 23 |
+
});
|
| 24 |
+
} catch (error: any) {
|
| 25 |
+
res.status(500).json({
|
| 26 |
+
success: false,
|
| 27 |
+
error: error.message
|
| 28 |
+
});
|
| 29 |
+
}
|
| 30 |
+
});
|
| 31 |
+
|
| 32 |
+
/**
|
| 33 |
+
* POST /api/neo4j-sync/sync
|
| 34 |
+
* Trigger a sync (incremental by default)
|
| 35 |
+
*/
|
| 36 |
+
router.post('/sync', async (req: Request, res: Response) => {
|
| 37 |
+
try {
|
| 38 |
+
const { full = false } = req.body || {};
|
| 39 |
+
const syncService = getNeo4jAutoSync();
|
| 40 |
+
|
| 41 |
+
// Return immediately, sync runs in background
|
| 42 |
+
res.json({
|
| 43 |
+
success: true,
|
| 44 |
+
message: `${full ? 'Full' : 'Incremental'} sync started`,
|
| 45 |
+
data: { type: full ? 'full' : 'incremental' }
|
| 46 |
+
});
|
| 47 |
+
|
| 48 |
+
// Run sync in background
|
| 49 |
+
syncService.sync(full).catch(err => {
|
| 50 |
+
console.error('Background sync failed:', err);
|
| 51 |
+
});
|
| 52 |
+
|
| 53 |
+
} catch (error: any) {
|
| 54 |
+
res.status(500).json({
|
| 55 |
+
success: false,
|
| 56 |
+
error: error.message
|
| 57 |
+
});
|
| 58 |
+
}
|
| 59 |
+
});
|
| 60 |
+
|
| 61 |
+
/**
|
| 62 |
+
* POST /api/neo4j-sync/scheduler/start
|
| 63 |
+
* Start the scheduler
|
| 64 |
+
*/
|
| 65 |
+
router.post('/scheduler/start', (_req: Request, res: Response) => {
|
| 66 |
+
try {
|
| 67 |
+
const syncService = getNeo4jAutoSync();
|
| 68 |
+
syncService.startScheduler();
|
| 69 |
+
res.json({
|
| 70 |
+
success: true,
|
| 71 |
+
message: 'Scheduler started',
|
| 72 |
+
data: syncService.getStatus()
|
| 73 |
+
});
|
| 74 |
+
} catch (error: any) {
|
| 75 |
+
res.status(500).json({
|
| 76 |
+
success: false,
|
| 77 |
+
error: error.message
|
| 78 |
+
});
|
| 79 |
+
}
|
| 80 |
+
});
|
| 81 |
+
|
| 82 |
+
/**
|
| 83 |
+
* POST /api/neo4j-sync/scheduler/stop
|
| 84 |
+
* Stop the scheduler
|
| 85 |
+
*/
|
| 86 |
+
router.post('/scheduler/stop', (_req: Request, res: Response) => {
|
| 87 |
+
try {
|
| 88 |
+
const syncService = getNeo4jAutoSync();
|
| 89 |
+
syncService.stopScheduler();
|
| 90 |
+
res.json({
|
| 91 |
+
success: true,
|
| 92 |
+
message: 'Scheduler stopped'
|
| 93 |
+
});
|
| 94 |
+
} catch (error: any) {
|
| 95 |
+
res.status(500).json({
|
| 96 |
+
success: false,
|
| 97 |
+
error: error.message
|
| 98 |
+
});
|
| 99 |
+
}
|
| 100 |
+
});
|
| 101 |
+
|
| 102 |
+
export default router;
|
apps/backend/src/scripts/neo4j-auto-sync.ts
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
/**
|
| 3 |
+
* Neo4j Auto-Sync Runner
|
| 4 |
+
*
|
| 5 |
+
* Runs as a standalone service for automated Neo4j synchronization.
|
| 6 |
+
*
|
| 7 |
+
* Usage:
|
| 8 |
+
* npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts [command]
|
| 9 |
+
*
|
| 10 |
+
* Commands:
|
| 11 |
+
* start - Start scheduler (default)
|
| 12 |
+
* sync - Run single sync now
|
| 13 |
+
* full - Run full sync (ignores checkpoint)
|
| 14 |
+
* status - Show sync status
|
| 15 |
+
* help - Show this help
|
| 16 |
+
*
|
| 17 |
+
* Environment Variables:
|
| 18 |
+
* NEO4J_LOCAL_URI - Local Neo4j URI (default: bolt://localhost:7687)
|
| 19 |
+
* NEO4J_LOCAL_USER - Local Neo4j user (default: neo4j)
|
| 20 |
+
* NEO4J_LOCAL_PASSWORD - Local Neo4j password (default: password)
|
| 21 |
+
* NEO4J_URI - Cloud Neo4j URI
|
| 22 |
+
* NEO4J_USER - Cloud Neo4j user
|
| 23 |
+
* NEO4J_PASSWORD - Cloud Neo4j password
|
| 24 |
+
* NEO4J_SYNC_SCHEDULE - Cron schedule (default: "0 *\/6 * * *" = every 6 hours)
|
| 25 |
+
*/
|
| 26 |
+
|
| 27 |
+
import dotenv from 'dotenv';
|
| 28 |
+
import path from 'path';
|
| 29 |
+
import fs from 'fs';
|
| 30 |
+
import { Neo4jAutoSync } from '../services/Neo4jAutoSync.js';
|
| 31 |
+
|
| 32 |
+
// Load environment
|
| 33 |
+
dotenv.config({ path: path.resolve(process.cwd(), '.env.production') });
|
| 34 |
+
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
|
| 35 |
+
|
| 36 |
+
const STATUS_FILE = path.join(process.cwd(), '.neo4j-sync-status.json');
|
| 37 |
+
|
| 38 |
+
async function main() {
|
| 39 |
+
const command = process.argv[2] || 'start';
|
| 40 |
+
|
| 41 |
+
console.log('β'.repeat(60));
|
| 42 |
+
console.log('π Neo4j Auto-Sync Service');
|
| 43 |
+
console.log('β'.repeat(60));
|
| 44 |
+
console.log('');
|
| 45 |
+
|
| 46 |
+
switch (command) {
|
| 47 |
+
case 'start':
|
| 48 |
+
await startScheduler();
|
| 49 |
+
break;
|
| 50 |
+
|
| 51 |
+
case 'sync':
|
| 52 |
+
await runSync(false);
|
| 53 |
+
break;
|
| 54 |
+
|
| 55 |
+
case 'full':
|
| 56 |
+
await runSync(true);
|
| 57 |
+
break;
|
| 58 |
+
|
| 59 |
+
case 'status':
|
| 60 |
+
showStatus();
|
| 61 |
+
break;
|
| 62 |
+
|
| 63 |
+
case 'help':
|
| 64 |
+
default:
|
| 65 |
+
showHelp();
|
| 66 |
+
break;
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
async function startScheduler() {
|
| 71 |
+
const syncService = new Neo4jAutoSync();
|
| 72 |
+
const schedule = process.env.NEO4J_SYNC_SCHEDULE || '0 */6 * * *';
|
| 73 |
+
|
| 74 |
+
console.log('Configuration:');
|
| 75 |
+
console.log(` Local: ${process.env.NEO4J_LOCAL_URI || 'bolt://localhost:7687'}`);
|
| 76 |
+
console.log(` Cloud: ${process.env.NEO4J_URI || '(not configured)'}`);
|
| 77 |
+
console.log(` Schedule: ${schedule}`);
|
| 78 |
+
console.log('');
|
| 79 |
+
|
| 80 |
+
// Run initial sync
|
| 81 |
+
console.log('Running initial sync...\n');
|
| 82 |
+
await syncService.sync();
|
| 83 |
+
|
| 84 |
+
// Start scheduler
|
| 85 |
+
syncService.startScheduler();
|
| 86 |
+
|
| 87 |
+
console.log('\nβ
Auto-sync service running. Press Ctrl+C to stop.\n');
|
| 88 |
+
|
| 89 |
+
// Keep process alive
|
| 90 |
+
process.on('SIGINT', () => {
|
| 91 |
+
console.log('\nβΉοΈ Shutting down...');
|
| 92 |
+
syncService.stopScheduler();
|
| 93 |
+
process.exit(0);
|
| 94 |
+
});
|
| 95 |
+
|
| 96 |
+
// Keep alive
|
| 97 |
+
setInterval(() => {
|
| 98 |
+
// Heartbeat
|
| 99 |
+
}, 60000);
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
async function runSync(forceFullSync: boolean) {
|
| 103 |
+
const syncService = new Neo4jAutoSync();
|
| 104 |
+
|
| 105 |
+
console.log(`Running ${forceFullSync ? 'FULL' : 'incremental'} sync...\n`);
|
| 106 |
+
|
| 107 |
+
const status = await syncService.sync(forceFullSync);
|
| 108 |
+
|
| 109 |
+
console.log('\n' + 'β'.repeat(60));
|
| 110 |
+
console.log('Sync Result:');
|
| 111 |
+
console.log(` Status: ${status.status}`);
|
| 112 |
+
console.log(` Type: ${status.lastSyncType}`);
|
| 113 |
+
console.log(` Nodes: ${status.nodesSync}`);
|
| 114 |
+
console.log(` Relationships: ${status.relationshipsSync}`);
|
| 115 |
+
console.log(` Duration: ${status.lastSyncDuration}ms`);
|
| 116 |
+
if (status.error) {
|
| 117 |
+
console.log(` Error: ${status.error}`);
|
| 118 |
+
}
|
| 119 |
+
console.log('β'.repeat(60));
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
function showStatus() {
|
| 123 |
+
if (fs.existsSync(STATUS_FILE)) {
|
| 124 |
+
const status = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf-8'));
|
| 125 |
+
console.log('Current Sync Status:');
|
| 126 |
+
console.log('');
|
| 127 |
+
console.log(` Status: ${status.status}`);
|
| 128 |
+
console.log(` Last Sync: ${status.lastSyncTime || 'Never'}`);
|
| 129 |
+
console.log(` Sync Type: ${status.lastSyncType || 'N/A'}`);
|
| 130 |
+
console.log(` Nodes Synced: ${status.nodesSync || 0}`);
|
| 131 |
+
console.log(` Rels Synced: ${status.relationshipsSync || 0}`);
|
| 132 |
+
console.log(` Duration: ${status.lastSyncDuration ? status.lastSyncDuration + 'ms' : 'N/A'}`);
|
| 133 |
+
console.log(` Next Scheduled: ${status.nextScheduledSync || 'N/A'}`);
|
| 134 |
+
if (status.error) {
|
| 135 |
+
console.log(` Error: ${status.error}`);
|
| 136 |
+
}
|
| 137 |
+
} else {
|
| 138 |
+
console.log('No sync status found. Run a sync first.');
|
| 139 |
+
}
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
function showHelp() {
|
| 143 |
+
console.log(`
|
| 144 |
+
Usage: npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts [command]
|
| 145 |
+
|
| 146 |
+
Commands:
|
| 147 |
+
start Start the scheduler service (runs continuously)
|
| 148 |
+
sync Run a single sync (incremental if checkpoint exists)
|
| 149 |
+
full Run a full sync (ignores checkpoint, replaces all data)
|
| 150 |
+
status Show current sync status
|
| 151 |
+
help Show this help message
|
| 152 |
+
|
| 153 |
+
Environment Variables:
|
| 154 |
+
NEO4J_LOCAL_URI Local Neo4j URI (default: bolt://localhost:7687)
|
| 155 |
+
NEO4J_LOCAL_USER Local Neo4j user (default: neo4j)
|
| 156 |
+
NEO4J_LOCAL_PASSWORD Local Neo4j password (default: password)
|
| 157 |
+
NEO4J_URI Cloud Neo4j URI (required)
|
| 158 |
+
NEO4J_USER Cloud Neo4j user (default: neo4j)
|
| 159 |
+
NEO4J_PASSWORD Cloud Neo4j password (required)
|
| 160 |
+
NEO4J_SYNC_SCHEDULE Cron schedule (default: 0 */6 * * * = every 6 hours)
|
| 161 |
+
|
| 162 |
+
Schedule Examples:
|
| 163 |
+
0 */6 * * * Every 6 hours
|
| 164 |
+
0 */2 * * * Every 2 hours
|
| 165 |
+
0 0 * * * Daily at midnight
|
| 166 |
+
*/30 * * * * Every 30 minutes
|
| 167 |
+
0 9,18 * * * At 9am and 6pm
|
| 168 |
+
|
| 169 |
+
Examples:
|
| 170 |
+
# Start auto-sync service (runs continuously)
|
| 171 |
+
npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts start
|
| 172 |
+
|
| 173 |
+
# Run single sync
|
| 174 |
+
npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts sync
|
| 175 |
+
|
| 176 |
+
# Force full sync
|
| 177 |
+
npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts full
|
| 178 |
+
|
| 179 |
+
# Check status
|
| 180 |
+
npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts status
|
| 181 |
+
`);
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
main().catch(console.error);
|
apps/backend/src/scripts/sync-neo4j-to-cloud.ts
ADDED
|
@@ -0,0 +1,470 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Neo4j Local to Cloud Sync Script
|
| 3 |
+
*
|
| 4 |
+
* Syncs data from local Neo4j (bruttopuljen/master) to AuraDB cloud
|
| 5 |
+
* Run: npx tsx apps/backend/src/scripts/sync-neo4j-to-cloud.ts
|
| 6 |
+
*/
|
| 7 |
+
|
| 8 |
+
import neo4j, { Driver, Session, Record as Neo4jRecord, isNode, isRelationship } from 'neo4j-driver';
|
| 9 |
+
import dotenv from 'dotenv';
|
| 10 |
+
import path from 'path';
|
| 11 |
+
|
| 12 |
+
// Load environment
|
| 13 |
+
dotenv.config({ path: path.resolve(process.cwd(), '.env.production') });
|
| 14 |
+
|
| 15 |
+
interface SyncConfig {
|
| 16 |
+
local: {
|
| 17 |
+
uri: string;
|
| 18 |
+
user: string;
|
| 19 |
+
password: string;
|
| 20 |
+
database: string;
|
| 21 |
+
};
|
| 22 |
+
cloud: {
|
| 23 |
+
uri: string;
|
| 24 |
+
user: string;
|
| 25 |
+
password: string;
|
| 26 |
+
database: string;
|
| 27 |
+
};
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
interface NodeData {
|
| 31 |
+
labels: string[];
|
| 32 |
+
properties: Record<string, any>;
|
| 33 |
+
elementId: string;
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
interface RelationshipData {
|
| 37 |
+
type: string;
|
| 38 |
+
properties: Record<string, any>;
|
| 39 |
+
startElementId: string;
|
| 40 |
+
endElementId: string;
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
interface SchemaItem {
|
| 44 |
+
type: 'constraint' | 'index';
|
| 45 |
+
name: string;
|
| 46 |
+
definition: string;
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
const config: SyncConfig = {
|
| 50 |
+
local: {
|
| 51 |
+
uri: 'bolt://localhost:7687',
|
| 52 |
+
user: 'neo4j',
|
| 53 |
+
password: 'password', // Update if different
|
| 54 |
+
database: 'neo4j'
|
| 55 |
+
},
|
| 56 |
+
cloud: {
|
| 57 |
+
uri: process.env.NEO4J_URI || 'neo4j+s://054eff27.databases.neo4j.io',
|
| 58 |
+
user: process.env.NEO4J_USER || 'neo4j',
|
| 59 |
+
password: process.env.NEO4J_PASSWORD || '',
|
| 60 |
+
database: process.env.NEO4J_DATABASE || 'neo4j'
|
| 61 |
+
}
|
| 62 |
+
};
|
| 63 |
+
|
| 64 |
+
async function createDriver(connectionConfig: SyncConfig['local'] | SyncConfig['cloud']): Promise<Driver> {
|
| 65 |
+
return neo4j.driver(
|
| 66 |
+
connectionConfig.uri,
|
| 67 |
+
neo4j.auth.basic(connectionConfig.user, connectionConfig.password),
|
| 68 |
+
{
|
| 69 |
+
maxConnectionLifetime: 3 * 60 * 60 * 1000,
|
| 70 |
+
connectionAcquisitionTimeout: 2 * 60 * 1000,
|
| 71 |
+
}
|
| 72 |
+
);
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
async function exportSchema(session: Session): Promise<SchemaItem[]> {
|
| 76 |
+
console.log('π Exporting schema from local...');
|
| 77 |
+
const schema: SchemaItem[] = [];
|
| 78 |
+
|
| 79 |
+
// Export constraints
|
| 80 |
+
const constraintResult = await session.run('SHOW CONSTRAINTS');
|
| 81 |
+
for (const record of constraintResult.records) {
|
| 82 |
+
const name = record.get('name');
|
| 83 |
+
const type = record.get('type');
|
| 84 |
+
const entityType = record.get('entityType');
|
| 85 |
+
const labelsOrTypes = record.get('labelsOrTypes');
|
| 86 |
+
const properties = record.get('properties');
|
| 87 |
+
|
| 88 |
+
let definition = '';
|
| 89 |
+
if (type === 'UNIQUENESS' && entityType === 'NODE') {
|
| 90 |
+
const label = labelsOrTypes?.[0] || 'Unknown';
|
| 91 |
+
const prop = properties?.[0] || 'id';
|
| 92 |
+
definition = `CREATE CONSTRAINT ${name} IF NOT EXISTS FOR (n:${label}) REQUIRE n.${prop} IS UNIQUE`;
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
if (definition) {
|
| 96 |
+
schema.push({ type: 'constraint', name, definition });
|
| 97 |
+
console.log(` β Constraint: ${name}`);
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
// Export indexes (excluding auto-generated and constraint-backed)
|
| 102 |
+
const indexResult = await session.run('SHOW INDEXES');
|
| 103 |
+
for (const record of indexResult.records) {
|
| 104 |
+
const name = record.get('name');
|
| 105 |
+
const type = record.get('type');
|
| 106 |
+
const entityType = record.get('entityType');
|
| 107 |
+
const labelsOrTypes = record.get('labelsOrTypes');
|
| 108 |
+
const properties = record.get('properties');
|
| 109 |
+
const owningConstraint = record.get('owningConstraint');
|
| 110 |
+
|
| 111 |
+
// Skip constraint-backed indexes
|
| 112 |
+
if (owningConstraint) continue;
|
| 113 |
+
|
| 114 |
+
let definition = '';
|
| 115 |
+
if (type === 'RANGE' && entityType === 'NODE') {
|
| 116 |
+
const label = labelsOrTypes?.[0] || 'Unknown';
|
| 117 |
+
const props = properties?.join(', ') || 'id';
|
| 118 |
+
definition = `CREATE INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${props.split(',')[0].trim()})`;
|
| 119 |
+
} else if (type === 'FULLTEXT' && entityType === 'NODE') {
|
| 120 |
+
const label = labelsOrTypes?.[0] || 'Unknown';
|
| 121 |
+
const props = (properties || []).map((p: string) => `n.${p}`).join(', ');
|
| 122 |
+
definition = `CREATE FULLTEXT INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON EACH [${props}]`;
|
| 123 |
+
} else if (type === 'VECTOR') {
|
| 124 |
+
const label = labelsOrTypes?.[0] || 'Unknown';
|
| 125 |
+
const prop = properties?.[0] || 'embedding';
|
| 126 |
+
definition = `CREATE VECTOR INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${prop}) OPTIONS {indexConfig: {\`vector.dimensions\`: 384, \`vector.similarity_function\`: 'cosine'}}`;
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
if (definition) {
|
| 130 |
+
schema.push({ type: 'index', name, definition });
|
| 131 |
+
console.log(` β Index: ${name} (${type})`);
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
console.log(` Total: ${schema.length} schema items`);
|
| 136 |
+
return schema;
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
async function exportNodes(session: Session): Promise<NodeData[]> {
|
| 140 |
+
console.log('\nπ¦ Exporting nodes from local...');
|
| 141 |
+
const nodes: NodeData[] = [];
|
| 142 |
+
|
| 143 |
+
const result = await session.run('MATCH (n) RETURN n, elementId(n) as elementId');
|
| 144 |
+
|
| 145 |
+
for (const record of result.records) {
|
| 146 |
+
const node = record.get('n');
|
| 147 |
+
const elementId = record.get('elementId');
|
| 148 |
+
|
| 149 |
+
if (isNode(node)) {
|
| 150 |
+
// Clone properties and convert Neo4j types to JS types
|
| 151 |
+
const properties: Record<string, any> = {};
|
| 152 |
+
for (const [key, value] of Object.entries(node.properties)) {
|
| 153 |
+
properties[key] = convertNeo4jValue(value);
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
nodes.push({
|
| 157 |
+
labels: [...node.labels],
|
| 158 |
+
properties,
|
| 159 |
+
elementId
|
| 160 |
+
});
|
| 161 |
+
}
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
console.log(` β Exported ${nodes.length} nodes`);
|
| 165 |
+
|
| 166 |
+
// Show label distribution
|
| 167 |
+
const labelCounts: Record<string, number> = {};
|
| 168 |
+
for (const node of nodes) {
|
| 169 |
+
for (const label of node.labels) {
|
| 170 |
+
labelCounts[label] = (labelCounts[label] || 0) + 1;
|
| 171 |
+
}
|
| 172 |
+
}
|
| 173 |
+
for (const [label, count] of Object.entries(labelCounts)) {
|
| 174 |
+
console.log(` - ${label}: ${count}`);
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
return nodes;
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
async function exportRelationships(session: Session): Promise<RelationshipData[]> {
|
| 181 |
+
console.log('\nπ Exporting relationships from local...');
|
| 182 |
+
const relationships: RelationshipData[] = [];
|
| 183 |
+
|
| 184 |
+
const result = await session.run(`
|
| 185 |
+
MATCH (a)-[r]->(b)
|
| 186 |
+
RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
|
| 187 |
+
`);
|
| 188 |
+
|
| 189 |
+
for (const record of result.records) {
|
| 190 |
+
const rel = record.get('r');
|
| 191 |
+
const type = record.get('type');
|
| 192 |
+
const startId = record.get('startId');
|
| 193 |
+
const endId = record.get('endId');
|
| 194 |
+
|
| 195 |
+
// Clone properties and convert Neo4j types
|
| 196 |
+
const properties: Record<string, any> = {};
|
| 197 |
+
if (isRelationship(rel)) {
|
| 198 |
+
for (const [key, value] of Object.entries(rel.properties)) {
|
| 199 |
+
properties[key] = convertNeo4jValue(value);
|
| 200 |
+
}
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
relationships.push({
|
| 204 |
+
type,
|
| 205 |
+
properties,
|
| 206 |
+
startElementId: startId,
|
| 207 |
+
endElementId: endId
|
| 208 |
+
});
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
console.log(` β Exported ${relationships.length} relationships`);
|
| 212 |
+
|
| 213 |
+
// Show relationship type distribution
|
| 214 |
+
const typeCounts: Record<string, number> = {};
|
| 215 |
+
for (const rel of relationships) {
|
| 216 |
+
typeCounts[rel.type] = (typeCounts[rel.type] || 0) + 1;
|
| 217 |
+
}
|
| 218 |
+
for (const [type, count] of Object.entries(typeCounts)) {
|
| 219 |
+
console.log(` - ${type}: ${count}`);
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
return relationships;
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
function convertNeo4jValue(value: any): any {
|
| 226 |
+
if (value === null || value === undefined) return value;
|
| 227 |
+
|
| 228 |
+
// Handle Neo4j Integer
|
| 229 |
+
if (typeof value === 'object' && value.constructor?.name === 'Integer') {
|
| 230 |
+
return value.toNumber();
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
// Handle Neo4j DateTime
|
| 234 |
+
if (typeof value === 'object' && value.constructor?.name === 'DateTime') {
|
| 235 |
+
return value.toString();
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
// Handle Neo4j Date
|
| 239 |
+
if (typeof value === 'object' && value.constructor?.name === 'Date') {
|
| 240 |
+
return value.toString();
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
// Handle arrays
|
| 244 |
+
if (Array.isArray(value)) {
|
| 245 |
+
return value.map(convertNeo4jValue);
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
// Handle Float arrays (embeddings)
|
| 249 |
+
if (typeof value === 'object' && value.constructor?.name === 'Float64Array') {
|
| 250 |
+
return Array.from(value);
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
return value;
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
async function clearCloud(session: Session): Promise<void> {
|
| 257 |
+
console.log('\nποΈ Clearing cloud database...');
|
| 258 |
+
|
| 259 |
+
// Delete in batches to avoid memory issues
|
| 260 |
+
let deleted = 0;
|
| 261 |
+
do {
|
| 262 |
+
const result = await session.run(`
|
| 263 |
+
MATCH (n)
|
| 264 |
+
WITH n LIMIT 1000
|
| 265 |
+
DETACH DELETE n
|
| 266 |
+
RETURN count(*) as deleted
|
| 267 |
+
`);
|
| 268 |
+
deleted = result.records[0]?.get('deleted')?.toNumber() || 0;
|
| 269 |
+
if (deleted > 0) {
|
| 270 |
+
console.log(` Deleted batch of ${deleted} nodes...`);
|
| 271 |
+
}
|
| 272 |
+
} while (deleted > 0);
|
| 273 |
+
|
| 274 |
+
console.log(' β Cloud database cleared');
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
async function applySchema(session: Session, schema: SchemaItem[]): Promise<void> {
|
| 278 |
+
console.log('\nπ Applying schema to cloud...');
|
| 279 |
+
|
| 280 |
+
for (const item of schema) {
|
| 281 |
+
try {
|
| 282 |
+
await session.run(item.definition);
|
| 283 |
+
console.log(` β ${item.type}: ${item.name}`);
|
| 284 |
+
} catch (error: any) {
|
| 285 |
+
if (error.message?.includes('already exists') || error.message?.includes('equivalent')) {
|
| 286 |
+
console.log(` β ${item.type} ${item.name} already exists`);
|
| 287 |
+
} else {
|
| 288 |
+
console.error(` β Failed to create ${item.type} ${item.name}: ${error.message}`);
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
}
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
async function importNodes(session: Session, nodes: NodeData[], elementIdMap: Map<string, string>): Promise<void> {
|
| 295 |
+
console.log('\nπ¦ Importing nodes to cloud...');
|
| 296 |
+
|
| 297 |
+
let imported = 0;
|
| 298 |
+
const batchSize = 100;
|
| 299 |
+
|
| 300 |
+
for (let i = 0; i < nodes.length; i += batchSize) {
|
| 301 |
+
const batch = nodes.slice(i, i + batchSize);
|
| 302 |
+
|
| 303 |
+
for (const node of batch) {
|
| 304 |
+
try {
|
| 305 |
+
// Create node with labels and properties
|
| 306 |
+
const labelStr = node.labels.join(':');
|
| 307 |
+
|
| 308 |
+
// Generate a unique ID if not present
|
| 309 |
+
const nodeId = node.properties.id || `node_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
| 310 |
+
node.properties.id = nodeId;
|
| 311 |
+
|
| 312 |
+
const query = `
|
| 313 |
+
MERGE (n:${labelStr} {id: $nodeId})
|
| 314 |
+
SET n = $props
|
| 315 |
+
RETURN elementId(n) as newElementId
|
| 316 |
+
`;
|
| 317 |
+
|
| 318 |
+
const result = await session.run(query, {
|
| 319 |
+
nodeId,
|
| 320 |
+
props: node.properties
|
| 321 |
+
});
|
| 322 |
+
|
| 323 |
+
const newElementId = result.records[0]?.get('newElementId');
|
| 324 |
+
if (newElementId) {
|
| 325 |
+
elementIdMap.set(node.elementId, newElementId);
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
imported++;
|
| 329 |
+
} catch (error: any) {
|
| 330 |
+
console.error(` β Failed to import node: ${error.message?.slice(0, 100)}`);
|
| 331 |
+
}
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
console.log(` Progress: ${imported}/${nodes.length} nodes`);
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
console.log(` β Imported ${imported} nodes`);
|
| 338 |
+
}
|
| 339 |
+
|
| 340 |
+
async function importRelationships(
|
| 341 |
+
session: Session,
|
| 342 |
+
relationships: RelationshipData[],
|
| 343 |
+
elementIdMap: Map<string, string>,
|
| 344 |
+
nodes: NodeData[]
|
| 345 |
+
): Promise<void> {
|
| 346 |
+
console.log('\nπ Importing relationships to cloud...');
|
| 347 |
+
|
| 348 |
+
// Create a map from old elementId to node id (for relationship matching)
|
| 349 |
+
const elementIdToNodeId = new Map<string, string>();
|
| 350 |
+
for (const node of nodes) {
|
| 351 |
+
elementIdToNodeId.set(node.elementId, node.properties.id);
|
| 352 |
+
}
|
| 353 |
+
|
| 354 |
+
let imported = 0;
|
| 355 |
+
let skipped = 0;
|
| 356 |
+
|
| 357 |
+
for (const rel of relationships) {
|
| 358 |
+
try {
|
| 359 |
+
const startNodeId = elementIdToNodeId.get(rel.startElementId);
|
| 360 |
+
const endNodeId = elementIdToNodeId.get(rel.endElementId);
|
| 361 |
+
|
| 362 |
+
if (!startNodeId || !endNodeId) {
|
| 363 |
+
skipped++;
|
| 364 |
+
continue;
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
const query = `
|
| 368 |
+
MATCH (a {id: $startId})
|
| 369 |
+
MATCH (b {id: $endId})
|
| 370 |
+
MERGE (a)-[r:${rel.type}]->(b)
|
| 371 |
+
SET r = $props
|
| 372 |
+
`;
|
| 373 |
+
|
| 374 |
+
await session.run(query, {
|
| 375 |
+
startId: startNodeId,
|
| 376 |
+
endId: endNodeId,
|
| 377 |
+
props: rel.properties
|
| 378 |
+
});
|
| 379 |
+
|
| 380 |
+
imported++;
|
| 381 |
+
} catch (error: any) {
|
| 382 |
+
console.error(` β Failed to import relationship: ${error.message?.slice(0, 100)}`);
|
| 383 |
+
}
|
| 384 |
+
}
|
| 385 |
+
|
| 386 |
+
console.log(` β Imported ${imported} relationships (skipped ${skipped})`);
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
async function syncNeo4jToCloud() {
|
| 390 |
+
console.log('β'.repeat(60));
|
| 391 |
+
console.log('π Neo4j Local β Cloud Sync');
|
| 392 |
+
console.log('β'.repeat(60));
|
| 393 |
+
console.log(`\nLocal: ${config.local.uri}`);
|
| 394 |
+
console.log(`Cloud: ${config.cloud.uri}`);
|
| 395 |
+
console.log('');
|
| 396 |
+
|
| 397 |
+
let localDriver: Driver | null = null;
|
| 398 |
+
let cloudDriver: Driver | null = null;
|
| 399 |
+
|
| 400 |
+
try {
|
| 401 |
+
// Connect to local
|
| 402 |
+
console.log('π‘ Connecting to local Neo4j...');
|
| 403 |
+
localDriver = await createDriver(config.local);
|
| 404 |
+
await localDriver.verifyConnectivity();
|
| 405 |
+
console.log(' β Connected to local');
|
| 406 |
+
|
| 407 |
+
// Connect to cloud
|
| 408 |
+
console.log('βοΈ Connecting to AuraDB cloud...');
|
| 409 |
+
cloudDriver = await createDriver(config.cloud);
|
| 410 |
+
await cloudDriver.verifyConnectivity();
|
| 411 |
+
console.log(' β Connected to cloud');
|
| 412 |
+
|
| 413 |
+
// Export from local
|
| 414 |
+
const localSession = localDriver.session({ database: config.local.database });
|
| 415 |
+
|
| 416 |
+
const schema = await exportSchema(localSession);
|
| 417 |
+
const nodes = await exportNodes(localSession);
|
| 418 |
+
const relationships = await exportRelationships(localSession);
|
| 419 |
+
|
| 420 |
+
await localSession.close();
|
| 421 |
+
|
| 422 |
+
// Confirm before clearing cloud
|
| 423 |
+
console.log('\nβ οΈ This will REPLACE all data in cloud with local data!');
|
| 424 |
+
console.log(` Nodes to sync: ${nodes.length}`);
|
| 425 |
+
console.log(` Relationships to sync: ${relationships.length}`);
|
| 426 |
+
|
| 427 |
+
// Import to cloud
|
| 428 |
+
const cloudSession = cloudDriver.session({ database: config.cloud.database });
|
| 429 |
+
|
| 430 |
+
await clearCloud(cloudSession);
|
| 431 |
+
await applySchema(cloudSession, schema);
|
| 432 |
+
|
| 433 |
+
const elementIdMap = new Map<string, string>();
|
| 434 |
+
await importNodes(cloudSession, nodes, elementIdMap);
|
| 435 |
+
await importRelationships(cloudSession, relationships, elementIdMap, nodes);
|
| 436 |
+
|
| 437 |
+
await cloudSession.close();
|
| 438 |
+
|
| 439 |
+
// Verify sync
|
| 440 |
+
console.log('\nβ
Verifying sync...');
|
| 441 |
+
const verifySession = cloudDriver.session({ database: config.cloud.database });
|
| 442 |
+
|
| 443 |
+
const nodeCountResult = await verifySession.run('MATCH (n) RETURN count(n) as count');
|
| 444 |
+
const cloudNodeCount = nodeCountResult.records[0]?.get('count')?.toNumber() || 0;
|
| 445 |
+
|
| 446 |
+
const relCountResult = await verifySession.run('MATCH ()-[r]->() RETURN count(r) as count');
|
| 447 |
+
const cloudRelCount = relCountResult.records[0]?.get('count')?.toNumber() || 0;
|
| 448 |
+
|
| 449 |
+
await verifySession.close();
|
| 450 |
+
|
| 451 |
+
console.log(` Cloud nodes: ${cloudNodeCount} (expected: ${nodes.length})`);
|
| 452 |
+
console.log(` Cloud relationships: ${cloudRelCount} (expected: ${relationships.length})`);
|
| 453 |
+
|
| 454 |
+
console.log('\n' + 'β'.repeat(60));
|
| 455 |
+
console.log('β
SYNC COMPLETE');
|
| 456 |
+
console.log('β'.repeat(60));
|
| 457 |
+
|
| 458 |
+
} catch (error: any) {
|
| 459 |
+
console.error('\nβ Sync failed:', error.message);
|
| 460 |
+
throw error;
|
| 461 |
+
} finally {
|
| 462 |
+
if (localDriver) await localDriver.close();
|
| 463 |
+
if (cloudDriver) await cloudDriver.close();
|
| 464 |
+
}
|
| 465 |
+
}
|
| 466 |
+
|
| 467 |
+
// Run
|
| 468 |
+
syncNeo4jToCloud()
|
| 469 |
+
.then(() => process.exit(0))
|
| 470 |
+
.catch(() => process.exit(1));
|
apps/backend/src/services/Neo4jAutoSync.ts
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Neo4j Auto-Sync Service
|
| 3 |
+
*
|
| 4 |
+
* Automated synchronization between local Neo4j and AuraDB cloud.
|
| 5 |
+
* Supports full sync, incremental sync, and scheduled execution.
|
| 6 |
+
*/
|
| 7 |
+
|
| 8 |
+
import neo4j, { Driver, Session } from 'neo4j-driver';
|
| 9 |
+
import cron from 'node-cron';
|
| 10 |
+
import fs from 'fs';
|
| 11 |
+
import path from 'path';
|
| 12 |
+
|
| 13 |
+
export interface SyncConfig {
|
| 14 |
+
local: {
|
| 15 |
+
uri: string;
|
| 16 |
+
user: string;
|
| 17 |
+
password: string;
|
| 18 |
+
database: string;
|
| 19 |
+
};
|
| 20 |
+
cloud: {
|
| 21 |
+
uri: string;
|
| 22 |
+
user: string;
|
| 23 |
+
password: string;
|
| 24 |
+
database: string;
|
| 25 |
+
};
|
| 26 |
+
schedule?: string; // Cron expression
|
| 27 |
+
incrementalEnabled?: boolean;
|
| 28 |
+
batchSize?: number;
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
export interface SyncStatus {
|
| 32 |
+
lastSyncTime: string | null;
|
| 33 |
+
lastSyncType: 'full' | 'incremental' | null;
|
| 34 |
+
lastSyncDuration: number | null;
|
| 35 |
+
nodesSync: number;
|
| 36 |
+
relationshipsSync: number;
|
| 37 |
+
status: 'idle' | 'running' | 'success' | 'error';
|
| 38 |
+
error?: string;
|
| 39 |
+
nextScheduledSync?: string;
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
export interface SyncCheckpoint {
|
| 43 |
+
timestamp: string;
|
| 44 |
+
nodeCount: number;
|
| 45 |
+
relationshipCount: number;
|
| 46 |
+
lastNodeIds: string[];
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
const CHECKPOINT_FILE = path.join(process.cwd(), '.neo4j-sync-checkpoint.json');
|
| 50 |
+
const STATUS_FILE = path.join(process.cwd(), '.neo4j-sync-status.json');
|
| 51 |
+
|
| 52 |
+
export class Neo4jAutoSync {
|
| 53 |
+
private config: SyncConfig;
|
| 54 |
+
private localDriver: Driver | null = null;
|
| 55 |
+
private cloudDriver: Driver | null = null;
|
| 56 |
+
private cronJob: cron.ScheduledTask | null = null;
|
| 57 |
+
private status: SyncStatus = {
|
| 58 |
+
lastSyncTime: null,
|
| 59 |
+
lastSyncType: null,
|
| 60 |
+
lastSyncDuration: null,
|
| 61 |
+
nodesSync: 0,
|
| 62 |
+
relationshipsSync: 0,
|
| 63 |
+
status: 'idle'
|
| 64 |
+
};
|
| 65 |
+
|
| 66 |
+
constructor(config?: Partial<SyncConfig>) {
|
| 67 |
+
this.config = {
|
| 68 |
+
local: {
|
| 69 |
+
uri: config?.local?.uri || process.env.NEO4J_LOCAL_URI || 'bolt://localhost:7687',
|
| 70 |
+
user: config?.local?.user || process.env.NEO4J_LOCAL_USER || 'neo4j',
|
| 71 |
+
password: config?.local?.password || process.env.NEO4J_LOCAL_PASSWORD || 'password',
|
| 72 |
+
database: config?.local?.database || process.env.NEO4J_LOCAL_DATABASE || 'neo4j'
|
| 73 |
+
},
|
| 74 |
+
cloud: {
|
| 75 |
+
uri: config?.cloud?.uri || process.env.NEO4J_URI || '',
|
| 76 |
+
user: config?.cloud?.user || process.env.NEO4J_USER || 'neo4j',
|
| 77 |
+
password: config?.cloud?.password || process.env.NEO4J_PASSWORD || '',
|
| 78 |
+
database: config?.cloud?.database || process.env.NEO4J_DATABASE || 'neo4j'
|
| 79 |
+
},
|
| 80 |
+
schedule: config?.schedule || process.env.NEO4J_SYNC_SCHEDULE || '0 */6 * * *', // Every 6 hours
|
| 81 |
+
incrementalEnabled: config?.incrementalEnabled ?? true,
|
| 82 |
+
batchSize: config?.batchSize || 500
|
| 83 |
+
};
|
| 84 |
+
|
| 85 |
+
this.loadStatus();
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
private loadStatus(): void {
|
| 89 |
+
try {
|
| 90 |
+
if (fs.existsSync(STATUS_FILE)) {
|
| 91 |
+
const data = fs.readFileSync(STATUS_FILE, 'utf-8');
|
| 92 |
+
this.status = JSON.parse(data);
|
| 93 |
+
this.status.status = 'idle'; // Reset running status on restart
|
| 94 |
+
}
|
| 95 |
+
} catch {
|
| 96 |
+
// Ignore errors, use default status
|
| 97 |
+
}
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
private saveStatus(): void {
|
| 101 |
+
try {
|
| 102 |
+
fs.writeFileSync(STATUS_FILE, JSON.stringify(this.status, null, 2));
|
| 103 |
+
} catch (error) {
|
| 104 |
+
console.error('Failed to save sync status:', error);
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
private loadCheckpoint(): SyncCheckpoint | null {
|
| 109 |
+
try {
|
| 110 |
+
if (fs.existsSync(CHECKPOINT_FILE)) {
|
| 111 |
+
const data = fs.readFileSync(CHECKPOINT_FILE, 'utf-8');
|
| 112 |
+
return JSON.parse(data);
|
| 113 |
+
}
|
| 114 |
+
} catch {
|
| 115 |
+
// Ignore errors
|
| 116 |
+
}
|
| 117 |
+
return null;
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
private saveCheckpoint(checkpoint: SyncCheckpoint): void {
|
| 121 |
+
try {
|
| 122 |
+
fs.writeFileSync(CHECKPOINT_FILE, JSON.stringify(checkpoint, null, 2));
|
| 123 |
+
} catch (error) {
|
| 124 |
+
console.error('Failed to save checkpoint:', error);
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
async connect(): Promise<void> {
|
| 129 |
+
console.log('π‘ Connecting to Neo4j instances...');
|
| 130 |
+
|
| 131 |
+
this.localDriver = neo4j.driver(
|
| 132 |
+
this.config.local.uri,
|
| 133 |
+
neo4j.auth.basic(this.config.local.user, this.config.local.password),
|
| 134 |
+
{ maxConnectionLifetime: 3 * 60 * 60 * 1000 }
|
| 135 |
+
);
|
| 136 |
+
await this.localDriver.verifyConnectivity();
|
| 137 |
+
console.log(' β Connected to local Neo4j');
|
| 138 |
+
|
| 139 |
+
this.cloudDriver = neo4j.driver(
|
| 140 |
+
this.config.cloud.uri,
|
| 141 |
+
neo4j.auth.basic(this.config.cloud.user, this.config.cloud.password),
|
| 142 |
+
{ maxConnectionLifetime: 3 * 60 * 60 * 1000 }
|
| 143 |
+
);
|
| 144 |
+
await this.cloudDriver.verifyConnectivity();
|
| 145 |
+
console.log(' β Connected to AuraDB cloud');
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
async disconnect(): Promise<void> {
|
| 149 |
+
if (this.localDriver) await this.localDriver.close();
|
| 150 |
+
if (this.cloudDriver) await this.cloudDriver.close();
|
| 151 |
+
this.localDriver = null;
|
| 152 |
+
this.cloudDriver = null;
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
/**
|
| 156 |
+
* Start scheduled auto-sync
|
| 157 |
+
*/
|
| 158 |
+
startScheduler(): void {
|
| 159 |
+
if (this.cronJob) {
|
| 160 |
+
this.cronJob.stop();
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
console.log(`β° Starting Neo4j auto-sync scheduler: ${this.config.schedule}`);
|
| 164 |
+
|
| 165 |
+
this.cronJob = cron.schedule(this.config.schedule!, async () => {
|
| 166 |
+
console.log(`\nπ [${new Date().toISOString()}] Scheduled sync starting...`);
|
| 167 |
+
try {
|
| 168 |
+
await this.sync();
|
| 169 |
+
} catch (error) {
|
| 170 |
+
console.error('Scheduled sync failed:', error);
|
| 171 |
+
}
|
| 172 |
+
});
|
| 173 |
+
|
| 174 |
+
// Calculate next run time
|
| 175 |
+
const nextRun = this.getNextScheduledTime();
|
| 176 |
+
this.status.nextScheduledSync = nextRun;
|
| 177 |
+
this.saveStatus();
|
| 178 |
+
|
| 179 |
+
console.log(` Next sync: ${nextRun}`);
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
stopScheduler(): void {
|
| 183 |
+
if (this.cronJob) {
|
| 184 |
+
this.cronJob.stop();
|
| 185 |
+
this.cronJob = null;
|
| 186 |
+
console.log('βΉοΈ Scheduler stopped');
|
| 187 |
+
}
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
private getNextScheduledTime(): string {
|
| 191 |
+
// Simple calculation for common cron patterns
|
| 192 |
+
const now = new Date();
|
| 193 |
+
const schedule = this.config.schedule!;
|
| 194 |
+
|
| 195 |
+
if (schedule.includes('*/6')) {
|
| 196 |
+
// Every 6 hours
|
| 197 |
+
const nextHour = Math.ceil(now.getHours() / 6) * 6;
|
| 198 |
+
const next = new Date(now);
|
| 199 |
+
next.setHours(nextHour, 0, 0, 0);
|
| 200 |
+
if (next <= now) next.setHours(next.getHours() + 6);
|
| 201 |
+
return next.toISOString();
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
return 'See cron expression: ' + schedule;
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
/**
|
| 208 |
+
* Perform sync (auto-detects full vs incremental)
|
| 209 |
+
*/
|
| 210 |
+
async sync(forceFullSync: boolean = false): Promise<SyncStatus> {
|
| 211 |
+
const startTime = Date.now();
|
| 212 |
+
this.status.status = 'running';
|
| 213 |
+
this.saveStatus();
|
| 214 |
+
|
| 215 |
+
try {
|
| 216 |
+
await this.connect();
|
| 217 |
+
|
| 218 |
+
const checkpoint = this.loadCheckpoint();
|
| 219 |
+
const shouldDoFullSync = forceFullSync || !checkpoint || !this.config.incrementalEnabled;
|
| 220 |
+
|
| 221 |
+
if (shouldDoFullSync) {
|
| 222 |
+
await this.fullSync();
|
| 223 |
+
this.status.lastSyncType = 'full';
|
| 224 |
+
} else {
|
| 225 |
+
const hasChanges = await this.checkForChanges(checkpoint);
|
| 226 |
+
if (hasChanges) {
|
| 227 |
+
await this.incrementalSync(checkpoint);
|
| 228 |
+
this.status.lastSyncType = 'incremental';
|
| 229 |
+
} else {
|
| 230 |
+
console.log('β No changes detected, skipping sync');
|
| 231 |
+
this.status.lastSyncType = 'incremental';
|
| 232 |
+
}
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
this.status.status = 'success';
|
| 236 |
+
this.status.lastSyncTime = new Date().toISOString();
|
| 237 |
+
this.status.lastSyncDuration = Date.now() - startTime;
|
| 238 |
+
delete this.status.error;
|
| 239 |
+
|
| 240 |
+
} catch (error: any) {
|
| 241 |
+
this.status.status = 'error';
|
| 242 |
+
this.status.error = error.message;
|
| 243 |
+
console.error('β Sync failed:', error.message);
|
| 244 |
+
} finally {
|
| 245 |
+
await this.disconnect();
|
| 246 |
+
this.saveStatus();
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
return this.status;
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
private async checkForChanges(checkpoint: SyncCheckpoint): Promise<boolean> {
|
| 253 |
+
const session = this.localDriver!.session({ database: this.config.local.database });
|
| 254 |
+
try {
|
| 255 |
+
// Check node count
|
| 256 |
+
const nodeResult = await session.run('MATCH (n) RETURN count(n) as count');
|
| 257 |
+
const currentNodeCount = nodeResult.records[0].get('count').toNumber();
|
| 258 |
+
|
| 259 |
+
// Check relationship count
|
| 260 |
+
const relResult = await session.run('MATCH ()-[r]->() RETURN count(r) as count');
|
| 261 |
+
const currentRelCount = relResult.records[0].get('count').toNumber();
|
| 262 |
+
|
| 263 |
+
const hasChanges = currentNodeCount !== checkpoint.nodeCount ||
|
| 264 |
+
currentRelCount !== checkpoint.relationshipCount;
|
| 265 |
+
|
| 266 |
+
if (hasChanges) {
|
| 267 |
+
console.log(`π Changes detected: nodes ${checkpoint.nodeCount} β ${currentNodeCount}, rels ${checkpoint.relationshipCount} β ${currentRelCount}`);
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
return hasChanges;
|
| 271 |
+
} finally {
|
| 272 |
+
await session.close();
|
| 273 |
+
}
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
private async fullSync(): Promise<void> {
|
| 277 |
+
console.log('\nπ Starting FULL sync...');
|
| 278 |
+
|
| 279 |
+
const localSession = this.localDriver!.session({ database: this.config.local.database });
|
| 280 |
+
const cloudSession = this.cloudDriver!.session({ database: this.config.cloud.database });
|
| 281 |
+
|
| 282 |
+
try {
|
| 283 |
+
// Export schema
|
| 284 |
+
const schema = await this.exportSchema(localSession);
|
| 285 |
+
|
| 286 |
+
// Export nodes
|
| 287 |
+
const nodes = await this.exportNodes(localSession);
|
| 288 |
+
|
| 289 |
+
// Export relationships
|
| 290 |
+
const relationships = await this.exportRelationships(localSession);
|
| 291 |
+
|
| 292 |
+
// Clear cloud
|
| 293 |
+
await this.clearCloud(cloudSession);
|
| 294 |
+
|
| 295 |
+
// Apply schema
|
| 296 |
+
await this.applySchema(cloudSession, schema);
|
| 297 |
+
|
| 298 |
+
// Import nodes
|
| 299 |
+
const elementIdMap = new Map<string, string>();
|
| 300 |
+
await this.importNodes(cloudSession, nodes, elementIdMap);
|
| 301 |
+
|
| 302 |
+
// Import relationships
|
| 303 |
+
await this.importRelationships(cloudSession, relationships, elementIdMap, nodes);
|
| 304 |
+
|
| 305 |
+
// Save checkpoint
|
| 306 |
+
this.saveCheckpoint({
|
| 307 |
+
timestamp: new Date().toISOString(),
|
| 308 |
+
nodeCount: nodes.length,
|
| 309 |
+
relationshipCount: relationships.length,
|
| 310 |
+
lastNodeIds: nodes.slice(-100).map(n => n.properties.id)
|
| 311 |
+
});
|
| 312 |
+
|
| 313 |
+
this.status.nodesSync = nodes.length;
|
| 314 |
+
this.status.relationshipsSync = relationships.length;
|
| 315 |
+
|
| 316 |
+
console.log(`\nβ
Full sync complete: ${nodes.length} nodes, ${relationships.length} relationships`);
|
| 317 |
+
|
| 318 |
+
} finally {
|
| 319 |
+
await localSession.close();
|
| 320 |
+
await cloudSession.close();
|
| 321 |
+
}
|
| 322 |
+
}
|
| 323 |
+
|
| 324 |
+
private async incrementalSync(checkpoint: SyncCheckpoint): Promise<void> {
|
| 325 |
+
console.log('\nπ Starting INCREMENTAL sync...');
|
| 326 |
+
|
| 327 |
+
const localSession = this.localDriver!.session({ database: this.config.local.database });
|
| 328 |
+
const cloudSession = this.cloudDriver!.session({ database: this.config.cloud.database });
|
| 329 |
+
|
| 330 |
+
try {
|
| 331 |
+
// Find new/modified nodes since checkpoint
|
| 332 |
+
const checkpointTime = new Date(checkpoint.timestamp);
|
| 333 |
+
|
| 334 |
+
// Get nodes created/modified after checkpoint
|
| 335 |
+
// This requires nodes to have a 'updatedAt' or 'createdAt' property
|
| 336 |
+
const newNodesResult = await localSession.run(`
|
| 337 |
+
MATCH (n)
|
| 338 |
+
WHERE n.createdAt > datetime($checkpointTime)
|
| 339 |
+
OR n.updatedAt > datetime($checkpointTime)
|
| 340 |
+
OR NOT n.id IN $existingIds
|
| 341 |
+
RETURN n, elementId(n) as elementId
|
| 342 |
+
LIMIT 10000
|
| 343 |
+
`, {
|
| 344 |
+
checkpointTime: checkpointTime.toISOString(),
|
| 345 |
+
existingIds: checkpoint.lastNodeIds
|
| 346 |
+
});
|
| 347 |
+
|
| 348 |
+
const newNodes: any[] = [];
|
| 349 |
+
for (const record of newNodesResult.records) {
|
| 350 |
+
const node = record.get('n');
|
| 351 |
+
newNodes.push({
|
| 352 |
+
labels: [...node.labels],
|
| 353 |
+
properties: this.convertProperties(node.properties),
|
| 354 |
+
elementId: record.get('elementId')
|
| 355 |
+
});
|
| 356 |
+
}
|
| 357 |
+
|
| 358 |
+
if (newNodes.length === 0) {
|
| 359 |
+
console.log(' No new nodes to sync');
|
| 360 |
+
return;
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
console.log(` Found ${newNodes.length} new/modified nodes`);
|
| 364 |
+
|
| 365 |
+
// Import new nodes
|
| 366 |
+
const elementIdMap = new Map<string, string>();
|
| 367 |
+
await this.importNodes(cloudSession, newNodes, elementIdMap);
|
| 368 |
+
|
| 369 |
+
// Get new relationships for these nodes
|
| 370 |
+
const nodeIds = newNodes.map(n => n.properties.id).filter(Boolean);
|
| 371 |
+
if (nodeIds.length > 0) {
|
| 372 |
+
const newRelsResult = await localSession.run(`
|
| 373 |
+
MATCH (a)-[r]->(b)
|
| 374 |
+
WHERE a.id IN $nodeIds OR b.id IN $nodeIds
|
| 375 |
+
RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
|
| 376 |
+
`, { nodeIds });
|
| 377 |
+
|
| 378 |
+
const newRels: any[] = [];
|
| 379 |
+
for (const record of newRelsResult.records) {
|
| 380 |
+
const rel = record.get('r');
|
| 381 |
+
newRels.push({
|
| 382 |
+
type: record.get('type'),
|
| 383 |
+
properties: this.convertProperties(rel.properties),
|
| 384 |
+
startElementId: record.get('startId'),
|
| 385 |
+
endElementId: record.get('endId')
|
| 386 |
+
});
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
if (newRels.length > 0) {
|
| 390 |
+
await this.importRelationships(cloudSession, newRels, elementIdMap, newNodes);
|
| 391 |
+
}
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
// Update checkpoint
|
| 395 |
+
const nodeCountResult = await localSession.run('MATCH (n) RETURN count(n) as count');
|
| 396 |
+
const relCountResult = await localSession.run('MATCH ()-[r]->() RETURN count(r) as count');
|
| 397 |
+
|
| 398 |
+
this.saveCheckpoint({
|
| 399 |
+
timestamp: new Date().toISOString(),
|
| 400 |
+
nodeCount: nodeCountResult.records[0].get('count').toNumber(),
|
| 401 |
+
relationshipCount: relCountResult.records[0].get('count').toNumber(),
|
| 402 |
+
lastNodeIds: [...checkpoint.lastNodeIds, ...nodeIds].slice(-100)
|
| 403 |
+
});
|
| 404 |
+
|
| 405 |
+
this.status.nodesSync = newNodes.length;
|
| 406 |
+
this.status.relationshipsSync = 0;
|
| 407 |
+
|
| 408 |
+
console.log(`β
Incremental sync complete: ${newNodes.length} nodes`);
|
| 409 |
+
|
| 410 |
+
} finally {
|
| 411 |
+
await localSession.close();
|
| 412 |
+
await cloudSession.close();
|
| 413 |
+
}
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
private convertProperties(props: Record<string, any>): Record<string, any> {
|
| 417 |
+
const result: Record<string, any> = {};
|
| 418 |
+
for (const [key, value] of Object.entries(props)) {
|
| 419 |
+
if (value === null || value === undefined) {
|
| 420 |
+
result[key] = value;
|
| 421 |
+
} else if (typeof value === 'object' && value.constructor?.name === 'Integer') {
|
| 422 |
+
result[key] = value.toNumber();
|
| 423 |
+
} else if (typeof value === 'object' && (value.constructor?.name === 'DateTime' || value.constructor?.name === 'Date')) {
|
| 424 |
+
result[key] = value.toString();
|
| 425 |
+
} else if (Array.isArray(value)) {
|
| 426 |
+
result[key] = value.map(v => this.convertProperties({ v }).v);
|
| 427 |
+
} else {
|
| 428 |
+
result[key] = value;
|
| 429 |
+
}
|
| 430 |
+
}
|
| 431 |
+
return result;
|
| 432 |
+
}
|
| 433 |
+
|
| 434 |
+
private async exportSchema(session: Session): Promise<any[]> {
|
| 435 |
+
const schema: any[] = [];
|
| 436 |
+
|
| 437 |
+
const constraintResult = await session.run('SHOW CONSTRAINTS');
|
| 438 |
+
for (const record of constraintResult.records) {
|
| 439 |
+
const name = record.get('name');
|
| 440 |
+
const type = record.get('type');
|
| 441 |
+
const entityType = record.get('entityType');
|
| 442 |
+
const labelsOrTypes = record.get('labelsOrTypes');
|
| 443 |
+
const properties = record.get('properties');
|
| 444 |
+
|
| 445 |
+
if (type === 'UNIQUENESS' && entityType === 'NODE') {
|
| 446 |
+
const label = labelsOrTypes?.[0];
|
| 447 |
+
const prop = properties?.[0];
|
| 448 |
+
if (label && prop) {
|
| 449 |
+
schema.push({
|
| 450 |
+
type: 'constraint',
|
| 451 |
+
name,
|
| 452 |
+
definition: `CREATE CONSTRAINT ${name} IF NOT EXISTS FOR (n:${label}) REQUIRE n.${prop} IS UNIQUE`
|
| 453 |
+
});
|
| 454 |
+
}
|
| 455 |
+
}
|
| 456 |
+
}
|
| 457 |
+
|
| 458 |
+
const indexResult = await session.run('SHOW INDEXES');
|
| 459 |
+
for (const record of indexResult.records) {
|
| 460 |
+
const name = record.get('name');
|
| 461 |
+
const type = record.get('type');
|
| 462 |
+
const entityType = record.get('entityType');
|
| 463 |
+
const labelsOrTypes = record.get('labelsOrTypes');
|
| 464 |
+
const properties = record.get('properties');
|
| 465 |
+
const owningConstraint = record.get('owningConstraint');
|
| 466 |
+
|
| 467 |
+
if (owningConstraint) continue;
|
| 468 |
+
|
| 469 |
+
if (type === 'RANGE' && entityType === 'NODE') {
|
| 470 |
+
const label = labelsOrTypes?.[0];
|
| 471 |
+
const prop = properties?.[0];
|
| 472 |
+
if (label && prop) {
|
| 473 |
+
schema.push({
|
| 474 |
+
type: 'index',
|
| 475 |
+
name,
|
| 476 |
+
definition: `CREATE INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${prop})`
|
| 477 |
+
});
|
| 478 |
+
}
|
| 479 |
+
}
|
| 480 |
+
}
|
| 481 |
+
|
| 482 |
+
return schema;
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
private async exportNodes(session: Session): Promise<any[]> {
|
| 486 |
+
console.log('π¦ Exporting nodes...');
|
| 487 |
+
const result = await session.run('MATCH (n) RETURN n, elementId(n) as elementId');
|
| 488 |
+
const nodes = result.records.map(record => ({
|
| 489 |
+
labels: [...record.get('n').labels],
|
| 490 |
+
properties: this.convertProperties(record.get('n').properties),
|
| 491 |
+
elementId: record.get('elementId')
|
| 492 |
+
}));
|
| 493 |
+
console.log(` β Exported ${nodes.length} nodes`);
|
| 494 |
+
return nodes;
|
| 495 |
+
}
|
| 496 |
+
|
| 497 |
+
private async exportRelationships(session: Session): Promise<any[]> {
|
| 498 |
+
console.log('π Exporting relationships...');
|
| 499 |
+
const result = await session.run(`
|
| 500 |
+
MATCH (a)-[r]->(b)
|
| 501 |
+
RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
|
| 502 |
+
`);
|
| 503 |
+
const rels = result.records.map(record => ({
|
| 504 |
+
type: record.get('type'),
|
| 505 |
+
properties: this.convertProperties(record.get('r').properties),
|
| 506 |
+
startElementId: record.get('startId'),
|
| 507 |
+
endElementId: record.get('endId')
|
| 508 |
+
}));
|
| 509 |
+
console.log(` β Exported ${rels.length} relationships`);
|
| 510 |
+
return rels;
|
| 511 |
+
}
|
| 512 |
+
|
| 513 |
+
private async clearCloud(session: Session): Promise<void> {
|
| 514 |
+
console.log('ποΈ Clearing cloud database...');
|
| 515 |
+
let deleted = 0;
|
| 516 |
+
do {
|
| 517 |
+
const result = await session.run(`
|
| 518 |
+
MATCH (n) WITH n LIMIT 1000 DETACH DELETE n RETURN count(*) as deleted
|
| 519 |
+
`);
|
| 520 |
+
deleted = result.records[0]?.get('deleted')?.toNumber() || 0;
|
| 521 |
+
} while (deleted > 0);
|
| 522 |
+
console.log(' β Cloud cleared');
|
| 523 |
+
}
|
| 524 |
+
|
| 525 |
+
private async applySchema(session: Session, schema: any[]): Promise<void> {
|
| 526 |
+
console.log('π Applying schema...');
|
| 527 |
+
for (const item of schema) {
|
| 528 |
+
try {
|
| 529 |
+
await session.run(item.definition);
|
| 530 |
+
} catch (error: any) {
|
| 531 |
+
if (!error.message?.includes('already exists')) {
|
| 532 |
+
console.warn(` β ${item.name}: ${error.message?.slice(0, 50)}`);
|
| 533 |
+
}
|
| 534 |
+
}
|
| 535 |
+
}
|
| 536 |
+
console.log(` β Applied ${schema.length} schema items`);
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
private async importNodes(session: Session, nodes: any[], elementIdMap: Map<string, string>): Promise<void> {
|
| 540 |
+
console.log('π¦ Importing nodes...');
|
| 541 |
+
let imported = 0;
|
| 542 |
+
const batchSize = this.config.batchSize!;
|
| 543 |
+
|
| 544 |
+
for (let i = 0; i < nodes.length; i += batchSize) {
|
| 545 |
+
const batch = nodes.slice(i, i + batchSize);
|
| 546 |
+
|
| 547 |
+
for (const node of batch) {
|
| 548 |
+
try {
|
| 549 |
+
const labelStr = node.labels.join(':');
|
| 550 |
+
const nodeId = node.properties.id || `node_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
| 551 |
+
node.properties.id = nodeId;
|
| 552 |
+
|
| 553 |
+
const result = await session.run(`
|
| 554 |
+
MERGE (n:${labelStr} {id: $nodeId})
|
| 555 |
+
SET n = $props
|
| 556 |
+
RETURN elementId(n) as newElementId
|
| 557 |
+
`, { nodeId, props: node.properties });
|
| 558 |
+
|
| 559 |
+
const newElementId = result.records[0]?.get('newElementId');
|
| 560 |
+
if (newElementId) {
|
| 561 |
+
elementIdMap.set(node.elementId, newElementId);
|
| 562 |
+
}
|
| 563 |
+
imported++;
|
| 564 |
+
} catch (error: any) {
|
| 565 |
+
// Continue on error
|
| 566 |
+
}
|
| 567 |
+
}
|
| 568 |
+
|
| 569 |
+
if ((i + batchSize) % 5000 === 0 || i + batchSize >= nodes.length) {
|
| 570 |
+
console.log(` Progress: ${Math.min(i + batchSize, nodes.length)}/${nodes.length} nodes`);
|
| 571 |
+
}
|
| 572 |
+
}
|
| 573 |
+
|
| 574 |
+
console.log(` β Imported ${imported} nodes`);
|
| 575 |
+
}
|
| 576 |
+
|
| 577 |
+
private async importRelationships(
|
| 578 |
+
session: Session,
|
| 579 |
+
relationships: any[],
|
| 580 |
+
elementIdMap: Map<string, string>,
|
| 581 |
+
nodes: any[]
|
| 582 |
+
): Promise<void> {
|
| 583 |
+
console.log('π Importing relationships...');
|
| 584 |
+
|
| 585 |
+
const elementIdToNodeId = new Map<string, string>();
|
| 586 |
+
for (const node of nodes) {
|
| 587 |
+
elementIdToNodeId.set(node.elementId, node.properties.id);
|
| 588 |
+
}
|
| 589 |
+
|
| 590 |
+
let imported = 0;
|
| 591 |
+
const batchSize = this.config.batchSize!;
|
| 592 |
+
|
| 593 |
+
for (let i = 0; i < relationships.length; i += batchSize) {
|
| 594 |
+
const batch = relationships.slice(i, i + batchSize);
|
| 595 |
+
|
| 596 |
+
for (const rel of batch) {
|
| 597 |
+
try {
|
| 598 |
+
const startNodeId = elementIdToNodeId.get(rel.startElementId);
|
| 599 |
+
const endNodeId = elementIdToNodeId.get(rel.endElementId);
|
| 600 |
+
|
| 601 |
+
if (!startNodeId || !endNodeId) continue;
|
| 602 |
+
|
| 603 |
+
await session.run(`
|
| 604 |
+
MATCH (a {id: $startId})
|
| 605 |
+
MATCH (b {id: $endId})
|
| 606 |
+
MERGE (a)-[r:${rel.type}]->(b)
|
| 607 |
+
SET r = $props
|
| 608 |
+
`, { startId: startNodeId, endId: endNodeId, props: rel.properties });
|
| 609 |
+
|
| 610 |
+
imported++;
|
| 611 |
+
} catch (error: any) {
|
| 612 |
+
// Continue on error
|
| 613 |
+
}
|
| 614 |
+
}
|
| 615 |
+
|
| 616 |
+
if ((i + batchSize) % 50000 === 0 || i + batchSize >= relationships.length) {
|
| 617 |
+
console.log(` Progress: ${Math.min(i + batchSize, relationships.length)}/${relationships.length} relationships`);
|
| 618 |
+
}
|
| 619 |
+
}
|
| 620 |
+
|
| 621 |
+
console.log(` β Imported ${imported} relationships`);
|
| 622 |
+
}
|
| 623 |
+
|
| 624 |
+
getStatus(): SyncStatus {
|
| 625 |
+
return { ...this.status };
|
| 626 |
+
}
|
| 627 |
+
}
|
| 628 |
+
|
| 629 |
+
// Singleton instance
|
| 630 |
+
let autoSyncInstance: Neo4jAutoSync | null = null;
|
| 631 |
+
|
| 632 |
+
export function getNeo4jAutoSync(): Neo4jAutoSync {
|
| 633 |
+
if (!autoSyncInstance) {
|
| 634 |
+
autoSyncInstance = new Neo4jAutoSync();
|
| 635 |
+
}
|
| 636 |
+
return autoSyncInstance;
|
| 637 |
+
}
|