Kraft102 commited on
Commit
bbf18eb
Β·
verified Β·
1 Parent(s): e2322d8

Deploy from GitHub Actions 2025-12-15_23-41-14

Browse files
apps/backend/package.json CHANGED
@@ -13,7 +13,12 @@
13
  "lint": "eslint .",
14
  "neural-bridge": "tsx src/mcp/servers/NeuralBridgeServer.ts",
15
  "neural-bridge:build": "tsc && node dist/mcp/servers/NeuralBridgeServer.js",
16
- "ingest-drive": "tsx src/scripts/ingest-drive.ts"
 
 
 
 
 
17
  },
18
  "dependencies": {
19
  "@anthropic-ai/sdk": "^0.71.0",
 
13
  "lint": "eslint .",
14
  "neural-bridge": "tsx src/mcp/servers/NeuralBridgeServer.ts",
15
  "neural-bridge:build": "tsc && node dist/mcp/servers/NeuralBridgeServer.js",
16
+ "ingest-drive": "tsx src/scripts/ingest-drive.ts",
17
+ "sync-neo4j": "tsx src/scripts/sync-neo4j-to-cloud.ts",
18
+ "sync:start": "tsx src/scripts/neo4j-auto-sync.ts start",
19
+ "sync:now": "tsx src/scripts/neo4j-auto-sync.ts sync",
20
+ "sync:full": "tsx src/scripts/neo4j-auto-sync.ts full",
21
+ "sync:status": "tsx src/scripts/neo4j-auto-sync.ts status"
22
  },
23
  "dependencies": {
24
  "@anthropic-ai/sdk": "^0.71.0",
apps/backend/src/index.ts CHANGED
@@ -615,6 +615,11 @@ async function startServer() {
615
  app.use('/api/email', emailRoutes);
616
  console.log('πŸ“§ Email API mounted at /api/email');
617
 
 
 
 
 
 
618
  // Start KnowledgeCompiler auto-compilation (every 60 seconds)
619
  const { knowledgeCompiler } = await import('./services/Knowledge/index.js');
620
  knowledgeCompiler.startAutoCompilation(60000);
 
615
  app.use('/api/email', emailRoutes);
616
  console.log('πŸ“§ Email API mounted at /api/email');
617
 
618
+ // Neo4j Sync API - Auto-sync between local and cloud
619
+ const neo4jSyncRoutes = (await import('./routes/neo4j-sync.js')).default;
620
+ app.use('/api/neo4j-sync', neo4jSyncRoutes);
621
+ console.log('πŸ”„ Neo4j Sync API mounted at /api/neo4j-sync');
622
+
623
  // Start KnowledgeCompiler auto-compilation (every 60 seconds)
624
  const { knowledgeCompiler } = await import('./services/Knowledge/index.js');
625
  knowledgeCompiler.startAutoCompilation(60000);
apps/backend/src/routes/neo4j-sync.ts ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Neo4j Sync API Routes
3
+ *
4
+ * Provides REST endpoints for Neo4j sync status and control.
5
+ */
6
+
7
+ import { Router, Request, Response } from 'express';
8
+ import { getNeo4jAutoSync } from '../services/Neo4jAutoSync.js';
9
+
10
+ const router = Router();
11
+
12
+ /**
13
+ * GET /api/neo4j-sync/status
14
+ * Get current sync status
15
+ */
16
+ router.get('/status', (_req: Request, res: Response) => {
17
+ try {
18
+ const syncService = getNeo4jAutoSync();
19
+ const status = syncService.getStatus();
20
+ res.json({
21
+ success: true,
22
+ data: status
23
+ });
24
+ } catch (error: any) {
25
+ res.status(500).json({
26
+ success: false,
27
+ error: error.message
28
+ });
29
+ }
30
+ });
31
+
32
+ /**
33
+ * POST /api/neo4j-sync/sync
34
+ * Trigger a sync (incremental by default)
35
+ */
36
+ router.post('/sync', async (req: Request, res: Response) => {
37
+ try {
38
+ const { full = false } = req.body || {};
39
+ const syncService = getNeo4jAutoSync();
40
+
41
+ // Return immediately, sync runs in background
42
+ res.json({
43
+ success: true,
44
+ message: `${full ? 'Full' : 'Incremental'} sync started`,
45
+ data: { type: full ? 'full' : 'incremental' }
46
+ });
47
+
48
+ // Run sync in background
49
+ syncService.sync(full).catch(err => {
50
+ console.error('Background sync failed:', err);
51
+ });
52
+
53
+ } catch (error: any) {
54
+ res.status(500).json({
55
+ success: false,
56
+ error: error.message
57
+ });
58
+ }
59
+ });
60
+
61
+ /**
62
+ * POST /api/neo4j-sync/scheduler/start
63
+ * Start the scheduler
64
+ */
65
+ router.post('/scheduler/start', (_req: Request, res: Response) => {
66
+ try {
67
+ const syncService = getNeo4jAutoSync();
68
+ syncService.startScheduler();
69
+ res.json({
70
+ success: true,
71
+ message: 'Scheduler started',
72
+ data: syncService.getStatus()
73
+ });
74
+ } catch (error: any) {
75
+ res.status(500).json({
76
+ success: false,
77
+ error: error.message
78
+ });
79
+ }
80
+ });
81
+
82
+ /**
83
+ * POST /api/neo4j-sync/scheduler/stop
84
+ * Stop the scheduler
85
+ */
86
+ router.post('/scheduler/stop', (_req: Request, res: Response) => {
87
+ try {
88
+ const syncService = getNeo4jAutoSync();
89
+ syncService.stopScheduler();
90
+ res.json({
91
+ success: true,
92
+ message: 'Scheduler stopped'
93
+ });
94
+ } catch (error: any) {
95
+ res.status(500).json({
96
+ success: false,
97
+ error: error.message
98
+ });
99
+ }
100
+ });
101
+
102
+ export default router;
apps/backend/src/scripts/neo4j-auto-sync.ts ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Neo4j Auto-Sync Runner
4
+ *
5
+ * Runs as a standalone service for automated Neo4j synchronization.
6
+ *
7
+ * Usage:
8
+ * npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts [command]
9
+ *
10
+ * Commands:
11
+ * start - Start scheduler (default)
12
+ * sync - Run single sync now
13
+ * full - Run full sync (ignores checkpoint)
14
+ * status - Show sync status
15
+ * help - Show this help
16
+ *
17
+ * Environment Variables:
18
+ * NEO4J_LOCAL_URI - Local Neo4j URI (default: bolt://localhost:7687)
19
+ * NEO4J_LOCAL_USER - Local Neo4j user (default: neo4j)
20
+ * NEO4J_LOCAL_PASSWORD - Local Neo4j password (default: password)
21
+ * NEO4J_URI - Cloud Neo4j URI
22
+ * NEO4J_USER - Cloud Neo4j user
23
+ * NEO4J_PASSWORD - Cloud Neo4j password
24
+ * NEO4J_SYNC_SCHEDULE - Cron schedule (default: "0 *\/6 * * *" = every 6 hours)
25
+ */
26
+
27
+ import dotenv from 'dotenv';
28
+ import path from 'path';
29
+ import fs from 'fs';
30
+ import { Neo4jAutoSync } from '../services/Neo4jAutoSync.js';
31
+
32
+ // Load environment
33
+ dotenv.config({ path: path.resolve(process.cwd(), '.env.production') });
34
+ dotenv.config({ path: path.resolve(process.cwd(), '.env') });
35
+
36
+ const STATUS_FILE = path.join(process.cwd(), '.neo4j-sync-status.json');
37
+
38
+ async function main() {
39
+ const command = process.argv[2] || 'start';
40
+
41
+ console.log('═'.repeat(60));
42
+ console.log('πŸ”„ Neo4j Auto-Sync Service');
43
+ console.log('═'.repeat(60));
44
+ console.log('');
45
+
46
+ switch (command) {
47
+ case 'start':
48
+ await startScheduler();
49
+ break;
50
+
51
+ case 'sync':
52
+ await runSync(false);
53
+ break;
54
+
55
+ case 'full':
56
+ await runSync(true);
57
+ break;
58
+
59
+ case 'status':
60
+ showStatus();
61
+ break;
62
+
63
+ case 'help':
64
+ default:
65
+ showHelp();
66
+ break;
67
+ }
68
+ }
69
+
70
+ async function startScheduler() {
71
+ const syncService = new Neo4jAutoSync();
72
+ const schedule = process.env.NEO4J_SYNC_SCHEDULE || '0 */6 * * *';
73
+
74
+ console.log('Configuration:');
75
+ console.log(` Local: ${process.env.NEO4J_LOCAL_URI || 'bolt://localhost:7687'}`);
76
+ console.log(` Cloud: ${process.env.NEO4J_URI || '(not configured)'}`);
77
+ console.log(` Schedule: ${schedule}`);
78
+ console.log('');
79
+
80
+ // Run initial sync
81
+ console.log('Running initial sync...\n');
82
+ await syncService.sync();
83
+
84
+ // Start scheduler
85
+ syncService.startScheduler();
86
+
87
+ console.log('\nβœ… Auto-sync service running. Press Ctrl+C to stop.\n');
88
+
89
+ // Keep process alive
90
+ process.on('SIGINT', () => {
91
+ console.log('\n⏹️ Shutting down...');
92
+ syncService.stopScheduler();
93
+ process.exit(0);
94
+ });
95
+
96
+ // Keep alive
97
+ setInterval(() => {
98
+ // Heartbeat
99
+ }, 60000);
100
+ }
101
+
102
+ async function runSync(forceFullSync: boolean) {
103
+ const syncService = new Neo4jAutoSync();
104
+
105
+ console.log(`Running ${forceFullSync ? 'FULL' : 'incremental'} sync...\n`);
106
+
107
+ const status = await syncService.sync(forceFullSync);
108
+
109
+ console.log('\n' + '═'.repeat(60));
110
+ console.log('Sync Result:');
111
+ console.log(` Status: ${status.status}`);
112
+ console.log(` Type: ${status.lastSyncType}`);
113
+ console.log(` Nodes: ${status.nodesSync}`);
114
+ console.log(` Relationships: ${status.relationshipsSync}`);
115
+ console.log(` Duration: ${status.lastSyncDuration}ms`);
116
+ if (status.error) {
117
+ console.log(` Error: ${status.error}`);
118
+ }
119
+ console.log('═'.repeat(60));
120
+ }
121
+
122
+ function showStatus() {
123
+ if (fs.existsSync(STATUS_FILE)) {
124
+ const status = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf-8'));
125
+ console.log('Current Sync Status:');
126
+ console.log('');
127
+ console.log(` Status: ${status.status}`);
128
+ console.log(` Last Sync: ${status.lastSyncTime || 'Never'}`);
129
+ console.log(` Sync Type: ${status.lastSyncType || 'N/A'}`);
130
+ console.log(` Nodes Synced: ${status.nodesSync || 0}`);
131
+ console.log(` Rels Synced: ${status.relationshipsSync || 0}`);
132
+ console.log(` Duration: ${status.lastSyncDuration ? status.lastSyncDuration + 'ms' : 'N/A'}`);
133
+ console.log(` Next Scheduled: ${status.nextScheduledSync || 'N/A'}`);
134
+ if (status.error) {
135
+ console.log(` Error: ${status.error}`);
136
+ }
137
+ } else {
138
+ console.log('No sync status found. Run a sync first.');
139
+ }
140
+ }
141
+
142
+ function showHelp() {
143
+ console.log(`
144
+ Usage: npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts [command]
145
+
146
+ Commands:
147
+ start Start the scheduler service (runs continuously)
148
+ sync Run a single sync (incremental if checkpoint exists)
149
+ full Run a full sync (ignores checkpoint, replaces all data)
150
+ status Show current sync status
151
+ help Show this help message
152
+
153
+ Environment Variables:
154
+ NEO4J_LOCAL_URI Local Neo4j URI (default: bolt://localhost:7687)
155
+ NEO4J_LOCAL_USER Local Neo4j user (default: neo4j)
156
+ NEO4J_LOCAL_PASSWORD Local Neo4j password (default: password)
157
+ NEO4J_URI Cloud Neo4j URI (required)
158
+ NEO4J_USER Cloud Neo4j user (default: neo4j)
159
+ NEO4J_PASSWORD Cloud Neo4j password (required)
160
+ NEO4J_SYNC_SCHEDULE Cron schedule (default: 0 */6 * * * = every 6 hours)
161
+
162
+ Schedule Examples:
163
+ 0 */6 * * * Every 6 hours
164
+ 0 */2 * * * Every 2 hours
165
+ 0 0 * * * Daily at midnight
166
+ */30 * * * * Every 30 minutes
167
+ 0 9,18 * * * At 9am and 6pm
168
+
169
+ Examples:
170
+ # Start auto-sync service (runs continuously)
171
+ npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts start
172
+
173
+ # Run single sync
174
+ npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts sync
175
+
176
+ # Force full sync
177
+ npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts full
178
+
179
+ # Check status
180
+ npx tsx apps/backend/src/scripts/neo4j-auto-sync.ts status
181
+ `);
182
+ }
183
+
184
+ main().catch(console.error);
apps/backend/src/scripts/sync-neo4j-to-cloud.ts ADDED
@@ -0,0 +1,470 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Neo4j Local to Cloud Sync Script
3
+ *
4
+ * Syncs data from local Neo4j (bruttopuljen/master) to AuraDB cloud
5
+ * Run: npx tsx apps/backend/src/scripts/sync-neo4j-to-cloud.ts
6
+ */
7
+
8
+ import neo4j, { Driver, Session, Record as Neo4jRecord, isNode, isRelationship } from 'neo4j-driver';
9
+ import dotenv from 'dotenv';
10
+ import path from 'path';
11
+
12
+ // Load environment
13
+ dotenv.config({ path: path.resolve(process.cwd(), '.env.production') });
14
+
15
+ interface SyncConfig {
16
+ local: {
17
+ uri: string;
18
+ user: string;
19
+ password: string;
20
+ database: string;
21
+ };
22
+ cloud: {
23
+ uri: string;
24
+ user: string;
25
+ password: string;
26
+ database: string;
27
+ };
28
+ }
29
+
30
+ interface NodeData {
31
+ labels: string[];
32
+ properties: Record<string, any>;
33
+ elementId: string;
34
+ }
35
+
36
+ interface RelationshipData {
37
+ type: string;
38
+ properties: Record<string, any>;
39
+ startElementId: string;
40
+ endElementId: string;
41
+ }
42
+
43
+ interface SchemaItem {
44
+ type: 'constraint' | 'index';
45
+ name: string;
46
+ definition: string;
47
+ }
48
+
49
+ const config: SyncConfig = {
50
+ local: {
51
+ uri: 'bolt://localhost:7687',
52
+ user: 'neo4j',
53
+ password: 'password', // Update if different
54
+ database: 'neo4j'
55
+ },
56
+ cloud: {
57
+ uri: process.env.NEO4J_URI || 'neo4j+s://054eff27.databases.neo4j.io',
58
+ user: process.env.NEO4J_USER || 'neo4j',
59
+ password: process.env.NEO4J_PASSWORD || '',
60
+ database: process.env.NEO4J_DATABASE || 'neo4j'
61
+ }
62
+ };
63
+
64
+ async function createDriver(connectionConfig: SyncConfig['local'] | SyncConfig['cloud']): Promise<Driver> {
65
+ return neo4j.driver(
66
+ connectionConfig.uri,
67
+ neo4j.auth.basic(connectionConfig.user, connectionConfig.password),
68
+ {
69
+ maxConnectionLifetime: 3 * 60 * 60 * 1000,
70
+ connectionAcquisitionTimeout: 2 * 60 * 1000,
71
+ }
72
+ );
73
+ }
74
+
75
+ async function exportSchema(session: Session): Promise<SchemaItem[]> {
76
+ console.log('πŸ“‹ Exporting schema from local...');
77
+ const schema: SchemaItem[] = [];
78
+
79
+ // Export constraints
80
+ const constraintResult = await session.run('SHOW CONSTRAINTS');
81
+ for (const record of constraintResult.records) {
82
+ const name = record.get('name');
83
+ const type = record.get('type');
84
+ const entityType = record.get('entityType');
85
+ const labelsOrTypes = record.get('labelsOrTypes');
86
+ const properties = record.get('properties');
87
+
88
+ let definition = '';
89
+ if (type === 'UNIQUENESS' && entityType === 'NODE') {
90
+ const label = labelsOrTypes?.[0] || 'Unknown';
91
+ const prop = properties?.[0] || 'id';
92
+ definition = `CREATE CONSTRAINT ${name} IF NOT EXISTS FOR (n:${label}) REQUIRE n.${prop} IS UNIQUE`;
93
+ }
94
+
95
+ if (definition) {
96
+ schema.push({ type: 'constraint', name, definition });
97
+ console.log(` βœ“ Constraint: ${name}`);
98
+ }
99
+ }
100
+
101
+ // Export indexes (excluding auto-generated and constraint-backed)
102
+ const indexResult = await session.run('SHOW INDEXES');
103
+ for (const record of indexResult.records) {
104
+ const name = record.get('name');
105
+ const type = record.get('type');
106
+ const entityType = record.get('entityType');
107
+ const labelsOrTypes = record.get('labelsOrTypes');
108
+ const properties = record.get('properties');
109
+ const owningConstraint = record.get('owningConstraint');
110
+
111
+ // Skip constraint-backed indexes
112
+ if (owningConstraint) continue;
113
+
114
+ let definition = '';
115
+ if (type === 'RANGE' && entityType === 'NODE') {
116
+ const label = labelsOrTypes?.[0] || 'Unknown';
117
+ const props = properties?.join(', ') || 'id';
118
+ definition = `CREATE INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${props.split(',')[0].trim()})`;
119
+ } else if (type === 'FULLTEXT' && entityType === 'NODE') {
120
+ const label = labelsOrTypes?.[0] || 'Unknown';
121
+ const props = (properties || []).map((p: string) => `n.${p}`).join(', ');
122
+ definition = `CREATE FULLTEXT INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON EACH [${props}]`;
123
+ } else if (type === 'VECTOR') {
124
+ const label = labelsOrTypes?.[0] || 'Unknown';
125
+ const prop = properties?.[0] || 'embedding';
126
+ definition = `CREATE VECTOR INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${prop}) OPTIONS {indexConfig: {\`vector.dimensions\`: 384, \`vector.similarity_function\`: 'cosine'}}`;
127
+ }
128
+
129
+ if (definition) {
130
+ schema.push({ type: 'index', name, definition });
131
+ console.log(` βœ“ Index: ${name} (${type})`);
132
+ }
133
+ }
134
+
135
+ console.log(` Total: ${schema.length} schema items`);
136
+ return schema;
137
+ }
138
+
139
+ async function exportNodes(session: Session): Promise<NodeData[]> {
140
+ console.log('\nπŸ“¦ Exporting nodes from local...');
141
+ const nodes: NodeData[] = [];
142
+
143
+ const result = await session.run('MATCH (n) RETURN n, elementId(n) as elementId');
144
+
145
+ for (const record of result.records) {
146
+ const node = record.get('n');
147
+ const elementId = record.get('elementId');
148
+
149
+ if (isNode(node)) {
150
+ // Clone properties and convert Neo4j types to JS types
151
+ const properties: Record<string, any> = {};
152
+ for (const [key, value] of Object.entries(node.properties)) {
153
+ properties[key] = convertNeo4jValue(value);
154
+ }
155
+
156
+ nodes.push({
157
+ labels: [...node.labels],
158
+ properties,
159
+ elementId
160
+ });
161
+ }
162
+ }
163
+
164
+ console.log(` βœ“ Exported ${nodes.length} nodes`);
165
+
166
+ // Show label distribution
167
+ const labelCounts: Record<string, number> = {};
168
+ for (const node of nodes) {
169
+ for (const label of node.labels) {
170
+ labelCounts[label] = (labelCounts[label] || 0) + 1;
171
+ }
172
+ }
173
+ for (const [label, count] of Object.entries(labelCounts)) {
174
+ console.log(` - ${label}: ${count}`);
175
+ }
176
+
177
+ return nodes;
178
+ }
179
+
180
+ async function exportRelationships(session: Session): Promise<RelationshipData[]> {
181
+ console.log('\nπŸ”— Exporting relationships from local...');
182
+ const relationships: RelationshipData[] = [];
183
+
184
+ const result = await session.run(`
185
+ MATCH (a)-[r]->(b)
186
+ RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
187
+ `);
188
+
189
+ for (const record of result.records) {
190
+ const rel = record.get('r');
191
+ const type = record.get('type');
192
+ const startId = record.get('startId');
193
+ const endId = record.get('endId');
194
+
195
+ // Clone properties and convert Neo4j types
196
+ const properties: Record<string, any> = {};
197
+ if (isRelationship(rel)) {
198
+ for (const [key, value] of Object.entries(rel.properties)) {
199
+ properties[key] = convertNeo4jValue(value);
200
+ }
201
+ }
202
+
203
+ relationships.push({
204
+ type,
205
+ properties,
206
+ startElementId: startId,
207
+ endElementId: endId
208
+ });
209
+ }
210
+
211
+ console.log(` βœ“ Exported ${relationships.length} relationships`);
212
+
213
+ // Show relationship type distribution
214
+ const typeCounts: Record<string, number> = {};
215
+ for (const rel of relationships) {
216
+ typeCounts[rel.type] = (typeCounts[rel.type] || 0) + 1;
217
+ }
218
+ for (const [type, count] of Object.entries(typeCounts)) {
219
+ console.log(` - ${type}: ${count}`);
220
+ }
221
+
222
+ return relationships;
223
+ }
224
+
225
+ function convertNeo4jValue(value: any): any {
226
+ if (value === null || value === undefined) return value;
227
+
228
+ // Handle Neo4j Integer
229
+ if (typeof value === 'object' && value.constructor?.name === 'Integer') {
230
+ return value.toNumber();
231
+ }
232
+
233
+ // Handle Neo4j DateTime
234
+ if (typeof value === 'object' && value.constructor?.name === 'DateTime') {
235
+ return value.toString();
236
+ }
237
+
238
+ // Handle Neo4j Date
239
+ if (typeof value === 'object' && value.constructor?.name === 'Date') {
240
+ return value.toString();
241
+ }
242
+
243
+ // Handle arrays
244
+ if (Array.isArray(value)) {
245
+ return value.map(convertNeo4jValue);
246
+ }
247
+
248
+ // Handle Float arrays (embeddings)
249
+ if (typeof value === 'object' && value.constructor?.name === 'Float64Array') {
250
+ return Array.from(value);
251
+ }
252
+
253
+ return value;
254
+ }
255
+
256
+ async function clearCloud(session: Session): Promise<void> {
257
+ console.log('\nπŸ—‘οΈ Clearing cloud database...');
258
+
259
+ // Delete in batches to avoid memory issues
260
+ let deleted = 0;
261
+ do {
262
+ const result = await session.run(`
263
+ MATCH (n)
264
+ WITH n LIMIT 1000
265
+ DETACH DELETE n
266
+ RETURN count(*) as deleted
267
+ `);
268
+ deleted = result.records[0]?.get('deleted')?.toNumber() || 0;
269
+ if (deleted > 0) {
270
+ console.log(` Deleted batch of ${deleted} nodes...`);
271
+ }
272
+ } while (deleted > 0);
273
+
274
+ console.log(' βœ“ Cloud database cleared');
275
+ }
276
+
277
+ async function applySchema(session: Session, schema: SchemaItem[]): Promise<void> {
278
+ console.log('\nπŸ“‹ Applying schema to cloud...');
279
+
280
+ for (const item of schema) {
281
+ try {
282
+ await session.run(item.definition);
283
+ console.log(` βœ“ ${item.type}: ${item.name}`);
284
+ } catch (error: any) {
285
+ if (error.message?.includes('already exists') || error.message?.includes('equivalent')) {
286
+ console.log(` ⚠ ${item.type} ${item.name} already exists`);
287
+ } else {
288
+ console.error(` βœ— Failed to create ${item.type} ${item.name}: ${error.message}`);
289
+ }
290
+ }
291
+ }
292
+ }
293
+
294
+ async function importNodes(session: Session, nodes: NodeData[], elementIdMap: Map<string, string>): Promise<void> {
295
+ console.log('\nπŸ“¦ Importing nodes to cloud...');
296
+
297
+ let imported = 0;
298
+ const batchSize = 100;
299
+
300
+ for (let i = 0; i < nodes.length; i += batchSize) {
301
+ const batch = nodes.slice(i, i + batchSize);
302
+
303
+ for (const node of batch) {
304
+ try {
305
+ // Create node with labels and properties
306
+ const labelStr = node.labels.join(':');
307
+
308
+ // Generate a unique ID if not present
309
+ const nodeId = node.properties.id || `node_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
310
+ node.properties.id = nodeId;
311
+
312
+ const query = `
313
+ MERGE (n:${labelStr} {id: $nodeId})
314
+ SET n = $props
315
+ RETURN elementId(n) as newElementId
316
+ `;
317
+
318
+ const result = await session.run(query, {
319
+ nodeId,
320
+ props: node.properties
321
+ });
322
+
323
+ const newElementId = result.records[0]?.get('newElementId');
324
+ if (newElementId) {
325
+ elementIdMap.set(node.elementId, newElementId);
326
+ }
327
+
328
+ imported++;
329
+ } catch (error: any) {
330
+ console.error(` βœ— Failed to import node: ${error.message?.slice(0, 100)}`);
331
+ }
332
+ }
333
+
334
+ console.log(` Progress: ${imported}/${nodes.length} nodes`);
335
+ }
336
+
337
+ console.log(` βœ“ Imported ${imported} nodes`);
338
+ }
339
+
340
+ async function importRelationships(
341
+ session: Session,
342
+ relationships: RelationshipData[],
343
+ elementIdMap: Map<string, string>,
344
+ nodes: NodeData[]
345
+ ): Promise<void> {
346
+ console.log('\nπŸ”— Importing relationships to cloud...');
347
+
348
+ // Create a map from old elementId to node id (for relationship matching)
349
+ const elementIdToNodeId = new Map<string, string>();
350
+ for (const node of nodes) {
351
+ elementIdToNodeId.set(node.elementId, node.properties.id);
352
+ }
353
+
354
+ let imported = 0;
355
+ let skipped = 0;
356
+
357
+ for (const rel of relationships) {
358
+ try {
359
+ const startNodeId = elementIdToNodeId.get(rel.startElementId);
360
+ const endNodeId = elementIdToNodeId.get(rel.endElementId);
361
+
362
+ if (!startNodeId || !endNodeId) {
363
+ skipped++;
364
+ continue;
365
+ }
366
+
367
+ const query = `
368
+ MATCH (a {id: $startId})
369
+ MATCH (b {id: $endId})
370
+ MERGE (a)-[r:${rel.type}]->(b)
371
+ SET r = $props
372
+ `;
373
+
374
+ await session.run(query, {
375
+ startId: startNodeId,
376
+ endId: endNodeId,
377
+ props: rel.properties
378
+ });
379
+
380
+ imported++;
381
+ } catch (error: any) {
382
+ console.error(` βœ— Failed to import relationship: ${error.message?.slice(0, 100)}`);
383
+ }
384
+ }
385
+
386
+ console.log(` βœ“ Imported ${imported} relationships (skipped ${skipped})`);
387
+ }
388
+
389
+ async function syncNeo4jToCloud() {
390
+ console.log('═'.repeat(60));
391
+ console.log('πŸ”„ Neo4j Local β†’ Cloud Sync');
392
+ console.log('═'.repeat(60));
393
+ console.log(`\nLocal: ${config.local.uri}`);
394
+ console.log(`Cloud: ${config.cloud.uri}`);
395
+ console.log('');
396
+
397
+ let localDriver: Driver | null = null;
398
+ let cloudDriver: Driver | null = null;
399
+
400
+ try {
401
+ // Connect to local
402
+ console.log('πŸ“‘ Connecting to local Neo4j...');
403
+ localDriver = await createDriver(config.local);
404
+ await localDriver.verifyConnectivity();
405
+ console.log(' βœ“ Connected to local');
406
+
407
+ // Connect to cloud
408
+ console.log('☁️ Connecting to AuraDB cloud...');
409
+ cloudDriver = await createDriver(config.cloud);
410
+ await cloudDriver.verifyConnectivity();
411
+ console.log(' βœ“ Connected to cloud');
412
+
413
+ // Export from local
414
+ const localSession = localDriver.session({ database: config.local.database });
415
+
416
+ const schema = await exportSchema(localSession);
417
+ const nodes = await exportNodes(localSession);
418
+ const relationships = await exportRelationships(localSession);
419
+
420
+ await localSession.close();
421
+
422
+ // Confirm before clearing cloud
423
+ console.log('\n⚠️ This will REPLACE all data in cloud with local data!');
424
+ console.log(` Nodes to sync: ${nodes.length}`);
425
+ console.log(` Relationships to sync: ${relationships.length}`);
426
+
427
+ // Import to cloud
428
+ const cloudSession = cloudDriver.session({ database: config.cloud.database });
429
+
430
+ await clearCloud(cloudSession);
431
+ await applySchema(cloudSession, schema);
432
+
433
+ const elementIdMap = new Map<string, string>();
434
+ await importNodes(cloudSession, nodes, elementIdMap);
435
+ await importRelationships(cloudSession, relationships, elementIdMap, nodes);
436
+
437
+ await cloudSession.close();
438
+
439
+ // Verify sync
440
+ console.log('\nβœ… Verifying sync...');
441
+ const verifySession = cloudDriver.session({ database: config.cloud.database });
442
+
443
+ const nodeCountResult = await verifySession.run('MATCH (n) RETURN count(n) as count');
444
+ const cloudNodeCount = nodeCountResult.records[0]?.get('count')?.toNumber() || 0;
445
+
446
+ const relCountResult = await verifySession.run('MATCH ()-[r]->() RETURN count(r) as count');
447
+ const cloudRelCount = relCountResult.records[0]?.get('count')?.toNumber() || 0;
448
+
449
+ await verifySession.close();
450
+
451
+ console.log(` Cloud nodes: ${cloudNodeCount} (expected: ${nodes.length})`);
452
+ console.log(` Cloud relationships: ${cloudRelCount} (expected: ${relationships.length})`);
453
+
454
+ console.log('\n' + '═'.repeat(60));
455
+ console.log('βœ… SYNC COMPLETE');
456
+ console.log('═'.repeat(60));
457
+
458
+ } catch (error: any) {
459
+ console.error('\n❌ Sync failed:', error.message);
460
+ throw error;
461
+ } finally {
462
+ if (localDriver) await localDriver.close();
463
+ if (cloudDriver) await cloudDriver.close();
464
+ }
465
+ }
466
+
467
+ // Run
468
+ syncNeo4jToCloud()
469
+ .then(() => process.exit(0))
470
+ .catch(() => process.exit(1));
apps/backend/src/services/Neo4jAutoSync.ts ADDED
@@ -0,0 +1,637 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Neo4j Auto-Sync Service
3
+ *
4
+ * Automated synchronization between local Neo4j and AuraDB cloud.
5
+ * Supports full sync, incremental sync, and scheduled execution.
6
+ */
7
+
8
+ import neo4j, { Driver, Session } from 'neo4j-driver';
9
+ import cron from 'node-cron';
10
+ import fs from 'fs';
11
+ import path from 'path';
12
+
13
+ export interface SyncConfig {
14
+ local: {
15
+ uri: string;
16
+ user: string;
17
+ password: string;
18
+ database: string;
19
+ };
20
+ cloud: {
21
+ uri: string;
22
+ user: string;
23
+ password: string;
24
+ database: string;
25
+ };
26
+ schedule?: string; // Cron expression
27
+ incrementalEnabled?: boolean;
28
+ batchSize?: number;
29
+ }
30
+
31
+ export interface SyncStatus {
32
+ lastSyncTime: string | null;
33
+ lastSyncType: 'full' | 'incremental' | null;
34
+ lastSyncDuration: number | null;
35
+ nodesSync: number;
36
+ relationshipsSync: number;
37
+ status: 'idle' | 'running' | 'success' | 'error';
38
+ error?: string;
39
+ nextScheduledSync?: string;
40
+ }
41
+
42
+ export interface SyncCheckpoint {
43
+ timestamp: string;
44
+ nodeCount: number;
45
+ relationshipCount: number;
46
+ lastNodeIds: string[];
47
+ }
48
+
49
+ const CHECKPOINT_FILE = path.join(process.cwd(), '.neo4j-sync-checkpoint.json');
50
+ const STATUS_FILE = path.join(process.cwd(), '.neo4j-sync-status.json');
51
+
52
+ export class Neo4jAutoSync {
53
+ private config: SyncConfig;
54
+ private localDriver: Driver | null = null;
55
+ private cloudDriver: Driver | null = null;
56
+ private cronJob: cron.ScheduledTask | null = null;
57
+ private status: SyncStatus = {
58
+ lastSyncTime: null,
59
+ lastSyncType: null,
60
+ lastSyncDuration: null,
61
+ nodesSync: 0,
62
+ relationshipsSync: 0,
63
+ status: 'idle'
64
+ };
65
+
66
+ constructor(config?: Partial<SyncConfig>) {
67
+ this.config = {
68
+ local: {
69
+ uri: config?.local?.uri || process.env.NEO4J_LOCAL_URI || 'bolt://localhost:7687',
70
+ user: config?.local?.user || process.env.NEO4J_LOCAL_USER || 'neo4j',
71
+ password: config?.local?.password || process.env.NEO4J_LOCAL_PASSWORD || 'password',
72
+ database: config?.local?.database || process.env.NEO4J_LOCAL_DATABASE || 'neo4j'
73
+ },
74
+ cloud: {
75
+ uri: config?.cloud?.uri || process.env.NEO4J_URI || '',
76
+ user: config?.cloud?.user || process.env.NEO4J_USER || 'neo4j',
77
+ password: config?.cloud?.password || process.env.NEO4J_PASSWORD || '',
78
+ database: config?.cloud?.database || process.env.NEO4J_DATABASE || 'neo4j'
79
+ },
80
+ schedule: config?.schedule || process.env.NEO4J_SYNC_SCHEDULE || '0 */6 * * *', // Every 6 hours
81
+ incrementalEnabled: config?.incrementalEnabled ?? true,
82
+ batchSize: config?.batchSize || 500
83
+ };
84
+
85
+ this.loadStatus();
86
+ }
87
+
88
+ private loadStatus(): void {
89
+ try {
90
+ if (fs.existsSync(STATUS_FILE)) {
91
+ const data = fs.readFileSync(STATUS_FILE, 'utf-8');
92
+ this.status = JSON.parse(data);
93
+ this.status.status = 'idle'; // Reset running status on restart
94
+ }
95
+ } catch {
96
+ // Ignore errors, use default status
97
+ }
98
+ }
99
+
100
+ private saveStatus(): void {
101
+ try {
102
+ fs.writeFileSync(STATUS_FILE, JSON.stringify(this.status, null, 2));
103
+ } catch (error) {
104
+ console.error('Failed to save sync status:', error);
105
+ }
106
+ }
107
+
108
+ private loadCheckpoint(): SyncCheckpoint | null {
109
+ try {
110
+ if (fs.existsSync(CHECKPOINT_FILE)) {
111
+ const data = fs.readFileSync(CHECKPOINT_FILE, 'utf-8');
112
+ return JSON.parse(data);
113
+ }
114
+ } catch {
115
+ // Ignore errors
116
+ }
117
+ return null;
118
+ }
119
+
120
+ private saveCheckpoint(checkpoint: SyncCheckpoint): void {
121
+ try {
122
+ fs.writeFileSync(CHECKPOINT_FILE, JSON.stringify(checkpoint, null, 2));
123
+ } catch (error) {
124
+ console.error('Failed to save checkpoint:', error);
125
+ }
126
+ }
127
+
128
+ async connect(): Promise<void> {
129
+ console.log('πŸ“‘ Connecting to Neo4j instances...');
130
+
131
+ this.localDriver = neo4j.driver(
132
+ this.config.local.uri,
133
+ neo4j.auth.basic(this.config.local.user, this.config.local.password),
134
+ { maxConnectionLifetime: 3 * 60 * 60 * 1000 }
135
+ );
136
+ await this.localDriver.verifyConnectivity();
137
+ console.log(' βœ“ Connected to local Neo4j');
138
+
139
+ this.cloudDriver = neo4j.driver(
140
+ this.config.cloud.uri,
141
+ neo4j.auth.basic(this.config.cloud.user, this.config.cloud.password),
142
+ { maxConnectionLifetime: 3 * 60 * 60 * 1000 }
143
+ );
144
+ await this.cloudDriver.verifyConnectivity();
145
+ console.log(' βœ“ Connected to AuraDB cloud');
146
+ }
147
+
148
+ async disconnect(): Promise<void> {
149
+ if (this.localDriver) await this.localDriver.close();
150
+ if (this.cloudDriver) await this.cloudDriver.close();
151
+ this.localDriver = null;
152
+ this.cloudDriver = null;
153
+ }
154
+
155
+ /**
156
+ * Start scheduled auto-sync
157
+ */
158
+ startScheduler(): void {
159
+ if (this.cronJob) {
160
+ this.cronJob.stop();
161
+ }
162
+
163
+ console.log(`⏰ Starting Neo4j auto-sync scheduler: ${this.config.schedule}`);
164
+
165
+ this.cronJob = cron.schedule(this.config.schedule!, async () => {
166
+ console.log(`\nπŸ”„ [${new Date().toISOString()}] Scheduled sync starting...`);
167
+ try {
168
+ await this.sync();
169
+ } catch (error) {
170
+ console.error('Scheduled sync failed:', error);
171
+ }
172
+ });
173
+
174
+ // Calculate next run time
175
+ const nextRun = this.getNextScheduledTime();
176
+ this.status.nextScheduledSync = nextRun;
177
+ this.saveStatus();
178
+
179
+ console.log(` Next sync: ${nextRun}`);
180
+ }
181
+
182
+ stopScheduler(): void {
183
+ if (this.cronJob) {
184
+ this.cronJob.stop();
185
+ this.cronJob = null;
186
+ console.log('⏹️ Scheduler stopped');
187
+ }
188
+ }
189
+
190
+ private getNextScheduledTime(): string {
191
+ // Simple calculation for common cron patterns
192
+ const now = new Date();
193
+ const schedule = this.config.schedule!;
194
+
195
+ if (schedule.includes('*/6')) {
196
+ // Every 6 hours
197
+ const nextHour = Math.ceil(now.getHours() / 6) * 6;
198
+ const next = new Date(now);
199
+ next.setHours(nextHour, 0, 0, 0);
200
+ if (next <= now) next.setHours(next.getHours() + 6);
201
+ return next.toISOString();
202
+ }
203
+
204
+ return 'See cron expression: ' + schedule;
205
+ }
206
+
207
+ /**
208
+ * Perform sync (auto-detects full vs incremental)
209
+ */
210
+ async sync(forceFullSync: boolean = false): Promise<SyncStatus> {
211
+ const startTime = Date.now();
212
+ this.status.status = 'running';
213
+ this.saveStatus();
214
+
215
+ try {
216
+ await this.connect();
217
+
218
+ const checkpoint = this.loadCheckpoint();
219
+ const shouldDoFullSync = forceFullSync || !checkpoint || !this.config.incrementalEnabled;
220
+
221
+ if (shouldDoFullSync) {
222
+ await this.fullSync();
223
+ this.status.lastSyncType = 'full';
224
+ } else {
225
+ const hasChanges = await this.checkForChanges(checkpoint);
226
+ if (hasChanges) {
227
+ await this.incrementalSync(checkpoint);
228
+ this.status.lastSyncType = 'incremental';
229
+ } else {
230
+ console.log('βœ“ No changes detected, skipping sync');
231
+ this.status.lastSyncType = 'incremental';
232
+ }
233
+ }
234
+
235
+ this.status.status = 'success';
236
+ this.status.lastSyncTime = new Date().toISOString();
237
+ this.status.lastSyncDuration = Date.now() - startTime;
238
+ delete this.status.error;
239
+
240
+ } catch (error: any) {
241
+ this.status.status = 'error';
242
+ this.status.error = error.message;
243
+ console.error('❌ Sync failed:', error.message);
244
+ } finally {
245
+ await this.disconnect();
246
+ this.saveStatus();
247
+ }
248
+
249
+ return this.status;
250
+ }
251
+
252
+ private async checkForChanges(checkpoint: SyncCheckpoint): Promise<boolean> {
253
+ const session = this.localDriver!.session({ database: this.config.local.database });
254
+ try {
255
+ // Check node count
256
+ const nodeResult = await session.run('MATCH (n) RETURN count(n) as count');
257
+ const currentNodeCount = nodeResult.records[0].get('count').toNumber();
258
+
259
+ // Check relationship count
260
+ const relResult = await session.run('MATCH ()-[r]->() RETURN count(r) as count');
261
+ const currentRelCount = relResult.records[0].get('count').toNumber();
262
+
263
+ const hasChanges = currentNodeCount !== checkpoint.nodeCount ||
264
+ currentRelCount !== checkpoint.relationshipCount;
265
+
266
+ if (hasChanges) {
267
+ console.log(`πŸ“Š Changes detected: nodes ${checkpoint.nodeCount} β†’ ${currentNodeCount}, rels ${checkpoint.relationshipCount} β†’ ${currentRelCount}`);
268
+ }
269
+
270
+ return hasChanges;
271
+ } finally {
272
+ await session.close();
273
+ }
274
+ }
275
+
276
+ private async fullSync(): Promise<void> {
277
+ console.log('\nπŸ”„ Starting FULL sync...');
278
+
279
+ const localSession = this.localDriver!.session({ database: this.config.local.database });
280
+ const cloudSession = this.cloudDriver!.session({ database: this.config.cloud.database });
281
+
282
+ try {
283
+ // Export schema
284
+ const schema = await this.exportSchema(localSession);
285
+
286
+ // Export nodes
287
+ const nodes = await this.exportNodes(localSession);
288
+
289
+ // Export relationships
290
+ const relationships = await this.exportRelationships(localSession);
291
+
292
+ // Clear cloud
293
+ await this.clearCloud(cloudSession);
294
+
295
+ // Apply schema
296
+ await this.applySchema(cloudSession, schema);
297
+
298
+ // Import nodes
299
+ const elementIdMap = new Map<string, string>();
300
+ await this.importNodes(cloudSession, nodes, elementIdMap);
301
+
302
+ // Import relationships
303
+ await this.importRelationships(cloudSession, relationships, elementIdMap, nodes);
304
+
305
+ // Save checkpoint
306
+ this.saveCheckpoint({
307
+ timestamp: new Date().toISOString(),
308
+ nodeCount: nodes.length,
309
+ relationshipCount: relationships.length,
310
+ lastNodeIds: nodes.slice(-100).map(n => n.properties.id)
311
+ });
312
+
313
+ this.status.nodesSync = nodes.length;
314
+ this.status.relationshipsSync = relationships.length;
315
+
316
+ console.log(`\nβœ… Full sync complete: ${nodes.length} nodes, ${relationships.length} relationships`);
317
+
318
+ } finally {
319
+ await localSession.close();
320
+ await cloudSession.close();
321
+ }
322
+ }
323
+
324
+ private async incrementalSync(checkpoint: SyncCheckpoint): Promise<void> {
325
+ console.log('\nπŸ”„ Starting INCREMENTAL sync...');
326
+
327
+ const localSession = this.localDriver!.session({ database: this.config.local.database });
328
+ const cloudSession = this.cloudDriver!.session({ database: this.config.cloud.database });
329
+
330
+ try {
331
+ // Find new/modified nodes since checkpoint
332
+ const checkpointTime = new Date(checkpoint.timestamp);
333
+
334
+ // Get nodes created/modified after checkpoint
335
+ // This requires nodes to have a 'updatedAt' or 'createdAt' property
336
+ const newNodesResult = await localSession.run(`
337
+ MATCH (n)
338
+ WHERE n.createdAt > datetime($checkpointTime)
339
+ OR n.updatedAt > datetime($checkpointTime)
340
+ OR NOT n.id IN $existingIds
341
+ RETURN n, elementId(n) as elementId
342
+ LIMIT 10000
343
+ `, {
344
+ checkpointTime: checkpointTime.toISOString(),
345
+ existingIds: checkpoint.lastNodeIds
346
+ });
347
+
348
+ const newNodes: any[] = [];
349
+ for (const record of newNodesResult.records) {
350
+ const node = record.get('n');
351
+ newNodes.push({
352
+ labels: [...node.labels],
353
+ properties: this.convertProperties(node.properties),
354
+ elementId: record.get('elementId')
355
+ });
356
+ }
357
+
358
+ if (newNodes.length === 0) {
359
+ console.log(' No new nodes to sync');
360
+ return;
361
+ }
362
+
363
+ console.log(` Found ${newNodes.length} new/modified nodes`);
364
+
365
+ // Import new nodes
366
+ const elementIdMap = new Map<string, string>();
367
+ await this.importNodes(cloudSession, newNodes, elementIdMap);
368
+
369
+ // Get new relationships for these nodes
370
+ const nodeIds = newNodes.map(n => n.properties.id).filter(Boolean);
371
+ if (nodeIds.length > 0) {
372
+ const newRelsResult = await localSession.run(`
373
+ MATCH (a)-[r]->(b)
374
+ WHERE a.id IN $nodeIds OR b.id IN $nodeIds
375
+ RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
376
+ `, { nodeIds });
377
+
378
+ const newRels: any[] = [];
379
+ for (const record of newRelsResult.records) {
380
+ const rel = record.get('r');
381
+ newRels.push({
382
+ type: record.get('type'),
383
+ properties: this.convertProperties(rel.properties),
384
+ startElementId: record.get('startId'),
385
+ endElementId: record.get('endId')
386
+ });
387
+ }
388
+
389
+ if (newRels.length > 0) {
390
+ await this.importRelationships(cloudSession, newRels, elementIdMap, newNodes);
391
+ }
392
+ }
393
+
394
+ // Update checkpoint
395
+ const nodeCountResult = await localSession.run('MATCH (n) RETURN count(n) as count');
396
+ const relCountResult = await localSession.run('MATCH ()-[r]->() RETURN count(r) as count');
397
+
398
+ this.saveCheckpoint({
399
+ timestamp: new Date().toISOString(),
400
+ nodeCount: nodeCountResult.records[0].get('count').toNumber(),
401
+ relationshipCount: relCountResult.records[0].get('count').toNumber(),
402
+ lastNodeIds: [...checkpoint.lastNodeIds, ...nodeIds].slice(-100)
403
+ });
404
+
405
+ this.status.nodesSync = newNodes.length;
406
+ this.status.relationshipsSync = 0;
407
+
408
+ console.log(`βœ… Incremental sync complete: ${newNodes.length} nodes`);
409
+
410
+ } finally {
411
+ await localSession.close();
412
+ await cloudSession.close();
413
+ }
414
+ }
415
+
416
+ private convertProperties(props: Record<string, any>): Record<string, any> {
417
+ const result: Record<string, any> = {};
418
+ for (const [key, value] of Object.entries(props)) {
419
+ if (value === null || value === undefined) {
420
+ result[key] = value;
421
+ } else if (typeof value === 'object' && value.constructor?.name === 'Integer') {
422
+ result[key] = value.toNumber();
423
+ } else if (typeof value === 'object' && (value.constructor?.name === 'DateTime' || value.constructor?.name === 'Date')) {
424
+ result[key] = value.toString();
425
+ } else if (Array.isArray(value)) {
426
+ result[key] = value.map(v => this.convertProperties({ v }).v);
427
+ } else {
428
+ result[key] = value;
429
+ }
430
+ }
431
+ return result;
432
+ }
433
+
434
+ private async exportSchema(session: Session): Promise<any[]> {
435
+ const schema: any[] = [];
436
+
437
+ const constraintResult = await session.run('SHOW CONSTRAINTS');
438
+ for (const record of constraintResult.records) {
439
+ const name = record.get('name');
440
+ const type = record.get('type');
441
+ const entityType = record.get('entityType');
442
+ const labelsOrTypes = record.get('labelsOrTypes');
443
+ const properties = record.get('properties');
444
+
445
+ if (type === 'UNIQUENESS' && entityType === 'NODE') {
446
+ const label = labelsOrTypes?.[0];
447
+ const prop = properties?.[0];
448
+ if (label && prop) {
449
+ schema.push({
450
+ type: 'constraint',
451
+ name,
452
+ definition: `CREATE CONSTRAINT ${name} IF NOT EXISTS FOR (n:${label}) REQUIRE n.${prop} IS UNIQUE`
453
+ });
454
+ }
455
+ }
456
+ }
457
+
458
+ const indexResult = await session.run('SHOW INDEXES');
459
+ for (const record of indexResult.records) {
460
+ const name = record.get('name');
461
+ const type = record.get('type');
462
+ const entityType = record.get('entityType');
463
+ const labelsOrTypes = record.get('labelsOrTypes');
464
+ const properties = record.get('properties');
465
+ const owningConstraint = record.get('owningConstraint');
466
+
467
+ if (owningConstraint) continue;
468
+
469
+ if (type === 'RANGE' && entityType === 'NODE') {
470
+ const label = labelsOrTypes?.[0];
471
+ const prop = properties?.[0];
472
+ if (label && prop) {
473
+ schema.push({
474
+ type: 'index',
475
+ name,
476
+ definition: `CREATE INDEX ${name} IF NOT EXISTS FOR (n:${label}) ON (n.${prop})`
477
+ });
478
+ }
479
+ }
480
+ }
481
+
482
+ return schema;
483
+ }
484
+
485
+ private async exportNodes(session: Session): Promise<any[]> {
486
+ console.log('πŸ“¦ Exporting nodes...');
487
+ const result = await session.run('MATCH (n) RETURN n, elementId(n) as elementId');
488
+ const nodes = result.records.map(record => ({
489
+ labels: [...record.get('n').labels],
490
+ properties: this.convertProperties(record.get('n').properties),
491
+ elementId: record.get('elementId')
492
+ }));
493
+ console.log(` βœ“ Exported ${nodes.length} nodes`);
494
+ return nodes;
495
+ }
496
+
497
+ private async exportRelationships(session: Session): Promise<any[]> {
498
+ console.log('πŸ”— Exporting relationships...');
499
+ const result = await session.run(`
500
+ MATCH (a)-[r]->(b)
501
+ RETURN r, type(r) as type, elementId(a) as startId, elementId(b) as endId
502
+ `);
503
+ const rels = result.records.map(record => ({
504
+ type: record.get('type'),
505
+ properties: this.convertProperties(record.get('r').properties),
506
+ startElementId: record.get('startId'),
507
+ endElementId: record.get('endId')
508
+ }));
509
+ console.log(` βœ“ Exported ${rels.length} relationships`);
510
+ return rels;
511
+ }
512
+
513
+ private async clearCloud(session: Session): Promise<void> {
514
+ console.log('πŸ—‘οΈ Clearing cloud database...');
515
+ let deleted = 0;
516
+ do {
517
+ const result = await session.run(`
518
+ MATCH (n) WITH n LIMIT 1000 DETACH DELETE n RETURN count(*) as deleted
519
+ `);
520
+ deleted = result.records[0]?.get('deleted')?.toNumber() || 0;
521
+ } while (deleted > 0);
522
+ console.log(' βœ“ Cloud cleared');
523
+ }
524
+
525
+ private async applySchema(session: Session, schema: any[]): Promise<void> {
526
+ console.log('πŸ“‹ Applying schema...');
527
+ for (const item of schema) {
528
+ try {
529
+ await session.run(item.definition);
530
+ } catch (error: any) {
531
+ if (!error.message?.includes('already exists')) {
532
+ console.warn(` ⚠ ${item.name}: ${error.message?.slice(0, 50)}`);
533
+ }
534
+ }
535
+ }
536
+ console.log(` βœ“ Applied ${schema.length} schema items`);
537
+ }
538
+
539
+ private async importNodes(session: Session, nodes: any[], elementIdMap: Map<string, string>): Promise<void> {
540
+ console.log('πŸ“¦ Importing nodes...');
541
+ let imported = 0;
542
+ const batchSize = this.config.batchSize!;
543
+
544
+ for (let i = 0; i < nodes.length; i += batchSize) {
545
+ const batch = nodes.slice(i, i + batchSize);
546
+
547
+ for (const node of batch) {
548
+ try {
549
+ const labelStr = node.labels.join(':');
550
+ const nodeId = node.properties.id || `node_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
551
+ node.properties.id = nodeId;
552
+
553
+ const result = await session.run(`
554
+ MERGE (n:${labelStr} {id: $nodeId})
555
+ SET n = $props
556
+ RETURN elementId(n) as newElementId
557
+ `, { nodeId, props: node.properties });
558
+
559
+ const newElementId = result.records[0]?.get('newElementId');
560
+ if (newElementId) {
561
+ elementIdMap.set(node.elementId, newElementId);
562
+ }
563
+ imported++;
564
+ } catch (error: any) {
565
+ // Continue on error
566
+ }
567
+ }
568
+
569
+ if ((i + batchSize) % 5000 === 0 || i + batchSize >= nodes.length) {
570
+ console.log(` Progress: ${Math.min(i + batchSize, nodes.length)}/${nodes.length} nodes`);
571
+ }
572
+ }
573
+
574
+ console.log(` βœ“ Imported ${imported} nodes`);
575
+ }
576
+
577
+ private async importRelationships(
578
+ session: Session,
579
+ relationships: any[],
580
+ elementIdMap: Map<string, string>,
581
+ nodes: any[]
582
+ ): Promise<void> {
583
+ console.log('πŸ”— Importing relationships...');
584
+
585
+ const elementIdToNodeId = new Map<string, string>();
586
+ for (const node of nodes) {
587
+ elementIdToNodeId.set(node.elementId, node.properties.id);
588
+ }
589
+
590
+ let imported = 0;
591
+ const batchSize = this.config.batchSize!;
592
+
593
+ for (let i = 0; i < relationships.length; i += batchSize) {
594
+ const batch = relationships.slice(i, i + batchSize);
595
+
596
+ for (const rel of batch) {
597
+ try {
598
+ const startNodeId = elementIdToNodeId.get(rel.startElementId);
599
+ const endNodeId = elementIdToNodeId.get(rel.endElementId);
600
+
601
+ if (!startNodeId || !endNodeId) continue;
602
+
603
+ await session.run(`
604
+ MATCH (a {id: $startId})
605
+ MATCH (b {id: $endId})
606
+ MERGE (a)-[r:${rel.type}]->(b)
607
+ SET r = $props
608
+ `, { startId: startNodeId, endId: endNodeId, props: rel.properties });
609
+
610
+ imported++;
611
+ } catch (error: any) {
612
+ // Continue on error
613
+ }
614
+ }
615
+
616
+ if ((i + batchSize) % 50000 === 0 || i + batchSize >= relationships.length) {
617
+ console.log(` Progress: ${Math.min(i + batchSize, relationships.length)}/${relationships.length} relationships`);
618
+ }
619
+ }
620
+
621
+ console.log(` βœ“ Imported ${imported} relationships`);
622
+ }
623
+
624
+ getStatus(): SyncStatus {
625
+ return { ...this.status };
626
+ }
627
+ }
628
+
629
+ // Singleton instance
630
+ let autoSyncInstance: Neo4jAutoSync | null = null;
631
+
632
+ export function getNeo4jAutoSync(): Neo4jAutoSync {
633
+ if (!autoSyncInstance) {
634
+ autoSyncInstance = new Neo4jAutoSync();
635
+ }
636
+ return autoSyncInstance;
637
+ }