File size: 4,642 Bytes
383cb38 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | #!/usr/bin/env python3
import json
import os
from pathlib import Path
import re
import sys
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
# Add backend to path for imports
sys.path.append(str(Path(__file__).parent.parent))
from core.database import DATABASE_URL
from core.models import Base, IntegrationCatalog
def parse_ts_to_json(file_path):
"""Extracts the JSON array from a TypeScript export file"""
try:
content = Path(file_path).read_text(encoding="utf-8")
# Match the array part: export const NAME: Type[] = [ ... ];
match = re.search(r'=\s*(\[[\s\S]*\]);', content)
if not match:
print("Error: Could not find JSON array in {}".format(file_path))
return []
json_str = match.group(1)
# Clean up any trailing commas that JSON doesn't like but TS does
json_str = re.sub(r',(\s*[\]\}])', r'\1', json_str)
return json.loads(json_str)
except Exception as e:
print("Error parsing {}: {}".format(file_path, e))
return []
# Map Activepieces IDs to native Atom IDs
NATIVE_MAPPING = {
"@activepieces/piece-slack": "slack",
"@activepieces/piece-gmail": "gmail",
"@activepieces/piece-asana": "asana",
"@activepieces/piece-notion": "notion",
"@activepieces/piece-hubspot": "hubspot",
"@activepieces/piece-salesforce": "salesforce",
"@activepieces/piece-github": "github",
"@activepieces/piece-discord": "discord",
"@activepieces/piece-stripe": "stripe",
"@activepieces/piece-jira": "jira",
"@activepieces/piece-zendesk": "zendesk",
"@activepieces/piece-zoom": "zoom",
"@activepieces/piece-google-calendar": "google_calendar",
"@activepieces/piece-google-drive": "google_drive",
"@activepieces/piece-dropbox": "dropbox",
"@activepieces/piece-trello": "trello",
"@activepieces/piece-airtable": "airtable",
"@activepieces/piece-calendly": "calendly",
"@activepieces/piece-mailchimp": "mailchimp",
"@activepieces/piece-shopify": "shopify",
"@activepieces/piece-quickbooks": "quickbooks",
"@activepieces/piece-xero": "xero",
"@activepieces/piece-linear": "linear",
"@activepieces/piece-figma": "figma",
"@activepieces/piece-openai": "openai",
}
def seed_integrations():
print(f"Connecting to database: {DATABASE_URL}")
engine = create_engine(DATABASE_URL)
Session = sessionmaker(bind=engine)
session = Session()
# Ensure table exists (though migrations should handle this)
Base.metadata.create_all(engine)
# Path to the auto-generated pieces
ts_file = Path(__file__).parent.parent.parent / "frontend-nextjs" / "lib" / "auto-generated-pieces.ts"
if not ts_file.exists():
print(f"Error: {ts_file} not found. Run update-catalog.py first.")
return
pieces = parse_ts_to_json(ts_file)
print(f"Found {len(pieces)} pieces in {ts_file}")
# Add manual pieces if missing from auto-generated list
# For now, we trust the auto-generated list + our deduplication logic
count = 0
for p in pieces:
# Check if already exists
existing = session.query(IntegrationCatalog).filter_by(id=p['id']).first()
if existing:
# Update
existing.name = p['name']
existing.description = p.get('description', '')
existing.category = p['category']
existing.icon = p.get('icon', '')
existing.color = p.get('color', '#6366F1')
existing.auth_type = p.get('authType', 'none')
existing.triggers = p.get('triggers', [])
existing.actions = p.get('actions', [])
existing.native_id = NATIVE_MAPPING.get(p['id'])
else:
# Insert
new_piece = IntegrationCatalog(
id=p['id'],
name=p['name'],
description=p.get('description', ''),
category=p['category'],
icon=p.get('icon', ''),
color=p.get('color', '#6366F1'),
auth_type=p.get('authType', 'none'),
triggers=p.get('triggers', []),
actions=p.get('actions', []),
native_id=NATIVE_MAPPING.get(p['id'])
)
session.add(new_piece)
count += 1
if count % 100 == 0:
session.commit()
print(f"Processed {count} pieces...")
session.commit()
print(f"Successfully seeded {count} integrations into the database.")
session.close()
if __name__ == "__main__":
seed_integrations()
|