scripts/migrate_qdrant.py

import os
from qdrant_client import QdrantClient
from dotenv import load_dotenv

from qdrant_client.models import PointStruct

def migrate():
    # Load .env manually in case this is run directly
    load_dotenv()

    # 1. Local Client
    local_path = "./data/qdrant_store"
    if not os.path.exists(local_path):
        print(f"Error: Local Qdrant store not found at {local_path}")
        return

    local_client = QdrantClient(path=local_path)
    collection_name = "financial_rag"

    # 2. Cloud Client
    url = os.environ.get("QDRANT_URL")
    api_key = os.environ.get("QDRANT_API_KEY")

    if not url or not api_key:
        print("Error: QDRANT_URL or QDRANT_API_KEY not found in environment.")
        print("Please add them to your .env file.")
        return

    cloud_client = QdrantClient(url=url, port=443, api_key=api_key, timeout=60)

    print(f"Connected to Local ({local_path}) and Cloud ({url})")

    # 3. Create Cloud Collection if missing
    try:
        local_info = local_client.get_collection(collection_name)
    except Exception as e:
        print(f"Error reading local collection: {e}")
        return

    existing_cloud = [c.name for c in cloud_client.get_collections().collections]
    if collection_name not in existing_cloud:
        print(f"Creating collection '{collection_name}' in Cloud...")
        cloud_client.create_collection(
            collection_name=collection_name,
            vectors_config=local_info.config.params.vectors,
        )
        from qdrant_client.models import PayloadSchemaType
        for f in ['chunk_id', 'parent_chunk_id', 'company', 'ticker', 'source_file', 'fiscal_year', 'section', 'subsection', 'content_type', 'chunk_level']:
            cloud_client.create_payload_index(collection_name=collection_name, field_name=f, field_schema=PayloadSchemaType.KEYWORD)
        cloud_client.create_payload_index(collection_name=collection_name, field_name='page_number', field_schema=PayloadSchemaType.INTEGER)

    # 4. Migrate points in batches
    print("Starting migration...")
    batch_size = 100
    offset = None
    total_migrated = 0

    while True:
        # Scroll through local points
        records, next_offset = local_client.scroll(
            collection_name=collection_name,
            limit=batch_size,
            offset=offset,
            with_payload=True,
            with_vectors=True
        )

        if not records:
            break

        # Convert Record objects to PointStruct objects
        point_structs = [
            PointStruct(id=record.id, vector=record.vector, payload=record.payload)
            for record in records
        ]

        # Upsert to cloud
        cloud_client.upsert(
            collection_name=collection_name,
            points=point_structs
        )

        total_migrated += len(records)
        print(f"Migrated {total_migrated} points...")

        offset = next_offset
        if offset is None:
            break

    print(f"\nMigration complete! Successfully moved {total_migrated} vectors to Qdrant Cloud.")

if __name__ == "__main__":
    migrate()