export_pinecone_to_faiss.py

"""
Pinecone → FAISS Export Script
FeelEd Lite — Grade 11 Commerce TN Board
Run this LOCALLY before pushing to HF Space
"""

import os
import json
import numpy as np
import faiss
import pickle
from pinecone import Pinecone

# ─── CONFIG ──────────────────────────────
PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY", "your-api-key-here")
INDEX_NAME = "feeled-ai-syllabus"
INDEX_HOST = "feeled-ai-syllabus-5hnhu9u.svc.aped-4627-b74a.pinecone.io"
NAMESPACE = ""                     # namespace இருந்தா இங்க போடுங்க
GRADE_FILTER = "11"               # Grade 11 மட்டும் — "" போட்டா all grades
SUBJECT_FILTER = ""               # "" = all subjects

OUTPUT_DIR = "faiss_index"
BATCH_SIZE = 100
DIMENSION = 768                    # confirmed

# ─── INIT ─────────────────────────────────
pc = Pinecone(api_key=PINECONE_API_KEY)
index = pc.Index(INDEX_NAME, host=INDEX_HOST)

# Index stats பார்க்கலாம்
stats = index.describe_index_stats()
print(f"Total vectors: {stats['total_vector_count']}")
print(f"Dimension: {stats['dimension']}")
DIMENSION = stats['dimension']

# ─── FETCH ALL VECTORS ────────────────────
print("\nFetching vectors from Pinecone...")

all_vectors = []
all_metadata = []
all_ids = []

# Pinecone list + fetch approach
try:
    # Method 1: list all IDs then fetch
    id_list = []
    for ids in index.list(namespace=NAMESPACE):
        id_list.extend(ids)
        print(f"  Found {len(id_list)} IDs so far...", end="\r")

    print(f"\nTotal IDs found: {len(id_list)}")

    # Fetch in batches
    for i in range(0, len(id_list), BATCH_SIZE):
        batch_ids = id_list[i:i+BATCH_SIZE]
        response = index.fetch(ids=batch_ids, namespace=NAMESPACE)

        for vec_id, vec_data in response.vectors.items():
            metadata = vec_data.metadata or {}

            # Grade 11 Commerce filter
            grade = str(metadata.get("grade", ""))
            subject = str(metadata.get("subject", "")).lower()

            # Filter: Grade 11 only (remove filter to get all grades)
            if GRADE_FILTER and grade != GRADE_FILTER:
                continue

            all_ids.append(vec_id)
            all_vectors.append(vec_data.values)
            all_metadata.append(metadata)

        print(f"  Processed {min(i+BATCH_SIZE, len(id_list))}/{len(id_list)} vectors, kept {len(all_vectors)}", end="\r")

except Exception as e:
    print(f"\nMethod 1 failed: {e}")
    print("Trying Method 2: query-based fetch...")

    # Method 2: dummy query to get all vectors
    dummy_vector = [0.0] * DIMENSION
    response = index.query(
        vector=dummy_vector,
        top_k=10000,
        include_values=True,
        include_metadata=True,
        namespace=NAMESPACE,
        filter={"grade": {"$eq": "11"}} if GRADE_FILTER else {}
    )

    for match in response.matches:
        all_ids.append(match.id)
        all_vectors.append(match.values)
        all_metadata.append(match.metadata or {})

    print(f"\nFetched {len(all_vectors)} vectors via query")

print(f"\n✅ Total vectors to index: {len(all_vectors)}")

# ─── BUILD FAISS INDEX ────────────────────
print("\nBuilding FAISS index...")

if len(all_vectors) == 0:
    print("❌ No vectors found! Check your filter settings.")
    exit(1)

vectors_np = np.array(all_vectors, dtype=np.float32)

# Normalize for cosine similarity
faiss.normalize_L2(vectors_np)

# Create index
faiss_index = faiss.IndexFlatIP(DIMENSION)  # Inner Product = cosine after normalize
faiss_index.add(vectors_np)

print(f"✅ FAISS index built: {faiss_index.ntotal} vectors")

# ─── SAVE FILES ───────────────────────────
os.makedirs(OUTPUT_DIR, exist_ok=True)

# Save FAISS index
faiss.write_index(faiss_index, f"{OUTPUT_DIR}/index.faiss")
print(f"✅ Saved: {OUTPUT_DIR}/index.faiss")

# Save metadata
metadata_store = {
    "ids": all_ids,
    "metadata": all_metadata,
    "dimension": DIMENSION,
    "total": len(all_vectors)
}
with open(f"{OUTPUT_DIR}/metadata.pkl", "wb") as f:
    pickle.dump(metadata_store, f)
print(f"✅ Saved: {OUTPUT_DIR}/metadata.pkl")

# Save summary JSON
summary = {
    "total_vectors": len(all_vectors),
    "dimension": DIMENSION,
    "grade_filter": GRADE_FILTER,
    "subject_filter": SUBJECT_FILTER,
    "sample_metadata": all_metadata[:3] if all_metadata else []
}
with open(f"{OUTPUT_DIR}/summary.json", "w", encoding="utf-8") as f:
    json.dump(summary, f, ensure_ascii=False, indent=2)
print(f"✅ Saved: {OUTPUT_DIR}/summary.json")

print(f"\n🎉 Export complete!")
print(f"   Files in: ./{OUTPUT_DIR}/")
print(f"   Total vectors: {len(all_vectors)}")
print(f"\nNext step: Copy '{OUTPUT_DIR}' folder to feeled-lite/ and git push")