scripts/embed_books.py

import sqlite3
import json
import os
from sidecar.embedder import get_embedding

# Connect to your SQLite database
DB_PATH = "library_database.db"

def embed_all_books():
    if not os.path.exists(DB_PATH):
        print(f"Error: {DB_PATH} not found!")
        return

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    # Get books that haven't been embedded yet
    # Adjust column names if they differ in your schema
    cur.execute("""
        SELECT id, title, author, shelf, summary 
        FROM books 
        WHERE embedding IS NULL OR embedding = ''
    """)
    
    books = cur.fetchall()
    if not books:
        print("No new books to embed. ✅")
        return

    print(f"Found {len(books)} books to embed...")

    for i, (book_id, title, author, shelf, summary) in enumerate(books):
        # Construct meaningful text for the embedding
        # Task: search_document is used for the database entries
        text = f"Title: {title}. Author: {author}. Shelf: {shelf}. Summary: {summary}"
        
        try:
            vector = get_embedding(text, task="search_document")
            
            # Store vector as JSON string in SQLite
            cur.execute(
                "UPDATE books SET embedding = ? WHERE id = ?",
                (json.dumps(vector), book_id)
            )
            
            if (i + 1) % 50 == 0:
                conn.commit()
                print(f"  Processed {i+1}/{len(books)} books...")
        
        except Exception as e:
            print(f"Error embedding book {book_id}: {e}")

    conn.commit()
    cur.close()
    conn.close()
    print("All books embedded successfully! ✅")

if __name__ == "__main__":
    embed_all_books()