Adding 2 latest podcasts

build_index.py

@@ -0,0 +1,136 @@
+"""
+Build/rebuild the FAISS index from embedding chunks.
+
+Usage:
+    python build_index.py           # Full rebuild
+    python build_index.py --check   # Check index status only
+"""
+import os
+import sys
+import time
+import argparse
+import pandas as pd
+from dotenv import load_dotenv
+
+# Load environment
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+load_dotenv(dotenv_path=os.path.join(SCRIPT_DIR, '.env'))
+
+FAISS_INDEX_PATH = os.path.join(SCRIPT_DIR, "faiss_index.db")
+CHUNKS_PATH = os.path.join(SCRIPT_DIR, "data/episodes_embedding_chunks.csv")
+EPISODES_PATH = os.path.join(SCRIPT_DIR, "data/episodes_website.json")
+
+
+def check_index_status():
+    """Check the current status of data and index."""
+    print("=" * 60)
+    print("INDEX STATUS CHECK")
+    print("=" * 60)
+    
+    # Check data files
+    if os.path.exists(CHUNKS_PATH):
+        chunks_df = pd.read_csv(CHUNKS_PATH)
+        num_chunks = len(chunks_df)
+        num_episodes = chunks_df["episode_number"].nunique()
+        newest_episode = chunks_df["episode_number"].max()
+        print(f"\nData file: {CHUNKS_PATH}")
+        print(f"  Chunks: {num_chunks}")
+        print(f"  Episodes: {num_episodes}")
+        print(f"  Newest episode: #{newest_episode}")
+    else:
+        print(f"\n❌ Data file not found: {CHUNKS_PATH}")
+        return
+    
+    # Check FAISS index
+    if os.path.exists(FAISS_INDEX_PATH):
+        from langchain_openai import OpenAIEmbeddings
+        from langchain_community.vectorstores import FAISS
+        
+        embeddings = OpenAIEmbeddings()
+        vector_store = FAISS.load_local(
+            FAISS_INDEX_PATH,
+            embeddings=embeddings,
+            allow_dangerous_deserialization=True
+        )
+        index_size = vector_store.index.ntotal
+        print(f"\nFAISS index: {FAISS_INDEX_PATH}")
+        print(f"  Vectors: {index_size}")
+        
+        if index_size == num_chunks:
+            print("\n✅ Index is up to date!")
+        else:
+            print(f"\n⚠️  Index out of date: {index_size} vectors vs {num_chunks} chunks")
+            print("   Run 'python build_index.py' to rebuild")
+    else:
+        print(f"\n❌ FAISS index not found: {FAISS_INDEX_PATH}")
+        print("   Run 'python build_index.py' to create")
+
+
+def rebuild_index():
+    """Rebuild the FAISS index from scratch."""
+    from langchain_core.documents import Document
+    from langchain_openai import OpenAIEmbeddings
+    from langchain_community.vectorstores import FAISS
+    
+    print("=" * 60)
+    print("REBUILDING FAISS INDEX")
+    print("=" * 60)
+    
+    # Load embedding chunks
+    print(f"\nLoading chunks from: {CHUNKS_PATH}")
+    chunks_df = pd.read_csv(CHUNKS_PATH)
+    
+    num_chunks = len(chunks_df)
+    num_episodes = chunks_df["episode_number"].nunique()
+    newest = chunks_df["episode_number"].max()
+    
+    print(f"  Chunks: {num_chunks}")
+    print(f"  Episodes: {num_episodes}")
+    print(f"  Newest episode: #{newest}")
+    
+    # Convert to LangChain Documents
+    print("\nConverting to documents...")
+    documents = []
+    for _, row in chunks_df.iterrows():
+        doc = Document(
+            page_content=row["embedding_text"],
+            metadata={
+                "chunk_id": row["chunk_id"],
+                "episode_number": row["episode_number"],
+                "episode_slug": row["episode_slug"],
+                "chapter_title": row["chapter_title"],
+                "start_seconds": row["start_seconds"],
+                "youtube_url": row["youtube_url"],
+            }
+        )
+        documents.append(doc)
+    
+    # Create embeddings and index
+    print("\nCalling OpenAI API for embeddings...")
+    print("(This may take a few minutes for many chunks)")
+    
+    embeddings = OpenAIEmbeddings()
+    
+    t0 = time.perf_counter()
+    vector_store = FAISS.from_documents(documents, embeddings)
+    elapsed = time.perf_counter() - t0
+    
+    # Save index
+    vector_store.save_local(FAISS_INDEX_PATH)
+    
+    print(f"\n✅ FAISS index created successfully!")
+    print(f"   Vectors: {vector_store.index.ntotal}")
+    print(f"   Time: {elapsed:.1f}s")
+    print(f"   Saved to: {FAISS_INDEX_PATH}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Build FAISS index for podcast search")
+    parser.add_argument("--check", action="store_true", help="Check index status only")
+    
+    args = parser.parse_args()
+    
+    if args.check:
+        check_index_status()
+    else:
+        rebuild_index()

data/README.md

@@ -6,8 +6,8 @@ Source data and documentation for the vector index.
 
 | File | Description |
 |------|-------------|
-| `episodes_website.json` | Episode metadata (106 episodes): titles, guests, YouTube URLs |
-| `episodes_embedding_chunks.csv` | 14,514 transcript chunks with timestamps |
+| `episodes_website.json` | Episode metadata (108 episodes): titles, guests, YouTube URLs |
+| `episodes_embedding_chunks.csv` | 14,865 transcript chunks with timestamps |
 
 ## What Gets Embedded
 
@@ -41,7 +41,7 @@ AFTER (context):
 
 **Index:** FAISS flat index (`IndexFlatL2`)
 - Simple brute-force similarity search
-- Works well for our scale (~14K vectors)
+- Works well for our scale (~15K vectors)
 - No approximation — returns exact nearest neighbors
 - Trade-off: Larger indices (100K+) would benefit from IVF or HNSW for speed
 
@@ -56,11 +56,19 @@ To add new episodes or rebuild the index:
 1. Scrape new episode metadata and transcripts
 2. Process transcripts into chunks using the same BEFORE/MAIN/AFTER format
 3. Append to `episodes_embedding_chunks.csv`
-4. Rebuild the FAISS index from the full CSV
-5. Replace `faiss_index.db/` with the new index
+4. Rebuild the FAISS index:
+   ```bash
+   uv run python build_index.py
+   ```
+5. Check index status:
+   ```bash
+   uv run python build_index.py --check
+   ```
 
 The flat index must be fully rebuilt — it doesn't support incremental additions. For a production system with frequent updates, consider using a vector database like Pinecone or Weaviate that supports upserts.
 
+**Current data (as of Feb 2026):** 108 episodes (#276–#491), newest: Peter Steinberger (OpenClaw)
+
 ## Privacy
 
 No user data is stored here. All content is from publicly available podcast episodes.

data/episodes_embedding_chunks.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1165e6cdd3e3b1ec29519de6646148259930158e8e877d8f6465f7d826b16410
-size 27106060
+oid sha256:64e69c03f843ce97d2d2383f13c0cf9ab36c544657a2ec95712cc4cc3450157b
+size 27832156

data/episodes_website.json

@@ -1,4 +1,26 @@
 [
+  {
+    "episode_number": 491,
+    "guest_name": "Peter Steinberger",
+    "profile": "Creator of OpenClaw",
+    "title": "OpenClaw: The Viral AI Agent that Broke the Internet - Peter Steinberger",
+    "youtube_url": "https://www.youtube.com/watch?v=YFjfBk8HI5o",
+    "youtube_video_id": "YFjfBk8HI5o",
+    "episode_slug": "peter-steinberger",
+    "transcript_slug": "peter-steinberger-transcript",
+    "has_transcript": true
+  },
+  {
+    "episode_number": 490,
+    "guest_name": "Nathan Lambert & Sebastian Raschka",
+    "profile": "AI Researchers",
+    "title": "State of AI in 2026: LLMs, Coding, Scaling Laws, China, Agents, GPUs, AGI",
+    "youtube_url": "https://www.youtube.com/watch?v=EV7WhVT270Q",
+    "youtube_video_id": "EV7WhVT270Q",
+    "episode_slug": "ai-sota-2026",
+    "transcript_slug": "ai-sota-2026-transcript",
+    "has_transcript": true
+  },
   {
     "episode_number": 489,
     "guest_name": "Paul Rosolie",

faiss_index.db/index.faiss

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c609fc546b70173c230a6355c8417cc8a599d2a9b1c75d611c50584efcd78861
-size 89174061
+oid sha256:48131af84c43f26974390a6f078e89da9bceb4b2c2d36f5c7acbd8640f18e08d
+size 91330605

faiss_index.db/index.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c1cc082fbfe506298fb091029ec824c21c017b9fcbd591206080424ef7c3b1d
-size 27603569
+oid sha256:1816c664eac81f1cc44cb162e9055bb4e6a72c58dfd6fcdd5dde87c2f407c14c
+size 28366238