Upload 30 files

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/eu_ai_act_vectorstore/index.faiss filter=lfs diff=lfs merge=lfs -text
 data/known_bills_visualize.json filter=lfs diff=lfs merge=lfs -text
+data/data_updating_scripts/eu-ai-act.pdf filter=lfs diff=lfs merge=lfs -text

data/data_updating_scripts/PROMPTS/bill_summary_prompt.py

@@ -0,0 +1,29 @@
+# PROMPTS/bill_summary_prompt.py
+BILL_SUMMARY_PROMPT = """
+You are an expert legislative analyst specializing in AI governance and technology policy. Your task is to provide a clear, concise summary of the given bill text.
+
+Please analyze the bill and provide a comprehensive summary that includes:
+
+1. **Main Purpose**: What is the primary objective of this bill?
+2. **Key Provisions**: What are the main requirements, prohibitions, or authorizations?
+3. **AI-Related Elements**: How does this bill relate to artificial intelligence, if at all?
+4. **Scope and Impact**: Who does this bill affect and what are the potential consequences?
+5. **Implementation**: What mechanisms or processes does the bill establish?
+
+**Requirements:**
+- Keep the summary concise but comprehensive (aim for 200-400 words)
+- Use clear, professional language
+- Focus on the most important aspects of the bill
+- If the bill is not related to AI, clearly state this
+- Structure the response with clear sections using markdown formatting
+
+**Bill Information:**
+- Bill Number: {bill_number}
+- Bill Title: {bill_title}
+- State: {state}
+
+**Bill Text:**
+{bill_text}
+
+Please provide your analysis:
+""" 

data/data_updating_scripts/PROMPTS/suggested_questions_prompt.md

@@ -0,0 +1,25 @@
+You are an AI governance legislation expert. Your task is to analyze the provided bill and generate exactly 5 relevant, specific questions that users might want to ask about this particular bill.
+
+The questions should:
+- Be specific to the content and provisions of this bill
+- Cover different aspects of the legislation (definitions, scope, enforcement, compliance, etc.)
+- Be phrased as user-friendly questions that someone analyzing AI governance would ask
+- Be practical and actionable for understanding the bill's impact
+- Avoid generic questions that could apply to any bill
+
+Focus on aspects like:
+- Key definitions and terminology
+- Scope and applicability 
+- Enforcement mechanisms and penalties
+- Compliance requirements
+- Rights and obligations
+- Implementation timelines
+- Regulatory oversight
+- Specific AI technologies or systems mentioned
+
+Format your response as exactly 5 questions, one per line, with no numbering or bullet points. Each question should be complete and ready to use.
+
+### Bill Content
+{context}
+
+Generate 5 specific questions about this bill: 

data/data_updating_scripts/build_bills_vectorstore.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import argparse, os
+from pathlib import Path
+from dotenv import load_dotenv
+load_dotenv(dotenv_path=Path.cwd() / ".env")
+
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--source", default="data/known_bills_visualize.json")
+    p.add_argument("--backend", choices=["chroma","pinecone"], default=os.getenv("VECTOR_BACKEND","chroma"))
+    p.add_argument("--persist", default="data/bills_vectorstore")
+    p.add_argument("--collection", default="bills")
+    p.add_argument("--manifest", default="data/bills_vectorstore_manifest.json")
+    p.add_argument("--model", default=None)
+    p.add_argument("--batch", type=int, default=128)
+    args = p.parse_args()
+
+    if args.backend == "pinecone":
+        from vectorstore.pinecone_bills_vectorstore import upsert_from_bills_json
+        stats = upsert_from_bills_json(
+            source_json_path=args.source,
+            manifest_path=args.manifest,
+            embed_model=args.model,
+            batch_size=args.batch,
+        )
+    else:
+        from vectorstore.bills_vectorstore import upsert_from_bills_json
+        stats = upsert_from_bills_json(
+            source_json_path=args.source,
+            persist_dir=args.persist,
+            collection=args.collection,
+            manifest_path=args.manifest,
+            embed_model=args.model,
+            batch_size=args.batch,
+        )
+
+    print("✅ Vectorstore updated")
+    for k, v in stats.items():
+        print(f"  {k}: {v}")
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/build_bills_vectorstore_pinecone_delta.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import os, json, time
+from pathlib import Path
+from typing import List, Dict, Any
+from dotenv import load_dotenv
+import sys
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+load_dotenv(dotenv_path=Path.cwd() / ".env")
+
+from vectorstore.pinecone_delta_upsert import chunk_bill, upsert_changed_vectors
+
+SRC = "data/known_bills_visualize.json"
+BATCH = int(os.getenv("PINECONE_BATCH", "128"))
+
+def main():
+    p = Path(SRC)
+    if not p.exists():
+        raise SystemExit(f"Missing {SRC}")
+    bills: List[Dict[str, Any]] = json.loads(p.read_text(encoding="utf-8"))
+    bills = [b for b in bills if (b.get("text") or b.get("description") or b.get("title"))]
+
+    chunks: List[Dict[str, Any]] = []
+    for b in bills:
+        chunks.extend(chunk_bill(b))
+
+    print(f"Total chunks computed: {len(chunks):,}")
+
+    changed = 0
+    t0 = time.time()
+    for i in range(0, len(chunks), BATCH):
+        batch = chunks[i:i+BATCH]
+        changed += upsert_changed_vectors(batch)
+        if (i // BATCH) % 10 == 0:
+            print(f"… {i+len(batch):,}/{len(chunks):,} processed")
+    dt = time.time() - t0
+
+    print("✅ Pinecone delta upsert complete")
+    print(f"   changed_upserts: {changed}")
+    print(f"   elapsed_sec: {dt:.1f}")
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/config.py

@@ -0,0 +1,43 @@
+"""Configuration settings for LegiScan AI Governance Bills Tracker."""
+import os
+from pathlib import Path
+import dotenv
+
+dotenv.load_dotenv()
+
+class ConfigManager:
+    def __init__(self):
+        """
+        Initialize configuration with profile-specific settings.
+
+        Args:
+            profile (str): Configuration profile (production, development, testing)
+        """
+        self._load_base_config()
+
+    def _load_base_config(self):
+        """Load base configuration that applies to all profiles."""
+        self.OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+        self.OPENAI_LLM_MODEL = os.getenv("OPENAI_LLM_MODEL", "gpt-4o")
+
+    def reload(self):
+        """Reload configuration."""
+        self._load_base_config()
+        self._load_profile_config(self.profile)
+        self._validate_config()
+
+    def __str__(self) -> str:
+        """Return string representation of non-sensitive config."""
+        sensitive_keys = ["OPENAI_API_KEY", "LEGISCAN_API_KEY"]
+        config_str = f"Configuration Profile: {self.profile}\n"
+        for key, value in self.__dict__.items():
+            if key.startswith("_"):
+                continue
+            if key in sensitive_keys:
+                config_str += f"{key}: {'*' * 8}\n"
+            else:
+                config_str += f"{key}: {value}\n"
+        return config_str
+
+# Create default instance
+config = ConfigManager()

data/data_updating_scripts/eu-ai-act.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba630444b3278e881066774002a1d7824308934f49ccfa203e65be43692f55e
+size 2583319

data/data_updating_scripts/eu_vectorstore.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+# scripts/create_eu_ai_act_vectorstore.py
+
+"""
+Script to create and save a vectorstore from the EU AI Act PDF.
+This creates a FAISS vectorstore that can be loaded quickly in the main app.
+"""
+
+import os
+import logging
+from pathlib import Path
+import pickle
+from typing import Optional
+import dotenv
+
+# Import config
+from config import config
+
+# PDF processing
+import PyPDF2
+
+# LangChain components
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.schema import Document
+
+# Load environment variables
+dotenv.load_dotenv()
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/eu_vectorstore.log")],
+)
+
+logger = logging.getLogger(__name__)
+
+def extract_text_from_pdf(pdf_path: str) -> str:
+    """Extract text from PDF file."""
+    try:
+        with open(pdf_path, 'rb') as file:
+            pdf_reader = PyPDF2.PdfReader(file)
+            text = ""
+            
+            logger.info(f"Processing {len(pdf_reader.pages)} pages from {pdf_path}")
+            
+            for page_num, page in enumerate(pdf_reader.pages):
+                try:
+                    page_text = page.extract_text()
+                    text += f"\n\n--- Page {page_num + 1} ---\n\n{page_text}"
+                except Exception as e:
+                    logger.warning(f"Error extracting text from page {page_num + 1}: {e}")
+                    continue
+            
+            logger.info(f"Extracted {len(text)} characters from PDF")
+            return text
+            
+    except Exception as e:
+        logger.error(f"Error reading PDF {pdf_path}: {e}")
+        raise e
+
+def create_eu_ai_act_documents(text_content: str) -> list:
+    """Convert EU AI Act text to Document objects with metadata."""
+    try:
+        # Initialize text splitter with appropriate settings for legal documents
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1500,  # Larger chunks for legal text
+            chunk_overlap=200,  # More overlap for context preservation
+            length_function=len,
+            separators=["\n\n", "\n", ". ", " ", ""]
+        )
+        
+        # Create initial document
+        doc = Document(
+            page_content=text_content,
+            metadata={
+                'source': 'EU AI Act',
+                'document_type': 'regulation',
+                'jurisdiction': 'European Union',
+                'title': 'Regulation (EU) 2024/1689 on Artificial Intelligence (AI Act)'
+            }
+        )
+        
+        # Split into chunks
+        splits = text_splitter.split_documents([doc])
+        
+        # Add chunk-specific metadata
+        for i, split in enumerate(splits):
+            split.metadata.update({
+                'chunk_id': i,
+                'total_chunks': len(splits)
+            })
+        
+        logger.info(f"Created {len(splits)} document chunks")
+        return splits
+        
+    except Exception as e:
+        logger.error(f"Error creating documents: {e}")
+        raise e
+
+def create_and_save_eu_vectorstore(
+    pdf_path: str = "data_updating_scripts/eu-ai-act.pdf", 
+    vectorstore_path: str = "data/eu_ai_act_vectorstore",
+    openai_api_key: Optional[str] = None
+) -> bool:
+    """
+    Create FAISS vectorstore from EU AI Act PDF and save it locally.
+    
+    Args:
+        pdf_path: Path to the EU AI Act PDF file
+        vectorstore_path: Directory to save the vectorstore
+        openai_api_key: OpenAI API key (if not provided, uses environment variable)
+    
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    try:
+        # Check if PDF exists
+        if not Path(pdf_path).exists():
+            logger.error(f"PDF file not found: {pdf_path}")
+            return False
+        
+        # Get API key
+        api_key = openai_api_key or config.OPENAI_API_KEY
+        if not api_key:
+            logger.error("OpenAI API key not found")
+            return False
+        
+        logger.info("Starting EU AI Act vectorstore creation...")
+        
+        # Extract text from PDF
+        logger.info("Extracting text from PDF...")
+        text_content = extract_text_from_pdf(pdf_path)
+        
+        if not text_content or len(text_content) < 1000:
+            logger.error("Insufficient text extracted from PDF")
+            return False
+        
+        # Create documents
+        logger.info("Creating document chunks...")
+        documents = create_eu_ai_act_documents(text_content)
+        
+        if not documents:
+            logger.error("No documents created")
+            return False
+        
+        # Initialize embeddings
+        logger.info("Initializing embeddings...")
+        embeddings = OpenAIEmbeddings(
+            api_key=api_key,
+            model="text-embedding-3-small"
+        )
+        
+        # Create vectorstore
+        logger.info("Creating FAISS vectorstore...")
+        vectorstore = FAISS.from_documents(documents, embeddings)
+        
+        # Create directory if it doesn't exist
+        Path(vectorstore_path).mkdir(exist_ok=True)
+        
+        # Save vectorstore
+        logger.info(f"Saving vectorstore to {vectorstore_path}...")
+        vectorstore.save_local(vectorstore_path)
+        
+        # Save metadata
+        metadata = {
+            'pdf_path': pdf_path,
+            'total_chunks': len(documents),
+            'text_length': len(text_content),
+            'embedding_model': 'text-embedding-3-small',
+            'chunk_size': 1500,
+            'chunk_overlap': 200
+        }
+        
+        metadata_path = Path(vectorstore_path) / "metadata.pickle"
+        with open(metadata_path, 'wb') as f:
+            pickle.dump(metadata, f)
+        
+        logger.info(f"✅ EU AI Act vectorstore created successfully!")
+        logger.info(f"   - Total chunks: {len(documents)}")
+        logger.info(f"   - Text length: {len(text_content):,} characters")
+        logger.info(f"   - Saved to: {vectorstore_path}")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"Error creating EU AI Act vectorstore: {e}")
+        return False
+
+def load_eu_vectorstore(
+    vectorstore_path: str = "eu_ai_act_vectorstore",
+    openai_api_key: Optional[str] = None
+) -> Optional[FAISS]:
+    """
+    Load the EU AI Act vectorstore from disk.
+    
+    Args:
+        vectorstore_path: Path to the saved vectorstore
+        openai_api_key: OpenAI API key
+    
+    Returns:
+        FAISS vectorstore or None if failed
+    """
+    try:
+        if not Path(vectorstore_path).exists():
+            logger.error(f"Vectorstore not found: {vectorstore_path}")
+            return None
+        
+        # Get API key
+        api_key = openai_api_key or config.OPENAI_API_KEY
+        if not api_key:
+            logger.error("OpenAI API key not found")
+            return None
+        
+        # Initialize embeddings
+        embeddings = OpenAIEmbeddings(
+            api_key=api_key,
+            model="text-embedding-3-small"
+        )
+        
+        # Load vectorstore
+        vectorstore = FAISS.load_local(
+            vectorstore_path, 
+            embeddings,
+            allow_dangerous_deserialization=True  # Required for loading pickled objects
+        )
+        
+        logger.info(f"✅ EU AI Act vectorstore loaded from {vectorstore_path}")
+        return vectorstore
+        
+    except Exception as e:
+        logger.error(f"Error loading EU AI Act vectorstore: {e}")
+        return None
+
+def get_vectorstore_info(vectorstore_path: str = "data/eu_ai_act_vectorstore") -> dict:
+    """Get information about the saved vectorstore."""
+    try:
+        metadata_path = Path(vectorstore_path) / "metadata.pickle"
+        if metadata_path.exists():
+            with open(metadata_path, 'rb') as f:
+                metadata = pickle.load(f)
+            return metadata
+        else:
+            return {"error": "Metadata not found"}
+    except Exception as e:
+        return {"error": str(e)}
+
+if __name__ == "__main__":
+    # Create the vectorstore
+    success = create_and_save_eu_vectorstore()
+    
+    if success:
+        # Display info
+        info = get_vectorstore_info()
+        print("\n" + "="*50)
+        print("EU AI Act Vectorstore Information:")
+        print("="*50)
+        for key, value in info.items():
+            if key != 'error':
+                print(f"{key}: {value}")
+        print("="*50)
+    else:
+        print("❌ Failed to create EU AI Act vectorstore")
+        exit(1)

data/data_updating_scripts/fix_pdf_bills.py

@@ -0,0 +1,282 @@
+import os
+import json
+import base64
+import logging
+import sys
+from datetime import datetime, timezone
+import requests
+from dotenv import load_dotenv
+import PyPDF2
+from io import BytesIO
+import re
+import shutil
+
+# Load environment variables
+load_dotenv()
+API_KEY = os.getenv("LEGISCAN_API_KEY")
+
+# Files
+INPUT_FILE = "data/known_bills.json"
+OUTPUT_FILE = "data/known_bills_fixed.json"
+BACKUP_FILE = "data/known_bills_backup.json"
+
+# Rate limiting
+import time
+RATE_LIMIT = 0.2  # seconds between API requests
+
+# Logging configuration
+LOG_FILE = "data_updating_scripts/logs/fix_pdf_bills.log"
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(LOG_FILE)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+def is_pdf_content(text):
+    """Check if the text content is an unprocessed PDF."""
+    if not text:
+        return False
+    # Check for PDF header signatures
+    pdf_signatures = ["%PDF-1.3", "%PDF-1.4", "%PDF-1.5", "%PDF-1.6", "%PDF-1.7", "%PDF1.3", "%PDF1.4", "%PDF1.5", "%PDF1.6", "%PDF1.7"]
+    text_start = text[:20] if len(text) >= 20 else text
+    return any(text_start.startswith(sig) for sig in pdf_signatures)
+
+
+def extract_text_from_pdf_bytes(pdf_bytes):
+    """Extract text from PDF bytes using PyPDF2."""
+    try:
+        pdf_file = BytesIO(pdf_bytes)
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        
+        text_content = []
+        for page_num in range(len(pdf_reader.pages)):
+            page = pdf_reader.pages[page_num]
+            page_text = page.extract_text()
+            if page_text:
+                text_content.append(page_text)
+        
+        full_text = "\n".join(text_content)
+        
+        # Clean up the extracted text
+        # Remove excessive whitespace while preserving paragraph breaks
+        full_text = re.sub(r'\n{3,}', '\n\n', full_text)
+        full_text = re.sub(r' {2,}', ' ', full_text)
+        full_text = full_text.strip()
+        
+        return full_text
+    except Exception as e:
+        logger.error(f"Error extracting text from PDF: {e}")
+        return None
+
+
+def legi_request(op, params):
+    """Make a request to the LegiScan API."""
+    base = "https://api.legiscan.com/"
+    params.update({"key": API_KEY, "op": op})
+    try:
+        resp = requests.get(base, params=params, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("status") != "OK":
+            logger.error(f"API error {op}: {data.get('message', data)}")
+            return None
+        return data
+    except requests.RequestException as e:
+        logger.error(f"Request failed ({op}): {e}")
+        return None
+
+
+def fix_pdf_bill(bill):
+    """Fix a single bill with unprocessed PDF content."""
+    bill_id = bill.get("bill_id")
+    state = bill.get("state")
+    bill_num = bill.get("bill_number")
+    
+    logger.info(f"Fixing PDF content for {state} {bill_num} (ID: {bill_id})")
+    
+    # First, try to get the bill details again
+    details_resp = legi_request("getBill", {"id": bill_id})
+    if not details_resp:
+        logger.warning(f"Could not fetch bill details for {bill_id}")
+        return None
+    
+    details = details_resp.get("bill", {})
+    texts = details.get("texts", [])
+    
+    if not texts:
+        logger.warning(f"No text documents available for {bill_id}")
+        return None
+    
+    # Try to get the text document
+    doc_id = texts[0].get("doc_id")
+    text_resp = legi_request("getBillText", {"id": doc_id})
+    
+    if not text_resp or "text" not in text_resp:
+        logger.warning(f"Could not fetch text for {bill_id}")
+        return None
+    
+    raw_b64 = text_resp["text"].get("doc", "")
+    if not raw_b64:
+        logger.warning(f"No document content for {bill_id}")
+        return None
+    
+    try:
+        # Decode the base64 content
+        decoded = base64.b64decode(raw_b64)
+        
+        # Check if it's a PDF by looking at the magic bytes
+        if decoded[:4] == b'%PDF':
+            # It's a PDF, extract text
+            extracted_text = extract_text_from_pdf_bytes(decoded)
+            if extracted_text and len(extracted_text.strip()) > 100:  # Ensure we got meaningful text
+                logger.info(f"Successfully extracted {len(extracted_text)} characters from PDF for {bill_id}")
+                return extracted_text
+            else:
+                logger.warning(f"Extracted text too short or empty for {bill_id}")
+                return None
+        else:
+            # Try to decode as HTML (shouldn't happen for these cases, but just in case)
+            try:
+                from bs4 import BeautifulSoup
+                html = decoded.decode("utf-8", errors="ignore")
+                soup = BeautifulSoup(html, "html.parser")
+                plain_text = soup.get_text(separator="\n", strip=True)
+                if plain_text and len(plain_text.strip()) > 100:
+                    logger.info(f"Successfully extracted HTML text for {bill_id}")
+                    return plain_text
+            except:
+                pass
+                
+        logger.warning(f"Could not process document for {bill_id}")
+        return None
+        
+    except Exception as e:
+        logger.error(f"Error processing document for {bill_id}: {e}")
+        return None
+
+
+def main(overwrite: bool | None = None):
+    # Load the bills
+    logger.info(f"Loading bills from {INPUT_FILE}")
+    try:
+        with open(INPUT_FILE, 'r') as f:
+            bills = json.load(f)
+    except Exception as e:
+        logger.error(f"Could not load bills file: {e}")
+        sys.exit(1)
+    
+    logger.info(f"Loaded {len(bills)} bills")
+    
+    # Create a backup
+    logger.info(f"Creating backup at {BACKUP_FILE}")
+    with open(BACKUP_FILE, 'w') as f:
+        json.dump(bills, f, indent=2)
+    
+    # Find bills with unprocessed PDF content
+    pdf_bills = []
+    for i, bill in enumerate(bills):
+        if is_pdf_content(bill.get("text")):
+            pdf_bills.append(i)
+    
+    logger.info(f"Found {len(pdf_bills)} bills with unprocessed PDF content")
+    
+    # Process each PDF bill
+    fixed_count = 0
+    failed_count = 0
+    
+    for idx, bill_idx in enumerate(pdf_bills):
+        bill = bills[bill_idx]
+        logger.info(f"Processing {idx + 1}/{len(pdf_bills)}: {bill.get('state')} {bill.get('bill_number')}")
+        
+        # Try to fix the PDF content
+        fixed_text = fix_pdf_bill(bill)
+        
+        if fixed_text:
+            # Update the bill with the fixed text
+            bills[bill_idx]["text"] = fixed_text
+            bills[bill_idx]["lastUpdatedAt"] = datetime.now(timezone.utc).isoformat()
+            bills[bill_idx]["text_fixed"] = True  # Mark that we fixed this
+            fixed_count += 1
+            logger.info(f"Successfully fixed bill {bill.get('bill_id')}")
+        else:
+            # Mark that we tried but failed
+            bills[bill_idx]["text_extraction_failed"] = True
+            bills[bill_idx]["lastUpdatedAt"] = datetime.now(timezone.utc).isoformat()
+            failed_count += 1
+            logger.warning(f"Failed to fix bill {bill.get('bill_id')}")
+        
+        # Rate limiting
+        time.sleep(RATE_LIMIT)
+        
+        # Save progress every 50 bills
+        if (idx + 1) % 50 == 0:
+            logger.info(f"Saving progress... ({idx + 1}/{len(pdf_bills)} processed)")
+            with open(OUTPUT_FILE, 'w') as f:
+                json.dump(bills, f, indent=2)
+    
+    # Save final results
+    logger.info(f"Saving final results to {OUTPUT_FILE}")
+    with open(OUTPUT_FILE, 'w') as f:
+        json.dump(bills, f, indent=2)
+    
+    logger.info(f"Processing complete!")
+    logger.info(f"Successfully fixed: {fixed_count} bills")
+    logger.info(f"Failed to fix: {failed_count} bills")
+    logger.info(f"Output saved to: {OUTPUT_FILE}")
+
+    if fixed_count > 0:
+        # Decide overwrite behavior
+        if overwrite is None:
+            # CLI mode: ask the user (guardrail preserved)
+            try:
+                response = input(
+                    f"\nDo you want to overwrite {INPUT_FILE} with the fixed data? (y/n): "
+                )
+            except EOFError:
+                logger.error(
+                    "No input available (EOF). Leaving original file unchanged."
+                )
+                return
+            overwrite_flag = response.strip().lower().startswith("y")
+        else:
+            # Non-interactive mode (e.g. Streamlit pipeline)
+            overwrite_flag = overwrite
+
+        if overwrite_flag:
+            shutil.copy2(OUTPUT_FILE, INPUT_FILE)
+            logger.info(f"Original file {INPUT_FILE} has been updated with fixed data.")
+        else:
+            logger.info("Overwrite declined; original file left unchanged.")
+
+
+
+if __name__ == "__main__":
+    # If running under Streamlit / pipeline, we expect FIX_PDF_OVERWRITE in env:
+    #   "yes", "y", "true", "1"  -> overwrite=True
+    #   "no", "n", "false", "0"  -> overwrite=False
+    # If it's not set, we fall back to CLI mode and ask via input().
+    env_choice = os.getenv("FIX_PDF_OVERWRITE")
+
+    if env_choice is None:
+        # Local CLI run → still interactive
+        main(overwrite=None)
+    else:
+        choice = env_choice.strip().lower()
+        if choice in ("yes", "y", "true", "1"):
+            main(overwrite=True)
+        elif choice in ("no", "n", "false", "0"):
+            main(overwrite=False)
+        else:
+            logger.warning(
+                f"Invalid FIX_PDF_OVERWRITE='{env_choice}', defaulting to no overwrite."
+            )
+            main(overwrite=False)
+
+

data/data_updating_scripts/generate_reports.py

@@ -0,0 +1,274 @@
+"""
+generate_reports.py
+--------------------
+
+Generates detailed Markdown reports for AI-related bills from `known_bills_visualize.json`
+using the latest LangChain pipeline syntax.
+
+Now includes resume functionality - can be safely stopped and restarted.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+import dotenv
+
+
+dotenv.load_dotenv()
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Latest LangChain imports
+try:
+    from langchain_openai import ChatOpenAI
+    from langchain.prompts import ChatPromptTemplate
+except ImportError:  # pragma: no cover
+    ChatOpenAI = None  # type: ignore
+    ChatPromptTemplate = None  # type: ignore
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/generate_reports.log")],
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BillReport:
+    """Stores a bill ID and its generated detailed report."""
+    bill_id: str
+    report_markdown: str
+
+
+# Prompt template
+DETAILED_REPORT_PROMPT = ChatPromptTemplate.from_template(
+    """You are a seasoned legislative analyst adept at interpreting and
+    summarising bills related to artificial intelligence. Using the bill
+    information provided as JSON, produce a detailed report in Markdown
+    format for stakeholders.
+
+    Include:
+    - Bill's title, number, and state
+    - Status and key dates
+    - URL to the bill on legiscan
+    - Sponsors and scope
+    - Goals and intent
+    - Key provisions, regulatory approaches, implementation & enforcement
+    - Unique aspects or notable features
+
+    Format:
+    - Use Markdown headings and bullet points
+    - Paraphrase content
+    - Do not invent facts
+    - If bill text is truncated in source JSON, note this at the end
+
+    Bill JSON:
+    ```json
+    {bill_json}
+    ```
+
+    Now craft the detailed report.
+    """
+)
+
+
+def _ensure_llm() -> ChatOpenAI:
+    """Initialise ChatOpenAI with latest settings."""
+    if ChatOpenAI is None:
+        raise RuntimeError(
+            "The 'langchain' and 'openai' packages are required. Install them via 'pip install langchain openai'."
+        )
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("The OPENAI_API_KEY environment variable is not set.")
+    model_name = os.getenv("MODEL_NAME", "gpt-4o")
+    logger.debug("Initialising ChatOpenAI with model %s", model_name)
+    return ChatOpenAI(model=model_name, temperature=0)
+
+
+def create_detailed_report(
+    bill: Dict[str, Any], *, llm: Optional[ChatOpenAI] = None
+) -> BillReport:
+    """Generate a detailed report for a single bill using latest LangChain syntax."""
+    if llm is None:
+        llm = _ensure_llm()
+
+    bill_json = json.dumps(bill, ensure_ascii=False, indent=2)
+
+    # Latest syntax: prompt | llm
+    chain = DETAILED_REPORT_PROMPT | llm
+    result = chain.invoke({"bill_json": bill_json})
+
+    # result can be AIMessage; get text
+    report_text = getattr(result, "content", str(result))
+
+    return BillReport(bill_id=str(bill.get("bill_id")), report_markdown=report_text)
+
+
+def load_existing_reports(output_path: str) -> Dict[str, str]:
+    """Load existing reports from file if it exists."""
+    if os.path.exists(output_path):
+        try:
+            with open(output_path, "r", encoding="utf-8") as f:
+                reports_list = json.load(f)
+                # Convert list to dict for easy lookup
+                reports_dict = {
+                    report["bill_id"]: report["report_markdown"]
+                    for report in reports_list
+                    if "bill_id" in report and "report_markdown" in report
+                }
+                logger.info(f"Loaded {len(reports_dict)} existing reports from {output_path}")
+                return reports_dict
+        except Exception as e:
+            logger.warning(f"Could not load existing reports: {e}")
+            return {}
+    return {}
+
+
+def save_reports_to_file(reports_dict: Dict[str, str], output_path: str) -> None:
+    """Save reports dictionary to a JSON file."""
+    # Convert dict back to list format for consistency
+    out_list = [
+        {"bill_id": bill_id, "report_markdown": report_markdown}
+        for bill_id, report_markdown in reports_dict.items()
+    ]
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(out_list, f, ensure_ascii=False, indent=2)
+    logger.info("Saved %d reports to %s", len(out_list), output_path)
+
+
+def create_reports_with_resume(
+    bills: List[Dict[str, Any]], 
+    output_path: str,
+    *, 
+    llm: Optional[ChatOpenAI] = None,
+    save_interval: int = 10
+) -> Dict[str, str]:
+    """
+    Generate detailed reports for multiple bills with resume capability.
+    
+    Args:
+        bills: List of bill dictionaries
+        output_path: Path to save reports
+        llm: Optional LLM instance
+        save_interval: Save progress every N bills
+    
+    Returns:
+        Dictionary of bill_id -> report_markdown
+    """
+    if not bills:
+        return {}
+    
+    if llm is None:
+        llm = _ensure_llm()
+    
+    # Load existing reports
+    reports_dict = load_existing_reports(output_path)
+    
+    # Track progress
+    total_bills = len(bills)
+    processed = 0
+    skipped = 0
+    errors = 0
+    
+    logger.info(f"Starting report generation for {total_bills} bills")
+    
+    for i, bill in enumerate(bills, 1):
+        bill_id = str(bill.get("bill_id"))
+        
+        # Skip if already processed
+        if bill_id in reports_dict and reports_dict[bill_id] and not reports_dict[bill_id].startswith("ERROR:"):
+            logger.info(f"Skipping bill {bill_id} - already processed ({i}/{total_bills})")
+            skipped += 1
+            continue
+        
+        logger.info(f"Processing {i}/{total_bills}: Bill ID {bill_id}")
+        
+        try:
+            report = create_detailed_report(bill, llm=llm)
+            reports_dict[bill_id] = report.report_markdown
+            processed += 1
+            
+        except Exception as exc:
+            logger.exception(
+                "Failed to generate report for bill %s: %s", bill_id, exc
+            )
+            reports_dict[bill_id] = f"ERROR: Failed to generate report - {str(exc)}"
+            errors += 1
+        
+        # Save progress periodically
+        if i % save_interval == 0:
+            save_reports_to_file(reports_dict, output_path)
+            logger.info(f"Progress: {i}/{total_bills} - Processed: {processed}, Skipped: {skipped}, Errors: {errors}")
+        
+        # Rate limiting to avoid API throttling
+        if bill_id not in reports_dict or reports_dict[bill_id].startswith("ERROR:"):
+            time.sleep(1)  # 1 second delay between API calls
+    
+    # Final save
+    save_reports_to_file(reports_dict, output_path)
+    
+    logger.info(f"Report generation complete!")
+    logger.info(f"Total bills: {total_bills}")
+    logger.info(f"Successfully processed: {processed}")
+    logger.info(f"Skipped (already done): {skipped}")
+    logger.info(f"Errors: {errors}")
+    
+    return reports_dict
+
+
+def read_bills_from_file(path: str) -> List[Dict[str, Any]]:
+    """Read bill records from a JSON file."""
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+        if not isinstance(data, list):
+            raise ValueError(f"Expected list of bills in {path}, got {type(data)}")
+        return data
+
+
+def generate_reports_from_files(
+    input_path: str = "data/known_bills_visualize.json",
+    output_path: str = "data/bill_reports.json",
+) -> None:
+    """Read bills, generate reports with resume capability, and write them to disk."""
+    bills = read_bills_from_file(input_path)
+    create_reports_with_resume(bills, output_path)
+
+
+def main() -> None:
+    import argparse
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    parser = argparse.ArgumentParser(
+        description="Generate detailed AI legislation reports from bill data with resume capability."
+    )
+    parser.add_argument("--input", default="data/known_bills_visualize.json", help="Path to input JSON file")
+    parser.add_argument("--output", default="data/bill_reports.json", help="Path to output JSON file")
+    parser.add_argument("--save-interval", type=int, default=10, help="Save progress every N bills (default: 10)")
+    args = parser.parse_args()
+    
+    try:
+        bills = read_bills_from_file(args.input)
+        create_reports_with_resume(bills, args.output, save_interval=args.save_interval)
+        print(f"✅ Report generation completed successfully!")
+        print(f"   Reports saved to: {args.output}")
+    except Exception as e:
+        logger.error(f"Fatal error: {e}")
+        print(f"❌ Error: {e}")
+        import sys
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/generate_suggested_questions.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""
+Script to generate suggested questions for all bills in known_bills_visualize.json.
+
+This script reads all bills from known_bills_visualize.json, generates 5 suggested questions using OpenAI API,
+and saves them to data/bill_suggested_questions.json to avoid repeated API calls.
+"""
+
+import json
+import logging
+import time
+import pandas as pd
+from pathlib import Path
+from typing import Dict, List, Optional
+import sys
+import os
+
+# Add the project root to the path
+sys.path.append(str(Path(__file__).parent.parent))
+
+from config import ConfigManager
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.documents import Document
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/generate_suggested_questions.log")]
+)
+logger = logging.getLogger(__name__)
+
+class SuggestedQuestionsGenerator:
+    """Generates suggested questions for all bills in known_bills_visualize.json."""
+    
+    def __init__(self):
+        """Initialize the questions generator with configuration."""
+        self.config = ConfigManager()
+        self.known_bills_file = Path("data/known_bills_visualize.json")
+        self.questions_file = Path("data/bill_suggested_questions.json")
+        
+        # Initialize OpenAI LLM
+        if not self.config.OPENAI_API_KEY:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+        
+        self.llm = ChatOpenAI(
+            model=self.config.OPENAI_LLM_MODEL,
+            temperature=0.3,
+            max_tokens=500
+        )
+        
+        # Load the system prompt from markdown file
+        prompt_path = "data_updating_scripts/PROMPTS/suggested_questions_prompt.md"
+        if not os.path.exists(prompt_path):
+            raise FileNotFoundError(f"The specified file was not found: {prompt_path}")
+        
+        with open(prompt_path, "r") as file:
+            system_prompt = file.read()
+        
+        # Create the prompt and chain
+        self.prompt = ChatPromptTemplate.from_messages(
+            [
+                ("system", system_prompt),
+                ("human", "Generate 5 specific questions about this bill based on its content."),
+            ]
+        )
+        
+        self.question_generation_chain = create_stuff_documents_chain(
+            self.llm, self.prompt
+        )
+        
+        # Fallback questions
+        self.fallback_questions = [
+            "What are the key definitions in this bill?",
+            "What are the enforcement mechanisms?",
+            "Who does this bill apply to?",
+            "What are the compliance requirements?",
+            "What penalties are specified?"
+        ]
+        
+        logger.info(f"Initialized SuggestedQuestionsGenerator with model: {self.config.OPENAI_LLM_MODEL}")
+    
+    def dataframe_to_documents(self, df):
+        """Convert DataFrame to list of Document objects."""
+        documents = []
+        for _, row in df.iterrows():
+            if 'text' in row and pd.notna(row['text']) and row['text'].strip():
+                doc = Document(
+                    page_content=row['text'],
+                    metadata={
+                        'bill_key': f"{row.get('state', 'Unknown')}_{row.get('bill_number', 'Unknown')}",
+                        'state': row.get('state', 'Unknown'),
+                        'bill_number': row.get('bill_number', 'Unknown'),
+                        'title': row.get('title', 'No title')
+                    }
+                )
+                documents.append(doc)
+        return documents
+    
+    def load_known_bills(self) -> List[Dict]:
+        """Load bills from known_bills_visualize.json."""
+        try:
+            with open(self.known_bills_file, 'r', encoding='utf-8') as f:
+                bills = json.load(f)
+            logger.info(f"Loaded {len(bills)} bills from {self.known_bills_file}")
+            return bills
+        except FileNotFoundError:
+            logger.error(f"File not found: {self.known_bills_file}")
+            raise
+        except json.JSONDecodeError as e:
+            logger.error(f"Error parsing JSON: {e}")
+            raise
+    
+    def load_existing_questions(self) -> Dict:
+        """Load existing questions if available."""
+        if self.questions_file.exists():
+            try:
+                with open(self.questions_file, 'r', encoding='utf-8') as f:
+                    questions = json.load(f)
+                logger.info(f"Loaded {len(questions)} existing question sets")
+                return questions
+            except Exception as e:
+                logger.warning(f"Could not load existing questions: {e}")
+                return {}
+        return {}
+    
+    def save_questions(self, questions: Dict) -> None:
+        """Save questions to JSON file."""
+        try:
+            with open(self.questions_file, 'w', encoding='utf-8') as f:
+                json.dump(questions, f, indent=2, ensure_ascii=False)
+            logger.info(f"Saved {len(questions)} question sets to {self.questions_file}")
+        except Exception as e:
+            logger.error(f"Error saving questions: {e}")
+            raise
+    
+    def parse_questions_response(self, response: str) -> List[str]:
+        """Parse the LLM response into individual questions."""
+        questions = []
+        if isinstance(response, str):
+            # Split by lines and clean up
+            lines = [line.strip() for line in response.split('\n') if line.strip()]
+            # Filter out any numbering or bullet points
+            for line in lines:
+                # Remove common prefixes like "1.", "2.", "3.", "4.", "5.", "•", "-", "*", etc.
+                clean_line = line
+                if line.startswith(('1.', '2.', '3.', '4.', '5.', '•', '-', '*')):
+                    clean_line = line[2:].strip()
+                elif line.startswith(('1)', '2)', '3)', '4)', '5)')):
+                    clean_line = line[2:].strip()
+                
+                if clean_line and clean_line.endswith('?'):
+                    questions.append(clean_line)
+        
+        # Ensure we have exactly 5 questions
+        if len(questions) < 5:
+            # Use fallback questions to fill up to 5
+            questions.extend(self.fallback_questions[len(questions):])
+        
+        return questions[:5]  # Return only the first 5
+    
+    def generate_questions(self, bill: Dict) -> Optional[List[str]]:
+        """Generate suggested questions for a single bill."""
+        try:
+            bill_number = bill.get('bill_number', 'Unknown')
+            bill_title = bill.get('title', 'No title')
+            bill_text = bill.get('text', '')
+            
+            if not bill_text:
+                logger.warning(f"No text found for bill {bill_number}")
+                return self.fallback_questions
+            
+            # Convert bill to document format
+            df = pd.DataFrame([bill])
+            docs = self.dataframe_to_documents(df)
+            
+            if not docs:
+                logger.warning(f"No document created for bill {bill_number}")
+                return self.fallback_questions
+            
+            # Generate questions using the chain
+            response = self.question_generation_chain.invoke({"context": docs})
+            
+            # Parse the response into questions
+            questions = self.parse_questions_response(response)
+            
+            logger.info(f"Generated {len(questions)} questions for {bill_number}")
+            return questions
+            
+        except Exception as e:
+            logger.error(f"Error generating questions for bill {bill.get('bill_number', 'Unknown')}: {e}")
+            return self.fallback_questions
+    
+    def generate_all_questions(self) -> None:
+        """Generate suggested questions for all bills."""
+        # Load bills and existing questions
+        bills = self.load_known_bills()
+        existing_questions = self.load_existing_questions()
+        
+        # Track progress
+        total_bills = len(bills)
+        processed = 0
+        errors = 0
+        
+        logger.info(f"Starting question generation for {total_bills} bills")
+        
+        for i, bill in enumerate(bills, 1):
+            bill_key = f"{bill.get('state', 'Unknown')}_{bill.get('bill_number', 'Unknown')}"
+            
+            # Skip if already processed successfully
+            if bill_key in existing_questions and len(existing_questions[bill_key].get('suggested_questions', [])) == 5:
+                logger.info(f"Skipping {bill_key} - already processed")
+                processed += 1
+                continue
+            
+            logger.info(f"Processing {i}/{total_bills}: {bill_key}")
+            
+            # Generate questions
+            questions = self.generate_questions(bill)
+            
+            # Store result
+            existing_questions[bill_key] = {
+                'bill_number': bill.get('bill_number', 'Unknown'),
+                'title': bill.get('title', 'No title'),
+                'suggested_questions': questions
+            }
+            
+            if questions == self.fallback_questions:
+                errors += 1
+            else:
+                processed += 1
+            
+            # Save progress every 10 bills
+            if i % 10 == 0:
+                self.save_questions(existing_questions)
+                logger.info(f"Progress: {i}/{total_bills} processed, {errors} errors")
+            
+            # Rate limiting
+            time.sleep(1)  # 1 second delay between API calls
+        
+        # Final save
+        self.save_questions(existing_questions)
+        
+        logger.info(f"Question generation complete!")
+        logger.info(f"Total bills: {total_bills}")
+        logger.info(f"Successfully processed: {processed}")
+        logger.info(f"Errors: {errors}")
+        logger.info(f"Questions saved to: {self.questions_file}")
+
+def main():
+    """Main function to run the question generation."""
+    try:
+        generator = SuggestedQuestionsGenerator()
+        generator.generate_all_questions()
+        print("✅ Suggested questions generation completed successfully!")
+        print(f" Questions saved to: {generator.questions_file}")
+        
+    except Exception as e:
+        logger.error(f"Fatal error: {e}")
+        print(f"❌ Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main() 

data/data_updating_scripts/generate_summaries.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Script to generate summaries for all bills in known_bills_visualize.json.
+
+This script reads all bills from known_bills_visualize.json, generates summaries using OpenAI API,
+and saves them to data/bill_summaries.json to avoid repeated API calls.
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+import sys
+import os
+
+# Add the project root to the path
+sys.path.append(str(Path(__file__).parent.parent))
+
+from config import ConfigManager
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from PROMPTS.bill_summary_prompt import BILL_SUMMARY_PROMPT
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/generate_summaries.log")]
+)
+logger = logging.getLogger(__name__)
+
+class BillSummaryGenerator:
+    """Generates summaries for all bills in known_bills_visualize.json."""
+    
+    def __init__(self):
+        """Initialize the summary generator with configuration."""
+        self.config = ConfigManager()
+        self.known_bills_file = Path("data/known_bills_visualize.json")
+        self.summaries_file = Path("data/bill_summaries.json")
+        
+        # Initialize OpenAI LLM
+        if not self.config.OPENAI_API_KEY:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+        
+        self.llm = ChatOpenAI(
+            model=self.config.OPENAI_LLM_MODEL,
+            temperature=0.1,
+            max_tokens=1000
+        )
+        
+        # Create the prompt template
+        self.prompt_template = PromptTemplate(
+            template=BILL_SUMMARY_PROMPT,
+            input_variables=["bill_number", "bill_title", "state", "bill_text"]
+        )
+        
+        # Create the chain
+        self.chain = self.prompt_template | self.llm | StrOutputParser()
+        
+        logger.info(f"Initialized BillSummaryGenerator with model: {self.config.OPENAI_LLM_MODEL}")
+    
+    def load_known_bills(self) -> List[Dict]:
+        """Load bills from known_bills_visualize.json."""
+        try:
+            with open(self.known_bills_file, 'r', encoding='utf-8') as f:
+                bills = json.load(f)
+            logger.info(f"Loaded {len(bills)} bills from {self.known_bills_file}")
+            return bills
+        except FileNotFoundError:
+            logger.error(f"File not found: {self.known_bills_file}")
+            raise
+        except json.JSONDecodeError as e:
+            logger.error(f"Error parsing JSON: {e}")
+            raise
+    
+    def load_existing_summaries(self) -> Dict:
+        """Load existing summaries if available."""
+        if self.summaries_file.exists():
+            try:
+                with open(self.summaries_file, 'r', encoding='utf-8') as f:
+                    summaries = json.load(f)
+                logger.info(f"Loaded {len(summaries)} existing summaries")
+                return summaries
+            except Exception as e:
+                logger.warning(f"Could not load existing summaries: {e}")
+                return {}
+        return {}
+    
+    def save_summaries(self, summaries: Dict) -> None:
+        """Save summaries to JSON file."""
+        try:
+            with open(self.summaries_file, 'w', encoding='utf-8') as f:
+                json.dump(summaries, f, indent=2, ensure_ascii=False)
+            logger.info(f"Saved {len(summaries)} summaries to {self.summaries_file}")
+        except Exception as e:
+            logger.error(f"Error saving summaries: {e}")
+            raise
+    
+    def generate_summary(self, bill: Dict) -> Optional[str]:
+        """Generate summary for a single bill."""
+        try:
+            bill_number = bill.get('bill_number', 'Unknown')
+            bill_title = bill.get('title', 'No title')
+            state = bill.get('state', 'Unknown')
+            bill_text = bill.get('text', '')
+            
+            if not bill_text:
+                logger.warning(f"No text found for bill {bill_number}")
+                return "ERROR: No bill text available"
+            
+            # Prepare the input for the chain
+            chain_input = {
+                "bill_number": bill_number,
+                "bill_title": bill_title,
+                "state": state,
+                "bill_text": bill_text[:8000]  # Limit text length to avoid token limits
+            }
+            
+            # Generate summary using the chain
+            summary = self.chain.invoke(chain_input)
+            
+            logger.info(f"Generated summary for {bill_number}")
+            return summary
+            
+        except Exception as e:
+            logger.error(f"Error generating summary for bill {bill.get('bill_number', 'Unknown')}: {e}")
+            return f"ERROR: {str(e)}"
+    
+    def generate_all_summaries(self) -> None:
+        """Generate summaries for all bills."""
+        # Load bills and existing summaries
+        bills = self.load_known_bills()
+        existing_summaries = self.load_existing_summaries()
+        
+        # Track progress
+        total_bills = len(bills)
+        processed = 0
+        errors = 0
+        
+        logger.info(f"Starting summary generation for {total_bills} bills")
+        
+        for i, bill in enumerate(bills, 1):
+            bill_key = f"{bill.get('state', 'Unknown')}_{bill.get('bill_number', 'Unknown')}"
+            
+            # Skip if already processed successfully
+            if bill_key in existing_summaries and not existing_summaries[bill_key].get('summary', '').startswith('ERROR:'):
+                logger.info(f"Skipping {bill_key} - already processed")
+                processed += 1
+                continue
+            
+            logger.info(f"Processing {i}/{total_bills}: {bill_key}")
+            
+            # Generate summary
+            summary = self.generate_summary(bill)
+            
+            # Store result
+            existing_summaries[bill_key] = {
+                'bill_number': bill.get('bill_number', 'Unknown'),
+                'title': bill.get('title', 'No title'),
+                'summary': summary
+            }
+            
+            if summary.startswith('ERROR:'):
+                errors += 1
+            else:
+                processed += 1
+            
+            # Save progress every 10 bills
+            if i % 10 == 0:
+                self.save_summaries(existing_summaries)
+                logger.info(f"Progress: {i}/{total_bills} processed, {errors} errors")
+            
+            # Rate limiting
+            time.sleep(1)  # 1 second delay between API calls
+        
+        # Final save
+        self.save_summaries(existing_summaries)
+        
+        logger.info(f"Summary generation complete!")
+        logger.info(f"Total bills: {total_bills}")
+        logger.info(f"Successfully processed: {processed}")
+        logger.info(f"Errors: {errors}")
+        logger.info(f"Summaries saved to: {self.summaries_file}")
+
+def main():
+    """Main function to run the summary generation."""
+    try:
+        generator = BillSummaryGenerator()
+        generator.generate_all_summaries()
+        print("✅ Summary generation completed successfully!")
+        print(f" Summaries saved to: {generator.summaries_file}")
+        
+    except Exception as e:
+        logger.error(f"Fatal error: {e}")
+        print(f"❌ Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main() 

data/data_updating_scripts/get_data.py

@@ -0,0 +1,251 @@
+import os
+import sys
+import json
+import time
+import logging
+import base64
+from datetime import datetime, timezone
+import requests
+from dotenv import load_dotenv
+from bs4 import BeautifulSoup
+
+# Load environment variables from .env file
+load_dotenv()
+# Pull API key from environment
+API_KEY = os.getenv("LEGISCAN_API_KEY")  # Set your LegiScan API key in .env
+if not API_KEY:
+    print("Error: Please set LEGISCAN_API_KEY in your .env file.")
+    sys.exit(1)
+
+# Modes for testing
+# Quick test: pulls only TEST_MAX_BILLS bills
+TESTING_MODE = False
+# Full test: pulls all bills for TEST_STATE and TEST_YEAR without bill count cap
+FULL_TESTING_MODE = False
+TEST_STATE = 'CA'
+TEST_YEAR = 2023
+TEST_MAX_BILLS = 3
+
+# Output files
+CACHE_FILE = "data/bill_cache.json"        # Stores bill_id -> change_hash
+OUTPUT_FILE = "data/known_bills.json"     # Final bills data
+
+# Query settings
+QUERY = "artificial intelligence"
+START_YEAR = 2023
+END_YEAR = datetime.now(timezone.utc).year
+
+# Include all state legislatures plus U.S. Congress (both chambers)
+STATES = [
+    "AL","AK","AZ","AR","CA","CO","CT","DE","FL","GA",
+    "HI","ID","IL","IN","IA","KS","KY","LA","ME","MD",
+    "MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ",
+    "NM","NY","NC","ND","OH","OK","OR","PA","RI","SC",
+    "SD","TN","TX","UT","VT","VA","WA","WV","WI","WY",
+    "US"  # U.S. Congress
+]
+
+# Rate limiting (seconds between requests)
+RATE_LIMIT = 0.2
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Logging configuration
+LOG_FILE = "data_updating_scripts/logs/fetch_ai_bills.log"
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(LOG_FILE)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Apply testing overrides
+if TESTING_MODE:
+    logger.info(f"*** TESTING MODE: fetching only {TEST_MAX_BILLS} bills from {TEST_STATE} ({TEST_YEAR}) ***")
+    STATES = [TEST_STATE]
+if FULL_TESTING_MODE:
+    logger.info(f"*** FULL TESTING MODE: fetching all bills from {TEST_STATE} ({TEST_YEAR}) ***")
+    STATES = [TEST_STATE]
+
+
+def load_json(path, default):
+    try:
+        with open(path, 'r') as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return default
+
+
+def save_json(path, data):
+    # Create directory if it doesn't exist
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, 'w') as f:
+        json.dump(data, f, indent=2)
+    logger.info(f"Saved JSON to {path}")
+
+
+def legi_request(op, params):
+    base = "https://api.legiscan.com/"
+    params.update({"key": API_KEY, "op": op})
+    try:
+        resp = requests.get(base, params=params, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("status") != "OK":
+            logger.error(f"API error {op}: {data.get('message', data)}")
+            return None
+        return data
+    except requests.RequestException as e:
+        logger.error(f"Request failed ({op}): {e}")
+        return None
+
+
+def extract_plain_text(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, "html.parser")
+    return soup.get_text(separator="\n", strip=True)
+
+
+def main():
+    cache = load_json(CACHE_FILE, {})
+    existing = load_json(OUTPUT_FILE, [])
+    existing_map = {b.get("bill_id"): b for b in existing}
+    logger.info(f"Loaded cache entries: {len(cache)}, existing bills: {len(existing)}")
+
+    collected = []
+    total_fetched = 0
+    years = [TEST_YEAR] if (TESTING_MODE or FULL_TESTING_MODE) else list(range(START_YEAR, END_YEAR + 1))
+
+    for state in STATES:
+        for year in years:
+            page = 1
+            while True:
+                if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                    logger.info("Reached TEST_MAX_BILLS limit, stopping early.")
+                    break
+                params = {"state": state, "year": year, "query": QUERY, "page": page}
+                logger.info(f"Searching {state} for {year}, page {page}")
+                data = legi_request("getSearch", params)
+                if not data:
+                    break
+
+                results = data.get("searchresult", {})
+                summary = results.get("summary", {})
+                bills = [v for k, v in results.items() if k != "summary"]
+                if not bills:
+                    logger.info(f"No bills on page {page} for {state} {year}")
+                    break
+
+                logger.info(f"Found {len(bills)} bills on {state} {year} page {page}")
+                for bill in bills:
+                    if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                        break
+                    bill_id = str(bill.get("bill_id"))
+                    state_code = bill.get("state")
+                    bill_num = bill.get("bill_number")
+                    logger.info(f"Processing bill {state_code}_{bill_num} (ID: {bill_id})")
+
+                    details_resp = legi_request("getBill", {"id": bill_id})
+                    if not details_resp:
+                        continue
+                    details = details_resp.get("bill", {})
+                    sess_year = details.get("session", {}).get("year_start", 0)
+                    if sess_year < START_YEAR:
+                        continue
+
+                    new_hash = details.get("change_hash")
+                    old_hash = cache.get(bill_id)
+                    now_iso = datetime.now(timezone.utc).isoformat()
+
+                    # Extract all relevant dates
+                    explicit = details.get("last_action_date")
+                    status_date = details.get("status_date")
+                    last_vote_date = details.get("last_vote_date")
+                    last_amendment_date = details.get("last_amendment_date")
+                    actions = details.get("actions", [])
+                    action_dates = [a.get("action_date") for a in actions if a.get("action_date")]
+                    most_recent_action = max(action_dates) if action_dates else None
+                    candidates = [d for d in [explicit, status_date, last_vote_date, last_amendment_date, most_recent_action] if d]
+                    last_action_date = max(candidates) if candidates else None
+
+                    bill_url = details.get("url")  # Bill detail page URL
+
+                    if new_hash and new_hash == old_hash and bill_id in existing_map:
+                        entry = existing_map[bill_id]
+                        entry.update({
+                            "status": details.get("status"),
+                            "session_year": f"{details.get('session', {}).get('year_start', '')}-{details.get('session', {}).get('year_end', '')}",
+                            "last_action_date": last_action_date,
+                            "status_date": status_date,
+                            "last_vote_date": last_vote_date,
+                            "last_amendment_date": last_amendment_date,
+                            "actions": actions,
+                            "bill_url": bill_url,
+                            "lastUpdatedAt": now_iso
+                        })
+                        logger.info(f"Reused cache; updated status={entry['status']}, last_action_date={entry['last_action_date']}")
+                    else:
+                        plain_text = None
+                        texts = details.get("texts", [])
+                        if texts:
+                            doc_id = texts[0].get("doc_id")
+                            text_resp = legi_request("getBillText", {"id": doc_id})
+                            if text_resp and "text" in text_resp:
+                                raw_b64 = text_resp["text"].get("doc", "")
+                                try:
+                                    decoded = base64.b64decode(raw_b64)
+                                    html = decoded.decode("utf-8", errors="ignore")
+                                    plain_text = extract_plain_text(html)
+                                except Exception as e:
+                                    logger.error(f"Failed decoding HTML for {bill_id}: {e}")
+
+                        entry = {
+                            "bill_id": bill_id,
+                            "state": state_code,
+                            "bill_number": bill_num,
+                            "session_year": f"{details.get('session', {}).get('year_start', '')}-{details.get('session', {}).get('year_end', '')}",
+                            "title": details.get("title"),
+                            "description": details.get("description"),
+                            "status": details.get("status"),
+                            "sponsors": [s.get("name") for s in details.get("sponsors", [])],
+                            "text": plain_text,
+                            "last_action_date": last_action_date,
+                            "status_date": status_date,
+                            "last_vote_date": last_vote_date,
+                            "last_amendment_date": last_amendment_date,
+                            "actions": actions,
+                            "bill_url": bill_url,
+                            "change_hash": new_hash,
+                            "lastUpdatedAt": now_iso
+                        }
+                        cache[bill_id] = new_hash
+                        logger.info(
+                            f"Entry data: title='{entry['title']}', sponsors={len(entry['sponsors'])}, "
+                            f"status={entry['status']}, last_action_date={entry['last_action_date']}"
+                        )
+
+                    collected.append(entry)
+                    total_fetched += 1
+                    time.sleep(RATE_LIMIT)
+
+                if page >= summary.get("page_total", 1):
+                    break
+                page += 1
+                time.sleep(RATE_LIMIT)
+            if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                break
+        if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+            break
+
+    dedup = {e["bill_id"]: e for e in collected}
+    all_bills = list(dedup.values())
+    save_json(OUTPUT_FILE, all_bills)
+    save_json(CACHE_FILE, cache)
+    logger.info(f"Completed run, saved {len(all_bills)} bills to {OUTPUT_FILE}")
+
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/get_data_ORIGINAL.py

@@ -0,0 +1,251 @@
+import os
+import sys
+import json
+import time
+import logging
+import base64
+from datetime import datetime, timezone
+import requests
+from dotenv import load_dotenv
+from bs4 import BeautifulSoup
+
+# Load environment variables from .env file
+load_dotenv()
+# Pull API key from environment
+API_KEY = os.getenv("LEGISCAN_API_KEY")  # Set your LegiScan API key in .env
+if not API_KEY:
+    print("Error: Please set LEGISCAN_API_KEY in your .env file.")
+    sys.exit(1)
+
+# Modes for testing
+# Quick test: pulls only TEST_MAX_BILLS bills
+TESTING_MODE = False
+# Full test: pulls all bills for TEST_STATE and TEST_YEAR without bill count cap
+FULL_TESTING_MODE = False
+TEST_STATE = 'CA'
+TEST_YEAR = 2023
+TEST_MAX_BILLS = 3
+
+# Output files
+CACHE_FILE = "data/bill_cache.json"        # Stores bill_id -> change_hash
+OUTPUT_FILE = "data/known_bills.json"     # Final bills data
+
+# Query settings
+QUERY = "artificial intelligence"
+START_YEAR = 2023
+END_YEAR = datetime.now(timezone.utc).year
+
+# Include all state legislatures plus U.S. Congress (both chambers)
+STATES = [
+    "AL","AK","AZ","AR","CA","CO","CT","DE","FL","GA",
+    "HI","ID","IL","IN","IA","KS","KY","LA","ME","MD",
+    "MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ",
+    "NM","NY","NC","ND","OH","OK","OR","PA","RI","SC",
+    "SD","TN","TX","UT","VT","VA","WA","WV","WI","WY",
+    "US"  # U.S. Congress
+]
+
+# Rate limiting (seconds between requests)
+RATE_LIMIT = 0.2
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Logging configuration
+LOG_FILE = "data_updating_scripts/logs/fetch_ai_bills.log"
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(LOG_FILE)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Apply testing overrides
+if TESTING_MODE:
+    logger.info(f"*** TESTING MODE: fetching only {TEST_MAX_BILLS} bills from {TEST_STATE} ({TEST_YEAR}) ***")
+    STATES = [TEST_STATE]
+if FULL_TESTING_MODE:
+    logger.info(f"*** FULL TESTING MODE: fetching all bills from {TEST_STATE} ({TEST_YEAR}) ***")
+    STATES = [TEST_STATE]
+
+
+def load_json(path, default):
+    try:
+        with open(path, 'r') as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return default
+
+
+def save_json(path, data):
+    # Create directory if it doesn't exist
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, 'w') as f:
+        json.dump(data, f, indent=2)
+    logger.info(f"Saved JSON to {path}")
+
+
+def legi_request(op, params):
+    base = "https://api.legiscan.com/"
+    params.update({"key": API_KEY, "op": op})
+    try:
+        resp = requests.get(base, params=params, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("status") != "OK":
+            logger.error(f"API error {op}: {data.get('message', data)}")
+            return None
+        return data
+    except requests.RequestException as e:
+        logger.error(f"Request failed ({op}): {e}")
+        return None
+
+
+def extract_plain_text(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, "html.parser")
+    return soup.get_text(separator="\n", strip=True)
+
+
+def main():
+    cache = load_json(CACHE_FILE, {})
+    existing = load_json(OUTPUT_FILE, [])
+    existing_map = {b.get("bill_id"): b for b in existing}
+    logger.info(f"Loaded cache entries: {len(cache)}, existing bills: {len(existing)}")
+
+    collected = []
+    total_fetched = 0
+    years = [TEST_YEAR] if (TESTING_MODE or FULL_TESTING_MODE) else list(range(START_YEAR, END_YEAR + 1))
+
+    for state in STATES:
+        for year in years:
+            page = 1
+            while True:
+                if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                    logger.info("Reached TEST_MAX_BILLS limit, stopping early.")
+                    break
+                params = {"state": state, "year": year, "query": QUERY, "page": page}
+                logger.info(f"Searching {state} for {year}, page {page}")
+                data = legi_request("getSearch", params)
+                if not data:
+                    break
+
+                results = data.get("searchresult", {})
+                summary = results.get("summary", {})
+                bills = [v for k, v in results.items() if k != "summary"]
+                if not bills:
+                    logger.info(f"No bills on page {page} for {state} {year}")
+                    break
+
+                logger.info(f"Found {len(bills)} bills on {state} {year} page {page}")
+                for bill in bills:
+                    if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                        break
+                    bill_id = str(bill.get("bill_id"))
+                    state_code = bill.get("state")
+                    bill_num = bill.get("bill_number")
+                    logger.info(f"Processing bill {state_code}_{bill_num} (ID: {bill_id})")
+
+                    details_resp = legi_request("getBill", {"id": bill_id})
+                    if not details_resp:
+                        continue
+                    details = details_resp.get("bill", {})
+                    sess_year = details.get("session", {}).get("year_start", 0)
+                    if sess_year < START_YEAR:
+                        continue
+
+                    new_hash = details.get("change_hash")
+                    old_hash = cache.get(bill_id)
+                    now_iso = datetime.now(timezone.utc).isoformat()
+
+                    # Extract all relevant dates
+                    explicit = details.get("last_action_date")
+                    status_date = details.get("status_date")
+                    last_vote_date = details.get("last_vote_date")
+                    last_amendment_date = details.get("last_amendment_date")
+                    actions = details.get("actions", [])
+                    action_dates = [a.get("action_date") for a in actions if a.get("action_date")]
+                    most_recent_action = max(action_dates) if action_dates else None
+                    candidates = [d for d in [explicit, status_date, last_vote_date, last_amendment_date, most_recent_action] if d]
+                    last_action_date = max(candidates) if candidates else None
+
+                    bill_url = details.get("url")  # Bill detail page URL
+
+                    if new_hash and new_hash == old_hash and bill_id in existing_map:
+                        entry = existing_map[bill_id]
+                        entry.update({
+                            "status": details.get("status"),
+                            "session_year": f"{details.get('session', {}).get('year_start', '')}-{details.get('session', {}).get('year_end', '')}",
+                            "last_action_date": last_action_date,
+                            "status_date": status_date,
+                            "last_vote_date": last_vote_date,
+                            "last_amendment_date": last_amendment_date,
+                            "actions": actions,
+                            "bill_url": bill_url,
+                            "lastUpdatedAt": now_iso
+                        })
+                        logger.info(f"Reused cache; updated status={entry['status']}, last_action_date={entry['last_action_date']}")
+                    else:
+                        plain_text = None
+                        texts = details.get("texts", [])
+                        if texts:
+                            doc_id = texts[0].get("doc_id")
+                            text_resp = legi_request("getBillText", {"id": doc_id})
+                            if text_resp and "text" in text_resp:
+                                raw_b64 = text_resp["text"].get("doc", "")
+                                try:
+                                    decoded = base64.b64decode(raw_b64)
+                                    html = decoded.decode("utf-8", errors="ignore")
+                                    plain_text = extract_plain_text(html)
+                                except Exception as e:
+                                    logger.error(f"Failed decoding HTML for {bill_id}: {e}")
+
+                        entry = {
+                            "bill_id": bill_id,
+                            "state": state_code,
+                            "bill_number": bill_num,
+                            "session_year": f"{details.get('session', {}).get('year_start', '')}-{details.get('session', {}).get('year_end', '')}",
+                            "title": details.get("title"),
+                            "description": details.get("description"),
+                            "status": details.get("status"),
+                            "sponsors": [s.get("name") for s in details.get("sponsors", [])],
+                            "text": plain_text,
+                            "last_action_date": last_action_date,
+                            "status_date": status_date,
+                            "last_vote_date": last_vote_date,
+                            "last_amendment_date": last_amendment_date,
+                            "actions": actions,
+                            "bill_url": bill_url,
+                            "change_hash": new_hash,
+                            "lastUpdatedAt": now_iso
+                        }
+                        cache[bill_id] = new_hash
+                        logger.info(
+                            f"Entry data: title='{entry['title']}', sponsors={len(entry['sponsors'])}, "
+                            f"status={entry['status']}, last_action_date={entry['last_action_date']}"
+                        )
+
+                    collected.append(entry)
+                    total_fetched += 1
+                    time.sleep(RATE_LIMIT)
+
+                if page >= summary.get("page_total", 1):
+                    break
+                page += 1
+                time.sleep(RATE_LIMIT)
+            if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+                break
+        if TESTING_MODE and total_fetched >= TEST_MAX_BILLS:
+            break
+
+    dedup = {e["bill_id"]: e for e in collected}
+    all_bills = list(dedup.values())
+    save_json(OUTPUT_FILE, all_bills)
+    save_json(CACHE_FILE, cache)
+    logger.info(f"Completed run, saved {len(all_bills)} bills to {OUTPUT_FILE}")
+
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/known_bills_status.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+known_bills_status.py
+
+Reads known_bills_fixed.json and updates existing known_bills_visualize.json.
+Merges new bills and updates existing ones while preserving clean status fields.
+"""
+import json
+from pathlib import Path
+from datetime import datetime, timezone
+
+def map_status(original_status):
+    """Map LegiScan status codes to clean display text."""
+    
+    # Direct mapping for numeric codes
+    status_mapping = {
+        "0": "Inactive",           # Pre-filed
+        "1": "Active",             # Introduced  
+        "2": "Active",             # Engrossed
+        "3": "Active",             # Enrolled
+        "4": "Signed Into Law",    # Passed
+        "5": "Vetoed",             # Vetoed
+        "6": "Inactive",           # Failed
+        "7": "Signed Into Law",    # Override
+        "8": "Signed Into Law",    # Chaptered
+        "9": "Active",             # Refer
+        "10": "Active",            # Report Pass
+        "11": "Inactive",          # Report DNP
+        "12": "Active",            # Draft
+        
+        # Integer versions
+        0: "Inactive", 1: "Active", 2: "Active", 3: "Active",
+        4: "Signed Into Law", 5: "Vetoed", 6: "Inactive",
+        7: "Signed Into Law", 8: "Signed Into Law", 9: "Active",
+        10: "Active", 11: "Inactive", 12: "Active"
+    }
+    
+    # Try direct mapping first
+    if original_status in status_mapping:
+        return status_mapping[original_status]
+    
+    # Handle text statuses
+    if original_status:
+        status_str = str(original_status).lower()
+        if "pass" in status_str or "signed" in status_str or "enacted" in status_str:
+            return "Signed Into Law"
+        elif "veto" in status_str:
+            return "Vetoed"
+        elif "fail" in status_str or "dead" in status_str or "killed" in status_str:
+            return "Inactive"
+        elif "active" in status_str or "intro" in status_str or "pending" in status_str:
+            return "Active"
+    
+    # Default fallback
+    return "Inactive"
+
+def create_bill_key(bill):
+    """Create a unique key for each bill."""
+    return f"{bill.get('state', 'Unknown')}_{bill.get('bill_number', 'Unknown')}"
+
+def merge_bill_data(new_bill, existing_bill=None):
+    """Merge new bill data with existing bill, preserving processed fields."""
+    if not existing_bill:
+        # New bill - create clean version
+        merged_bill = new_bill.copy()
+        original_status = merged_bill.get('status')
+        merged_bill['original_status'] = original_status
+        merged_bill['status'] = map_status(original_status)
+        merged_bill['status_updated_at'] = datetime.now(timezone.utc).isoformat()
+        return merged_bill
+    
+    # Existing bill - merge carefully
+    merged_bill = existing_bill.copy()
+    
+    # Update with new data from source (except status fields)
+    for key, value in new_bill.items():
+        if key not in ['status', 'original_status', 'status_updated_at']:
+            merged_bill[key] = value
+    
+    # Check if original status actually changed
+    new_original_status = new_bill.get('status')
+    old_original_status = existing_bill.get('original_status')
+    
+    # Convert both to strings for comparison to handle int vs string
+    new_status_str = str(new_original_status) if new_original_status is not None else None
+    old_status_str = str(old_original_status) if old_original_status is not None else None
+    
+    if new_status_str != old_status_str:
+        # Real change in underlying data
+        new_clean_status = map_status(new_original_status)
+        merged_bill['original_status'] = new_original_status
+        merged_bill['status'] = new_clean_status
+        merged_bill['status_updated_at'] = datetime.now(timezone.utc).isoformat()
+        return merged_bill
+    
+    # No change - keep existing clean status but ensure it's properly mapped
+    if 'status' not in merged_bill or merged_bill['status'] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
+        # Only remap if status is still numeric (needs cleaning)
+        merged_bill['status'] = map_status(old_original_status)
+    
+    return merged_bill
+
+def main():
+    
+    # File paths
+    input_file = Path("data/known_bills_fixed.json")
+    output_file = Path("data/known_bills_visualize.json")
+    
+    print(f"Reading source bills from: {input_file}")
+    
+    # Load source bills data
+    with open(input_file, 'r', encoding='utf-8') as f:
+        source_bills = json.load(f)
+    
+    print(f"Loaded {len(source_bills)} bills from source")
+    
+    # Load existing visualization data if it exists
+    existing_bills = []
+    if output_file.exists():
+        print(f"Reading existing visualization data from: {output_file}")
+        with open(output_file, 'r', encoding='utf-8') as f:
+            existing_bills = json.load(f)
+        print(f"Loaded {len(existing_bills)} existing bills")
+    else:
+        print("No existing visualization data found - will create new file")
+    
+    # Create lookup dictionary for existing bills
+    existing_bills_dict = {}
+    for bill in existing_bills:
+        key = create_bill_key(bill)
+        existing_bills_dict[key] = bill
+    
+    # Process and merge bills
+    merged_bills = []
+    new_bills_count = 0
+    updated_bills_count = 0
+    unchanged_bills_count = 0
+    
+    print(f"\nProcessing {len(source_bills)} bills...")
+    
+    for source_bill in source_bills:
+        bill_key = create_bill_key(source_bill)
+        existing_bill = existing_bills_dict.get(bill_key)
+        
+        if existing_bill:
+            # Check if anything actually changed
+            old_original_status = existing_bill.get('original_status')
+            new_original_status = source_bill.get('status')
+            
+            if old_original_status != new_original_status:
+                updated_bills_count += 1
+            else:
+                unchanged_bills_count += 1
+        else:
+            new_bills_count += 1
+        
+        merged_bill = merge_bill_data(source_bill, existing_bill)
+        merged_bills.append(merged_bill)
+    
+    # Check for bills that exist in visualization but not in source (removed bills)
+    source_keys = {create_bill_key(bill) for bill in source_bills}
+    existing_keys = set(existing_bills_dict.keys())
+    removed_keys = existing_keys - source_keys
+    
+    # Save updated bills
+    print(f"\nSaving updated bills to: {output_file}")
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(merged_bills, f, indent=2, ensure_ascii=False)
+    
+    # Show status distribution
+    status_counts = {}
+    for bill in merged_bills:
+        status = bill['status']
+        status_counts[status] = status_counts.get(status, 0) + 1
+    
+    # Summary
+    print(f"\n✅ Update complete!")
+    print(f"  📊 Total bills: {len(merged_bills)}")
+    if new_bills_count > 0:
+        print(f"  🆕 New bills: {new_bills_count}")
+    if updated_bills_count > 0:
+        print(f"  🔄 Updated bills: {updated_bills_count}")
+    if unchanged_bills_count > 0:
+        print(f"  ✅ Unchanged bills: {unchanged_bills_count}")
+    if removed_keys:
+        print(f"  🗑️  Removed bills: {len(removed_keys)}")
+    
+    if new_bills_count == 0 and updated_bills_count == 0:
+        print(f"  🎉 All bills are up to date - no changes needed!")
+    
+    print(f"\n📈 Status distribution:")
+    for status, count in sorted(status_counts.items()):
+        print(f"  {status}: {count}")
+    
+    print(f"\n📁 Clean data saved to: {output_file}")
+    print("Now run: streamlit run scripts/visualize-MIT.py")
+
+if __name__ == "__main__":
+    main()

data/data_updating_scripts/logs/eu_vectorstore.log

@@ -0,0 +1,128 @@
+2025-11-03 11:40:25,451 [INFO] Starting EU AI Act vectorstore creation...
+2025-11-03 11:40:25,451 [INFO] Extracting text from PDF...
+2025-11-03 11:40:25,480 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-11-03 11:40:27,260 [INFO] Extracted 612396 characters from PDF
+2025-11-03 11:40:27,260 [INFO] Creating document chunks...
+2025-11-03 11:40:27,268 [INFO] Created 648 document chunks
+2025-11-03 11:40:27,268 [INFO] Initializing embeddings...
+2025-11-03 11:40:27,397 [INFO] Creating FAISS vectorstore...
+2025-11-03 11:40:31,088 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-11-03 11:40:31,414 [INFO] Loading faiss.
+2025-11-03 11:40:31,881 [INFO] Successfully loaded faiss.
+2025-11-03 11:40:31,936 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-11-03 11:40:31,945 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-11-03 11:40:31,945 [INFO]    - Total chunks: 648
+2025-11-03 11:40:31,945 [INFO]    - Text length: 612,396 characters
+2025-11-03 11:40:31,945 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-11-03 12:24:44,470 [INFO] Starting EU AI Act vectorstore creation...
+2025-11-03 12:24:44,471 [INFO] Extracting text from PDF...
+2025-11-03 12:24:44,492 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-11-03 12:24:46,209 [INFO] Extracted 612396 characters from PDF
+2025-11-03 12:24:46,209 [INFO] Creating document chunks...
+2025-11-03 12:24:46,217 [INFO] Created 648 document chunks
+2025-11-03 12:24:46,217 [INFO] Initializing embeddings...
+2025-11-03 12:24:46,357 [INFO] Creating FAISS vectorstore...
+2025-11-03 12:24:49,286 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-11-03 12:24:49,669 [INFO] Loading faiss.
+2025-11-03 12:24:49,700 [INFO] Successfully loaded faiss.
+2025-11-03 12:24:49,749 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-11-03 12:24:49,754 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-11-03 12:24:49,754 [INFO]    - Total chunks: 648
+2025-11-03 12:24:49,754 [INFO]    - Text length: 612,396 characters
+2025-11-03 12:24:49,754 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-11-04 15:55:15,879 [INFO] Starting EU AI Act vectorstore creation...
+2025-11-04 15:55:15,879 [INFO] Extracting text from PDF...
+2025-11-04 15:55:15,899 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-11-04 15:55:17,629 [INFO] Extracted 612396 characters from PDF
+2025-11-04 15:55:17,629 [INFO] Creating document chunks...
+2025-11-04 15:55:17,637 [INFO] Created 648 document chunks
+2025-11-04 15:55:17,637 [INFO] Initializing embeddings...
+2025-11-04 15:55:17,768 [INFO] Creating FAISS vectorstore...
+2025-11-04 15:55:21,406 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-11-04 15:55:21,846 [INFO] Loading faiss.
+2025-11-04 15:55:21,917 [INFO] Successfully loaded faiss.
+2025-11-04 15:55:21,968 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-11-04 15:55:21,981 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-11-04 15:55:21,981 [INFO]    - Total chunks: 648
+2025-11-04 15:55:21,981 [INFO]    - Text length: 612,396 characters
+2025-11-04 15:55:21,981 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-11-14 15:36:40,441 [INFO] Starting EU AI Act vectorstore creation...
+2025-11-14 15:36:40,442 [INFO] Extracting text from PDF...
+2025-11-14 15:36:40,455 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-11-14 15:36:41,830 [INFO] Extracted 612396 characters from PDF
+2025-11-14 15:36:41,830 [INFO] Creating document chunks...
+2025-11-14 15:36:41,837 [INFO] Created 648 document chunks
+2025-11-14 15:36:41,837 [INFO] Initializing embeddings...
+2025-11-14 15:36:41,983 [INFO] Creating FAISS vectorstore...
+2025-11-14 15:36:46,413 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-11-14 15:36:46,791 [INFO] Loading faiss.
+2025-11-14 15:36:47,362 [INFO] Successfully loaded faiss.
+2025-11-14 15:36:47,404 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-11-14 15:36:47,410 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-11-14 15:36:47,410 [INFO]    - Total chunks: 648
+2025-11-14 15:36:47,410 [INFO]    - Text length: 612,396 characters
+2025-11-14 15:36:47,410 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-11-20 14:15:10,012 [INFO] Starting EU AI Act vectorstore creation...
+2025-11-20 14:15:10,013 [INFO] Extracting text from PDF...
+2025-11-20 14:15:10,029 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-11-20 14:15:11,997 [INFO] Extracted 612396 characters from PDF
+2025-11-20 14:15:11,998 [INFO] Creating document chunks...
+2025-11-20 14:15:12,006 [INFO] Created 648 document chunks
+2025-11-20 14:15:12,006 [INFO] Initializing embeddings...
+2025-11-20 14:15:12,200 [INFO] Creating FAISS vectorstore...
+2025-11-20 14:15:16,058 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-11-20 14:15:16,386 [INFO] Loading faiss.
+2025-11-20 14:15:16,477 [INFO] Successfully loaded faiss.
+2025-11-20 14:15:16,521 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-11-20 14:15:16,529 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-11-20 14:15:16,529 [INFO]    - Total chunks: 648
+2025-11-20 14:15:16,529 [INFO]    - Text length: 612,396 characters
+2025-11-20 14:15:16,529 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-12-01 12:38:49,653 [INFO] Starting EU AI Act vectorstore creation...
+2025-12-01 12:38:49,653 [INFO] Extracting text from PDF...
+2025-12-01 12:38:49,669 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-12-01 12:38:51,518 [INFO] Extracted 612396 characters from PDF
+2025-12-01 12:38:51,518 [INFO] Creating document chunks...
+2025-12-01 12:38:51,526 [INFO] Created 648 document chunks
+2025-12-01 12:38:51,526 [INFO] Initializing embeddings...
+2025-12-01 12:38:51,709 [INFO] Creating FAISS vectorstore...
+2025-12-01 12:38:54,252 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-12-01 12:38:54,675 [INFO] Loading faiss.
+2025-12-01 12:38:54,817 [INFO] Successfully loaded faiss.
+2025-12-01 12:38:54,859 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-12-01 12:38:54,865 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-12-01 12:38:54,866 [INFO]    - Total chunks: 648
+2025-12-01 12:38:54,866 [INFO]    - Text length: 612,396 characters
+2025-12-01 12:38:54,866 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-12-01 13:21:15,236 [INFO] Starting EU AI Act vectorstore creation...
+2025-12-01 13:21:15,237 [INFO] Extracting text from PDF...
+2025-12-01 13:21:15,253 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-12-01 13:21:17,069 [INFO] Extracted 612396 characters from PDF
+2025-12-01 13:21:17,069 [INFO] Creating document chunks...
+2025-12-01 13:21:17,078 [INFO] Created 648 document chunks
+2025-12-01 13:21:17,078 [INFO] Initializing embeddings...
+2025-12-01 13:21:17,343 [INFO] Creating FAISS vectorstore...
+2025-12-01 13:21:20,254 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-12-01 13:21:20,654 [INFO] Loading faiss.
+2025-12-01 13:21:20,768 [INFO] Successfully loaded faiss.
+2025-12-01 13:21:20,815 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-12-01 13:21:20,821 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-12-01 13:21:20,821 [INFO]    - Total chunks: 648
+2025-12-01 13:21:20,821 [INFO]    - Text length: 612,396 characters
+2025-12-01 13:21:20,822 [INFO]    - Saved to: data/eu_ai_act_vectorstore
+2025-12-03 11:09:39,059 [INFO] Starting EU AI Act vectorstore creation...
+2025-12-03 11:09:39,060 [INFO] Extracting text from PDF...
+2025-12-03 11:09:39,075 [INFO] Processing 144 pages from data_updating_scripts/eu-ai-act.pdf
+2025-12-03 11:09:40,933 [INFO] Extracted 612396 characters from PDF
+2025-12-03 11:09:40,934 [INFO] Creating document chunks...
+2025-12-03 11:09:40,942 [INFO] Created 648 document chunks
+2025-12-03 11:09:40,942 [INFO] Initializing embeddings...
+2025-12-03 11:09:41,136 [INFO] Creating FAISS vectorstore...
+2025-12-03 11:09:44,436 [INFO] HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
+2025-12-03 11:09:44,820 [INFO] Loading faiss.
+2025-12-03 11:09:44,925 [INFO] Successfully loaded faiss.
+2025-12-03 11:09:44,968 [INFO] Saving vectorstore to data/eu_ai_act_vectorstore...
+2025-12-03 11:09:44,974 [INFO] ✅ EU AI Act vectorstore created successfully!
+2025-12-03 11:09:44,974 [INFO]    - Total chunks: 648
+2025-12-03 11:09:44,974 [INFO]    - Text length: 612,396 characters
+2025-12-03 11:09:44,974 [INFO]    - Saved to: data/eu_ai_act_vectorstore

data/data_updating_scripts/logs/mark_no_text_bills.log

@@ -0,0 +1,293 @@
+2025-11-03 11:36:37 [INFO] Starting no-text bill marking process
+2025-11-03 11:36:37 [INFO] Loaded 1 bills from data/known_bills_visualize.json
+2025-11-03 11:36:37 [INFO] Processing 1 bills to mark no-text bills
+2025-11-03 11:36:37 [INFO] Saved 1 bills to data/known_bills_visualize.json
+2025-11-03 11:36:37 [INFO] Processing complete!
+2025-11-03 11:36:37 [INFO] Total bills processed: 1
+2025-11-03 11:36:37 [INFO] Bills without text: 0
+2025-11-03 11:36:37 [INFO] Already had None categories: 0
+2025-11-03 11:36:37 [INFO] Newly marked as None: 0
+2025-11-03 11:36:37 [INFO] No-text bill marking process completed
+2025-11-03 11:40:21 [INFO] Starting no-text bill marking process
+2025-11-03 11:40:21 [INFO] Loaded 1 bills from data/known_bills_visualize.json
+2025-11-03 11:40:21 [INFO] Processing 1 bills to mark no-text bills
+2025-11-03 11:40:21 [INFO] Saved 1 bills to data/known_bills_visualize.json
+2025-11-03 11:40:21 [INFO] Processing complete!
+2025-11-03 11:40:21 [INFO] Total bills processed: 1
+2025-11-03 11:40:21 [INFO] Bills without text: 0
+2025-11-03 11:40:21 [INFO] Already had None categories: 0
+2025-11-03 11:40:21 [INFO] Newly marked as None: 0
+2025-11-03 11:40:21 [INFO] No-text bill marking process completed
+2025-11-03 12:24:40 [INFO] Starting no-text bill marking process
+2025-11-03 12:24:40 [INFO] Loaded 1 bills from data/known_bills_visualize.json
+2025-11-03 12:24:40 [INFO] Processing 1 bills to mark no-text bills
+2025-11-03 12:24:40 [INFO] Saved 1 bills to data/known_bills_visualize.json
+2025-11-03 12:24:40 [INFO] Processing complete!
+2025-11-03 12:24:40 [INFO] Total bills processed: 1
+2025-11-03 12:24:40 [INFO] Bills without text: 0
+2025-11-03 12:24:40 [INFO] Already had None categories: 0
+2025-11-03 12:24:40 [INFO] Newly marked as None: 0
+2025-11-03 12:24:40 [INFO] No-text bill marking process completed
+2025-11-04 15:55:11 [INFO] Starting no-text bill marking process
+2025-11-04 15:55:11 [INFO] Loaded 10 bills from data/known_bills_visualize.json
+2025-11-04 15:55:11 [INFO] Processing 10 bills to mark no-text bills
+2025-11-04 15:55:11 [INFO] Saved 10 bills to data/known_bills_visualize.json
+2025-11-04 15:55:11 [INFO] Processing complete!
+2025-11-04 15:55:11 [INFO] Total bills processed: 10
+2025-11-04 15:55:11 [INFO] Bills without text: 0
+2025-11-04 15:55:11 [INFO] Already had None categories: 0
+2025-11-04 15:55:11 [INFO] Newly marked as None: 0
+2025-11-04 15:55:11 [INFO] No-text bill marking process completed
+2025-11-14 15:31:16 [INFO] Starting no-text bill marking process
+2025-11-14 15:31:16 [INFO] Loaded 2564 bills from data/known_bills_visualize.json
+2025-11-14 15:31:16 [INFO] Processing 2564 bills to mark no-text bills
+2025-11-14 15:31:16 [INFO] Progress: 100/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 200/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 300/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 400/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 500/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 600/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 700/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 800/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 900/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1000/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1100/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1200/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1300/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1400/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1500/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1600/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1700/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1800/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 1900/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2000/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2100/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2200/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2300/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2400/2564 processed
+2025-11-14 15:31:16 [INFO] Progress: 2500/2564 processed
+2025-11-14 15:31:17 [INFO] Saved 2564 bills to data/known_bills_visualize.json
+2025-11-14 15:31:17 [INFO] Processing complete!
+2025-11-14 15:31:17 [INFO] Total bills processed: 2564
+2025-11-14 15:31:17 [INFO] Bills without text: 9
+2025-11-14 15:31:17 [INFO] Already had None categories: 9
+2025-11-14 15:31:17 [INFO] Newly marked as None: 0
+2025-11-14 15:31:17 [INFO] No-text bill marking process completed
+2025-11-17 21:13:12 [INFO] Starting no-text bill marking process
+2025-11-17 21:13:13 [INFO] Loaded 2564 bills from data/known_bills_visualize.json
+2025-11-17 21:13:13 [INFO] Processing 2564 bills to mark no-text bills
+2025-11-17 21:13:13 [INFO] Progress: 100/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 200/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 300/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 400/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 500/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 600/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 700/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 800/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 900/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1000/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1100/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1200/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1300/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1400/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1500/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1600/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1700/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1800/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 1900/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2000/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2100/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2200/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2300/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2400/2564 processed
+2025-11-17 21:13:13 [INFO] Progress: 2500/2564 processed
+2025-11-17 21:13:14 [INFO] Saved 2564 bills to data/known_bills_visualize.json
+2025-11-17 21:13:14 [INFO] Processing complete!
+2025-11-17 21:13:14 [INFO] Total bills processed: 2564
+2025-11-17 21:13:14 [INFO] Bills without text: 9
+2025-11-17 21:13:14 [INFO] Already had None categories: 9
+2025-11-17 21:13:14 [INFO] Newly marked as None: 0
+2025-11-17 21:13:14 [INFO] No-text bill marking process completed
+2025-11-20 13:52:45 [INFO] Starting no-text bill marking process
+2025-11-20 13:52:46 [INFO] Loaded 2596 bills from data/known_bills_visualize.json
+2025-11-20 13:52:46 [INFO] Processing 2596 bills to mark no-text bills
+2025-11-20 13:52:46 [INFO] Progress: 100/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 200/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 300/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 400/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 500/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 600/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 700/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 800/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 900/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1000/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1100/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1200/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1300/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1400/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1500/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1600/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1700/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1800/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 1900/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2000/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2100/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2200/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2300/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2400/2596 processed
+2025-11-20 13:52:46 [INFO] Progress: 2500/2596 processed
+2025-11-20 13:52:47 [INFO] Saved 2596 bills to data/known_bills_visualize.json
+2025-11-20 13:52:47 [INFO] Processing complete!
+2025-11-20 13:52:47 [INFO] Total bills processed: 2596
+2025-11-20 13:52:47 [INFO] Bills without text: 13
+2025-11-20 13:52:47 [INFO] Already had None categories: 13
+2025-11-20 13:52:47 [INFO] Newly marked as None: 0
+2025-11-20 13:52:47 [INFO] No-text bill marking process completed
+2025-12-01 12:30:17 [INFO] Starting no-text bill marking process
+2025-12-01 12:30:17 [INFO] Loaded 2605 bills from data/known_bills_visualize.json
+2025-12-01 12:30:17 [INFO] Processing 2605 bills to mark no-text bills
+2025-12-01 12:30:17 [INFO] Progress: 100/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 200/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 300/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 400/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 500/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 600/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 700/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 800/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 900/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1000/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1100/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1200/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1300/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1400/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1500/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1600/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1700/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1800/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 1900/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2000/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2100/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2200/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2300/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2400/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2500/2605 processed
+2025-12-01 12:30:17 [INFO] Progress: 2600/2605 processed
+2025-12-01 12:30:19 [INFO] Saved 2605 bills to data/known_bills_visualize.json
+2025-12-01 12:30:19 [INFO] Processing complete!
+2025-12-01 12:30:19 [INFO] Total bills processed: 2605
+2025-12-01 12:30:19 [INFO] Bills without text: 16
+2025-12-01 12:30:19 [INFO] Already had None categories: 16
+2025-12-01 12:30:19 [INFO] Newly marked as None: 0
+2025-12-01 12:30:19 [INFO] No-text bill marking process completed
+2025-12-01 13:11:46 [INFO] Starting no-text bill marking process
+2025-12-01 13:11:47 [INFO] Loaded 2605 bills from data/known_bills_visualize.json
+2025-12-01 13:11:47 [INFO] Processing 2605 bills to mark no-text bills
+2025-12-01 13:11:47 [INFO] Progress: 100/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 200/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 300/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 400/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 500/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 600/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 700/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 800/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 900/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1000/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1100/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1200/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1300/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1400/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1500/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1600/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1700/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1800/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 1900/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2000/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2100/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2200/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2300/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2400/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2500/2605 processed
+2025-12-01 13:11:47 [INFO] Progress: 2600/2605 processed
+2025-12-01 13:11:48 [INFO] Saved 2605 bills to data/known_bills_visualize.json
+2025-12-01 13:11:48 [INFO] Processing complete!
+2025-12-01 13:11:48 [INFO] Total bills processed: 2605
+2025-12-01 13:11:48 [INFO] Bills without text: 16
+2025-12-01 13:11:48 [INFO] Already had None categories: 16
+2025-12-01 13:11:48 [INFO] Newly marked as None: 0
+2025-12-01 13:11:48 [INFO] No-text bill marking process completed
+2025-12-01 13:16:12 [INFO] Starting no-text bill marking process
+2025-12-01 13:16:13 [ERROR] Error loading bills: Expecting ',' delimiter: line 70396 column 683331 (char 189968803)
+2025-12-01 13:16:13 [ERROR] No bills loaded. Exiting.
+2025-12-01 13:16:13 [INFO] No-text bill marking process completed
+2025-12-01 13:16:16 [INFO] Starting no-text bill marking process
+2025-12-01 13:16:17 [INFO] Loaded 2605 bills from data/known_bills_visualize.json
+2025-12-01 13:16:17 [INFO] Processing 2605 bills to mark no-text bills
+2025-12-01 13:16:17 [INFO] Progress: 100/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 200/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 300/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 400/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 500/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 600/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 700/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 800/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 900/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1000/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1100/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1200/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1300/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1400/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1500/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1600/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1700/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1800/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 1900/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2000/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2100/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2200/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2300/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2400/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2500/2605 processed
+2025-12-01 13:16:17 [INFO] Progress: 2600/2605 processed
+2025-12-01 13:16:18 [INFO] Saved 2605 bills to data/known_bills_visualize.json
+2025-12-01 13:16:18 [INFO] Processing complete!
+2025-12-01 13:16:18 [INFO] Total bills processed: 2605
+2025-12-01 13:16:18 [INFO] Bills without text: 16
+2025-12-01 13:16:18 [INFO] Already had None categories: 16
+2025-12-01 13:16:18 [INFO] Newly marked as None: 0
+2025-12-01 13:16:18 [INFO] No-text bill marking process completed
+2025-12-03 11:02:34 [INFO] Starting no-text bill marking process
+2025-12-03 11:02:35 [INFO] Loaded 2608 bills from data/known_bills_visualize.json
+2025-12-03 11:02:35 [INFO] Processing 2608 bills to mark no-text bills
+2025-12-03 11:02:35 [INFO] Progress: 100/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 200/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 300/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 400/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 500/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 600/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 700/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 800/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 900/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1000/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1100/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1200/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1300/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1400/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1500/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1600/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1700/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1800/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 1900/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2000/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2100/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2200/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2300/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2400/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2500/2608 processed
+2025-12-03 11:02:35 [INFO] Progress: 2600/2608 processed
+2025-12-03 11:02:36 [INFO] Saved 2608 bills to data/known_bills_visualize.json
+2025-12-03 11:02:36 [INFO] Processing complete!
+2025-12-03 11:02:36 [INFO] Total bills processed: 2608
+2025-12-03 11:02:36 [INFO] Bills without text: 16
+2025-12-03 11:02:36 [INFO] Already had None categories: 16
+2025-12-03 11:02:36 [INFO] Newly marked as None: 0
+2025-12-03 11:02:36 [INFO] No-text bill marking process completed

data/data_updating_scripts/mark_no_text_bills.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Script to mark bills without text as having None IAPP categories.
+
+This script reads known_bills_visualize.json, identifies bills without text,
+and sets their IAPP categories to None. The file is modified in-place.
+"""
+
+import json
+import os
+import logging
+from pathlib import Path
+from typing import Dict, List
+import sys
+
+# Add the project root to the path
+sys.path.append(str(Path(__file__).parent.parent))
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/mark_no_text_bills.log")]
+)
+logger = logging.getLogger(__name__)
+
+
+class NoTextBillMarker:
+    """Class to mark bills without text as having None IAPP categories."""
+    
+    def __init__(self):
+        self.visualize_file = "data/known_bills_visualize.json"
+    
+    def load_bills(self) -> List[Dict]:
+        """Load bills from known_bills_visualize.json."""
+        try:
+            with open(self.visualize_file, 'r', encoding='utf-8') as f:
+                bills = json.load(f)
+            logger.info(f"Loaded {len(bills)} bills from {self.visualize_file}")
+            return bills
+        except Exception as e:
+            logger.error(f"Error loading bills: {e}")
+            return []
+    
+    def save_bills(self, bills: List[Dict]) -> None:
+        """Save bills back to known_bills_visualize.json."""
+        try:
+            with open(self.visualize_file, 'w', encoding='utf-8') as f:
+                json.dump(bills, f, indent=2, ensure_ascii=False)
+            logger.info(f"Saved {len(bills)} bills to {self.visualize_file}")
+        except Exception as e:
+            logger.error(f"Error saving bills: {e}")
+    
+    def has_text(self, bill: Dict) -> bool:
+        text = bill.get('text')
+        return isinstance(text, str) and len(text.strip()) > 50
+    
+    def mark_no_text_bills(self) -> None:
+        """Mark bills without text as having None IAPP categories."""
+        # Load bills
+        bills = self.load_bills()
+        if not bills:
+            logger.error("No bills loaded. Exiting.")
+            return
+        
+        # Track progress
+        total_bills = len(bills)
+        no_text_count = 0
+        already_none_count = 0
+        
+        logger.info(f"Processing {total_bills} bills to mark no-text bills")
+        
+        for i, bill in enumerate(bills, 1):
+            bill_key = f"{bill.get('state', 'Unknown')}_{bill.get('bill_number', 'Unknown')}"
+            
+            # Check if bill has text
+            if not self.has_text(bill):
+                no_text_count += 1
+                
+                # Check if IAPP categories are already None
+                current_iapp = bill.get('iapp_categories')
+                if current_iapp is None:
+                    already_none_count += 1
+                    logger.debug(f"Bill {bill_key} already has None IAPP categories")
+                else:
+                    # Set IAPP categories to None
+                    bill['iapp_categories'] = None
+                    logger.info(f"Marked bill {bill_key} as having None IAPP categories (no text)")
+            
+            # Log progress every 100 bills
+            if i % 100 == 0:
+                logger.info(f"Progress: {i}/{total_bills} processed")
+        
+        # Save the modified bills
+        self.save_bills(bills)
+        
+        # Summary
+        logger.info(f"Processing complete!")
+        logger.info(f"Total bills processed: {total_bills}")
+        logger.info(f"Bills without text: {no_text_count}")
+        logger.info(f"Already had None categories: {already_none_count}")
+        logger.info(f"Newly marked as None: {no_text_count - already_none_count}")
+
+
+def main():
+    """Main function to run the no-text bill marker."""
+    logger.info("Starting no-text bill marking process")
+    
+    marker = NoTextBillMarker()
+    marker.mark_no_text_bills()
+    
+    logger.info("No-text bill marking process completed")
+
+
+if __name__ == "__main__":
+    main() 

data/data_updating_scripts/migrate_iapp_categories.py

@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+"""
+Script to migrate IAPP categories for bills with missing or invalid subcategories.
+
+This script reads bills from known_bills_fixed.json, analyzes bills with missing IAPP categories
+using OpenAI API, and saves the results to known_bills_visualize.json.
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+import sys
+import os
+import re
+import hashlib
+import argparse
+
+# Add the project root to the path
+sys.path.append(str(Path(__file__).parent.parent))
+
+from config import ConfigManager
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.documents import Document
+
+# Paths
+INPUT_FILE = Path("data/known_bills_fixed.json")
+VIS_FILE = Path("data/known_bills_visualize.json")
+CACHE_FILE = Path("data/iapp_categories_cache.json")
+
+# Create logs directory if it doesn't exist
+os.makedirs("data_updating_scripts/logs", exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(), logging.FileHandler("data_updating_scripts/logs/migrate_iapp_categories.log")]
+)
+logger = logging.getLogger(__name__)
+
+# Exact subcategory lists for validation
+EXACT_SUBCATEGORIES = {
+    "Governance": ["Program and documentation", "Assessments", "Training", "Responsible individual"],
+    "Transparency": ["General notice", "Labeling/notification", "Explanation/incident reporting", "Developer documentation"],
+    "Assurance": ["Registration", "Third-party review"],
+    "Individual Rights": ["Opt out/appeal", "Nondiscrimination"]
+}
+
+# Fallback categories for failed API calls
+FALLBACK_CATEGORIES = {
+    "Governance": ["Program and documentation"],
+    "Transparency": ["General notice"],
+    "Assurance": ["Registration"],
+    "Individual Rights": ["Opt out/appeal"]
+}
+
+def bill_key(b: Dict) -> str:
+    return f"{b.get('state','Unknown')}_{b.get('bill_number','Unknown')}"
+
+def sha256(s: Optional[str]) -> Optional[str]:
+    if not isinstance(s, str) or not s.strip():
+        return None
+    return hashlib.sha256(s.encode("utf-8")).hexdigest()
+
+def load_json(path: Path, default):
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return default
+
+def save_json(path: Path, data):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+class IAPPCategoriesMigrator:
+    """Migrates IAPP categories for bills with missing or invalid subcategories."""
+
+    def __init__(self, force: bool = False, rebuild_cache: bool = False, sleep_sec: float = 0.0):
+        """Initialize the migrator with configuration."""
+        self.config = ConfigManager()
+        if not self.config.OPENAI_API_KEY:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+
+        self.force = force
+        self.sleep_sec = max(0.0, sleep_sec)
+        # Cache
+        self.cache: Dict[str, Dict] = {} if rebuild_cache else load_json(CACHE_FILE, {})
+
+        self.llm = ChatOpenAI(
+            model=self.config.OPENAI_LLM_MODEL,
+            temperature=0.1,
+            max_tokens=1000
+        )
+
+        self.iapp_prompt = self._create_prompt()
+        self.chain = self.iapp_prompt | self.llm | StrOutputParser()
+        logger.info(
+            f"Initialized IAPPCategoriesMigrator | model={self.config.OPENAI_LLM_MODEL} | "
+            f"force={self.force} | rebuild_cache={rebuild_cache}"
+        )
+
+    def _create_prompt(self):
+        """Create the IAPP analysis prompt with relaxed subcategory matching."""
+        prompt_text = """
+Analyze the following AI-related bill content using the IAPP (International Association of Privacy Professionals) framework for AI governance categorization.
+
+Your response must be ONLY a JSON object in this exact format with nothing else before or after:
+{{"iapp_categories": {{"Governance": ["subcategory1", "subcategory2"], "Transparency": [], "Assurance": [], "Individual Rights": []}}}}
+
+Use these four main categories and their EXACT subcategories (no variations allowed):
+
+**Governance:**
+- Program and documentation
+- Assessments
+- Training
+- Responsible individual
+
+**Transparency:**
+- General notice
+- Labeling/notification
+- Explanation/incident reporting
+- Developer documentation
+
+**Assurance:**
+- Registration
+- Third-party review
+
+**Individual Rights:**
+- Opt out/appeal
+- Nondiscrimination
+
+Guidelines for categorization:
+- Select ALL applicable subcategories that the bill directly addresses or substantially discusses
+- If a category has no applicable subcategories, try to label it anyway based on surrounding context
+- Be specific – prioritize subcategories that are clearly supported, but use judgment if AI or governance themes are present
+- Focus on what the bill addresses or emphasizes, even if it doesn’t explicitly mandate requirements
+- If the bill discusses AI, automation, decision systems, digital governance, or national technology strategy, categorize it as best as possible
+- Avoid returning no categories when possible assuming that the bill is AI governance related, unless it truly could not be categorized into any of the four categories.
+
+Bill content to analyze: {context}
+"""
+        return ChatPromptTemplate.from_messages([
+            ("system", prompt_text),
+            ("human", "Analyze this bill for IAPP categories:")
+        ])
+
+    def docs_from_bill(self, bill: Dict) -> List[Document]:
+        txt = bill.get("text", "")
+        if not isinstance(txt, str) or not txt.strip():
+            return []
+        return [
+            Document(
+                page_content=txt,
+                metadata={
+                    "bill_key": bill_key(bill),
+                    "state": bill.get("state", "Unknown"),
+                    "bill_number": bill.get("bill_number", "Unknown"),
+                    "title": bill.get("title", "No title"),
+                },
+            )
+        ]
+
+    def is_valid_categories(self, iapp: Dict) -> bool:
+        if not isinstance(iapp, dict):
+            return False
+        for cat in ("Governance", "Transparency", "Assurance", "Individual Rights"):
+            if cat not in iapp or not isinstance(iapp[cat], list):
+                return False
+            for sub in iapp[cat]:
+                if sub not in EXACT_SUBCATEGORIES[cat]:
+                    return False
+        return True
+
+    def parse_llm(self, response: str) -> Optional[Dict]:
+        m = re.search(r"\{.*\}", response, re.DOTALL)
+        if not m:
+            return None
+        try:
+            obj = json.loads(m.group(0))
+            return obj.get("iapp_categories")
+        except Exception:
+            return None
+
+    def cached_match(self, b: Dict) -> Optional[Dict]:
+        bid = str(b.get("bill_id"))
+        ch = b.get("change_hash")
+        txt_hash = sha256(b.get("text"))
+        c = self.cache.get(bid)
+        if not c:
+            return None
+        if (ch and c.get("change_hash") == ch) or (txt_hash and c.get("text_sha256") == txt_hash):
+            return c.get("iapp_categories")
+        return None
+
+    def remember(self, b: Dict, iapp: Dict):
+        bid = str(b.get("bill_id"))
+        self.cache[bid] = {
+            "bill_id": bid,
+            "change_hash": b.get("change_hash"),
+            "text_sha256": sha256(b.get("text")),
+            "iapp_categories": iapp,
+            "updated_at": b.get("lastUpdatedAt"),
+            "state": b.get("state"),
+            "bill_number": b.get("bill_number"),
+            "title": b.get("title"),
+        }
+
+    def run(self):
+        if not INPUT_FILE.exists():
+            raise FileNotFoundError(f"Missing {INPUT_FILE}")
+        src_bills: List[Dict] = load_json(INPUT_FILE, [])
+        vis_bills: List[Dict] = load_json(VIS_FILE, [])
+
+        vis_map = {bill_key(b): b for b in vis_bills}
+
+        total = len(src_bills)
+        reused_cache = 0
+        reused_vis = 0
+        computed = 0
+        skipped_no_text = 0
+        errors = 0
+
+        out_bills: List[Dict] = []
+
+        logger.info(f"Loaded {total} source bills; visualize has {len(vis_bills)} existing entries; cache size={len(self.cache)}")
+
+        for i, b in enumerate(src_bills, 1):
+            key = bill_key(b)
+            txt = b.get("text", "")
+
+            out_rec = b.copy()
+
+            if not isinstance(txt, str) or len(txt.strip()) <= 50:
+                prev = vis_map.get(key)
+                if prev and "iapp_categories" in prev:
+                    out_rec["iapp_categories"] = prev["iapp_categories"]
+                else:
+                    out_rec["iapp_categories"] = None
+                out_bills.append(out_rec)
+                skipped_no_text += 1
+                if i % 50 == 0:
+                    logger.info(f"[{i}/{total}] progress...")
+                continue
+
+            if self.force:
+                iapp = self._compute_categories(b)
+                if iapp is None:
+                    iapp = FALLBACK_CATEGORIES
+                    errors += 1
+                else:
+                    computed += 1
+                out_rec["iapp_categories"] = iapp
+                self.remember(b, iapp)
+                out_bills.append(out_rec)
+                if self.sleep_sec:
+                    time.sleep(self.sleep_sec)
+                if i % 10 == 0:
+                    save_json(VIS_FILE, out_bills)
+                    save_json(CACHE_FILE, self.cache)
+                continue
+
+            cached = self.cached_match(b)
+            if cached and self.is_valid_categories(cached):
+                out_rec["iapp_categories"] = cached
+                out_bills.append(out_rec)
+                reused_cache += 1
+                if i % 50 == 0:
+                    logger.info(f"[{i}/{total}] progress...")
+                continue
+
+            prev = vis_map.get(key)
+            if prev and "iapp_categories" in prev:
+                prev_bid = str(prev.get("bill_id"))
+                prev_cache = self.cache.get(prev_bid)
+                if prev_cache:
+                    same = False
+                    if b.get("change_hash") and prev_cache.get("change_hash") == b.get("change_hash"):
+                        same = True
+                    elif sha256(b.get("text")) == prev_cache.get("text_sha256"):
+                        same = True
+                    if same and self.is_valid_categories(prev.get("iapp_categories", {})):
+                        out_rec["iapp_categories"] = prev["iapp_categories"]
+                        out_bills.append(out_rec)
+                        reused_vis += 1
+                        if i % 50 == 0:
+                            logger.info(f"[{i}/{total}] progress...")
+                        continue
+
+            iapp = self._compute_categories(b)
+            if iapp is None:
+                iapp = FALLBACK_CATEGORIES
+                errors += 1
+            else:
+                computed += 1
+            out_rec["iapp_categories"] = iapp
+            self.remember(b, iapp)
+            out_bills.append(out_rec)
+
+            if self.sleep_sec:
+                time.sleep(self.sleep_sec)
+            if i % 10 == 0:
+                save_json(VIS_FILE, out_bills)
+                save_json(CACHE_FILE, self.cache)
+
+        save_json(VIS_FILE, out_bills)
+        save_json(CACHE_FILE, self.cache)
+
+        logger.info("IAPP migration complete.")
+        logger.info(f"Total: {total} | reused_cache: {reused_cache} | reused_visualize: {reused_vis} | computed: {computed} | no_text: {skipped_no_text} | errors: {errors}")
+        print("✅ IAPP categories migration completed successfully!")
+        print(f"   Total: {total}")
+        print(f"   Reused (cache): {reused_cache}")
+        print(f"   Reused (visualize match): {reused_vis}")
+        print(f"   Newly computed: {computed}")
+        print(f"   No text: {skipped_no_text}")
+        print(f"   Fallback/errors: {errors}")
+        print(f"   Results: {VIS_FILE}")
+        print(f"   Cache:   {CACHE_FILE}")
+
+    def _compute_categories(self, bill: Dict) -> Optional[Dict]:
+        try:
+            docs = self.docs_from_bill(bill)
+            if not docs:
+                return None
+            resp = self.chain.invoke({"context": docs})
+            parsed = self.parse_llm(resp)
+            if parsed and self.is_valid_categories(parsed):
+                return parsed
+            resp = self.chain.invoke({"context": docs})
+            parsed = self.parse_llm(resp)
+            if parsed and self.is_valid_categories(parsed):
+                return parsed
+            txt = bill.get("text", "").lower()
+            if "ai" in txt or "artificial intelligence" in txt:
+                return FALLBACK_CATEGORIES
+            return {"Governance": [], "Transparency": [], "Assurance": [], "Individual Rights": []}
+        except Exception as e:
+            logger.exception(f"LLM error for {bill_key(bill)}: {e}")
+            return None
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--force", action="store_true", help="Recompute categories for all bills with text")
+    parser.add_argument("--rebuild-cache", action="store_true", help="Ignore existing cache file and rebuild it")
+    parser.add_argument("--sleep-sec", type=float, default=0.0, help="Sleep seconds between LLM calls (rate limiting)")
+    args = parser.parse_args()
+
+    migrator = IAPPCategoriesMigrator(force=args.force, rebuild_cache=args.rebuild_cache, sleep_sec=args.sleep_sec)
+    migrator.run()
+
+if __name__ == "__main__":
+    main()

data/generate_password_hash.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+Password Hash Generator for streamlit-authenticator
+
+Usage:
+    python generate_password_hash.py
+
+This will prompt you for a password and generate the bcrypt hash.
+"""
+
+import streamlit_authenticator as stauth
+
+def generate_hash():
+    print("=" * 50)
+    print("Password Hash Generator")
+    print("=" * 50)
+    print()
+    
+    # Get password from user
+    password = input("Enter password to hash: ").strip()
+    
+    if not password:
+        print("Password cannot be empty!")
+        return
+    
+    # Generate hash
+    print("\nGenerating hash...")
+    hashed_passwords = stauth.Hasher([password]).generate()
+    hash_value = hashed_passwords[0]
+    
+    print("\nHash generated successfully!")
+    print("=" * 50)
+    print(f"\nYour hashed password:\n{hash_value}")
+    print("=" * 50)
+    
+    # Show example usage
+    print("\nAdd to config.yaml:")
+    print("-" * 50)
+    print(f"""
+credentials:
+  usernames:
+    username_here:
+      email: user@example.com
+      name: User Name
+      password: {hash_value}
+""")
+    
+    print("\nOr add to secrets.toml:")
+    print("-" * 50)
+    print(f"""
+[auth.credentials.usernames.username_here]
+email = "user@example.com"
+name = "User Name"
+password = "{hash_value}"
+""")
+    
+    print("\nDone! Copy the hash above to your config file.")
+
+def generate_multiple():
+    print("=" * 50)
+    print("Multiple User Password Hash Generator")
+    print("=" * 50)
+    print()
+    
+    users = {}
+    
+    while True:
+        username = input("\nEnter username (or press Enter to finish): ").strip()
+        if not username:
+            break
+        
+        password = input(f"Enter password for {username}: ").strip()
+        if not password:
+            print("Password cannot be empty! Skipping user.")
+            continue
+        
+        email = input(f"Enter email for {username}: ").strip()
+        name = input(f"Enter full name for {username}: ").strip()
+        
+        users[username] = {
+            'password': password,
+            'email': email or f"{username}@example.com",
+            'name': name or username.title()
+        }
+    
+    if not users:
+        print("\n No users to process!")
+        return
+    
+    # Generate all hashes
+    print("\nGenerating hashes...")
+    passwords = [data['password'] for data in users.values()]
+    hashed_passwords = stauth.Hasher(passwords).generate()
+    
+    # Update users with hashed passwords
+    for i, username in enumerate(users.keys()):
+        users[username]['hashed'] = hashed_passwords[i]
+    
+    # Display results
+    print("\nHashes generated successfully!")
+    print("=" * 50)
+    
+    print("\nconfig.yaml format:")
+    print("-" * 50)
+    print("credentials:")
+    print("  usernames:")
+    for username, data in users.items():
+        print(f"    {username}:")
+        print(f"      email: {data['email']}")
+        print(f"      name: {data['name']}")
+        print(f"      password: {data['hashed']}")
+    
+    print("\nsecrets.toml format:")
+    print("-" * 50)
+    for username, data in users.items():
+        print(f"[auth.credentials.usernames.{username}]")
+        print(f'email = "{data["email"]}"')
+        print(f'name = "{data["name"]}"')
+        print(f'password = "{data["hashed"]}"')
+        print()
+    
+    print("Done! Copy the configuration above to your config file.")
+
+if __name__ == "__main__":
+    print("\nChoose an option:")
+    print("1. Generate single password hash")
+    print("2. Generate multiple user hashes")
+    choice = input("\nEnter choice (1 or 2): ").strip()
+    
+    if choice == "1":
+        generate_hash()
+    elif choice == "2":
+        generate_multiple()
+    else:
+        print("Invalid choice!")

data/huggingface_upload.py

@@ -0,0 +1,251 @@
+"""
+HuggingFace Dataset Upload Module
+- Tests HF connection
+- Uploads known_bills_visualize.json (legacy function)
+- Uploads ALL core data JSONs (new function) to HuggingFace Datasets Hub
+Works with the Admin panel HuggingFace tab
+"""
+
+from huggingface_hub import HfApi, create_repo
+import streamlit as st
+import os
+import json
+from pathlib import Path
+from typing import Dict, List, Tuple, Optional
+
+
+FILES_TO_UPLOAD = {
+    "data/known_bills_visualize.json": "known_bills_visualize.json",
+    "data/bill_summaries.json": "bill_summaries.json",
+    "data/bill_suggested_questions.json": "bill_suggested_questions.json",
+    "data/bill_reports.json": "bill_reports.json",
+    "data/bill_cache.json": "bill_cache.json",
+    "data/known_bills.json": "known_bills.json",
+    "data/known_bills_fixed.json": "known_bills_fixed.json",
+}
+
+
+
+def _get_hf_token_and_repo() -> Tuple[str, str]:
+    """
+    Get HF token + dataset repo.
+
+    Priority:
+    1. Streamlit secrets (for the Admin UI)
+    2. Environment variables (for CLI scripts like update_data.py)
+       - HUGGINGFACE_HUB_TOKEN
+       - HF_REPO_ID
+    """
+    token = None
+    repo_id = None
+
+    try:
+        token = st.secrets["huggingface"]["token"]
+        repo_id = st.secrets["huggingface"]["dataset_repo"]
+    except Exception:
+        pass
+
+    if not token:
+        token = os.getenv("HUGGINGFACE_HUB_TOKEN")
+    if not repo_id:
+        repo_id = os.getenv("HF_REPO_ID")
+
+    if not token or not repo_id:
+        raise KeyError(
+            "HuggingFace configuration missing. "
+            "Provide either Streamlit secrets "
+            "[huggingface.token] and [huggingface.dataset_repo] "
+            "or environment variables HUGGINGFACE_HUB_TOKEN and HF_REPO_ID."
+        )
+
+    return token, repo_id
+
+
+
+def test_hf_connection() -> Tuple[bool, str]:
+    """
+    Test connection to HuggingFace API
+    
+    Returns:
+        tuple: (success: bool, message: str)
+    """
+    try:
+        token, _ = _get_hf_token_and_repo()
+        api = HfApi()
+        user = api.whoami(token=token)
+        username = user.get("name") or user.get("fullname") or user.get("id") or "User"
+        return True, f"Connected as: {username}"
+    except KeyError:
+        return False, "HuggingFace token or dataset_repo not found in secrets"
+    except Exception as e:
+        return False, f"Connection failed: {str(e)}"
+
+
+def get_dataset_url(filename: str = "known_bills_visualize.json") -> Optional[str]:
+    """
+    Get the public URL of a file inside the dataset.
+    
+    Args:
+        filename: Name of the file in the HF dataset repo.
+    
+    Returns:
+        str | None: URL to the dataset file, or None if config missing
+    """
+    try:
+        repo = st.secrets["huggingface"]["dataset_repo"]
+        return f"https://huggingface.co/datasets/{repo}/resolve/main/{filename}"
+    except KeyError:
+        return None
+
+
+def _find_and_validate_json(possible_paths: List[Path]) -> Path:
+    """
+    Given a list of possible paths, return the first that exists,
+    and validate that it is valid JSON.
+    """
+    file_path = None
+    for path in possible_paths:
+        if path.exists():
+            file_path = path
+            break
+
+    if file_path is None:
+        raise FileNotFoundError(
+            "File not found.\n"
+            "Checked locations:\n" + "\n".join(f"  - {p}" for p in possible_paths)
+        )
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            if not isinstance(data, (dict, list)):
+                raise ValueError("JSON file must contain a dict or list")
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON file: {str(e)}")
+
+    return file_path
+
+
+def _ensure_dataset_exists(api: HfApi, repo_id: str, token: str) -> None:
+    """Create the dataset repo if it does not already exist."""
+    try:
+        create_repo(
+            repo_id=repo_id,
+            repo_type="dataset",
+            token=token,
+            exist_ok=True,
+            private=False,
+        )
+    except Exception:
+        pass
+
+
+def upload_to_huggingface() -> str:
+    """
+    Legacy function: Upload ONLY known_bills_visualize.json to HuggingFace Datasets Hub.
+    Used by existing Admin panel code. New code should prefer upload_all_to_huggingface().
+    
+    Returns:
+        str: Public URL to the uploaded file
+        
+    Raises:
+        FileNotFoundError: If JSON file doesn't exist
+        Exception: If upload fails
+    """
+    try:
+        token, repo_id = _get_hf_token_and_repo()
+        api = HfApi()
+
+        _ensure_dataset_exists(api, repo_id, token)
+
+        possible_paths = [
+            Path("data/known_bills_visualize.json"),
+            Path("known_bills_visualize.json"),
+        ]
+        file_path = _find_and_validate_json(possible_paths)
+
+        file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+
+        api.upload_file(
+            path_or_fileobj=str(file_path),
+            path_in_repo="known_bills_visualize.json",
+            repo_id=repo_id,
+            repo_type="dataset",
+            token=token,
+            commit_message=f"Update AI legislation data ({file_size_mb:.2f}MB)",
+        )
+
+        url = get_dataset_url("known_bills_visualize.json")
+        return url
+
+    except FileNotFoundError as e:
+        raise e
+    except KeyError as e:
+        raise Exception(f"Missing configuration in secrets.toml: {e}")
+    except Exception as e:
+        raise Exception(f"Upload failed: {str(e)}")
+
+
+def upload_all_to_huggingface() -> Dict[str, str]:
+    """
+    NEW: Upload ALL core JSON files to HuggingFace Datasets Hub.
+    
+    Returns:
+        dict: mapping from dataset filename -> public URL (for successfully uploaded files)
+    """
+    token, repo_id = _get_hf_token_and_repo()
+    api = HfApi()
+    _ensure_dataset_exists(api, repo_id, token)
+
+    uploaded_urls: Dict[str, str] = {}
+
+    for local_path, dest_name in FILES_TO_UPLOAD.items():
+        possible_paths = [Path(local_path), Path(dest_name)]
+
+        try:
+            file_path = _find_and_validate_json(possible_paths)
+        except FileNotFoundError:
+            msg = f"Skipping missing file: {local_path}"
+            print(msg)
+            st.write(msg)
+            continue
+        except ValueError as e:
+            msg = f"Skipping invalid JSON in {local_path}: {e}"
+            print(msg)
+            st.write(msg)
+            continue
+
+        file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+        commit_msg = f"Update {dest_name} ({file_size_mb:.2f}MB)"
+
+        print(f"Uploading {file_path} → {repo_id}/{dest_name} ...")
+        api.upload_file(
+            path_or_fileobj=str(file_path),
+            path_in_repo=dest_name,
+            repo_id=repo_id,
+            repo_type="dataset",
+            token=token,
+            commit_message=commit_msg,
+        )
+
+        url = get_dataset_url(dest_name)
+        if url:
+            uploaded_urls[dest_name] = url
+
+    return uploaded_urls
+
+
+if __name__ == "__main__":
+    print("Testing HuggingFace connection...")
+    success, msg = test_hf_connection()
+    print(msg)
+
+    if success:
+        print("\nAttempting upload of ALL files...")
+        try:
+            urls = upload_all_to_huggingface()
+            print("\nUpload successful!")
+            for name, url in urls.items():
+                print(f"- {name}: {url}")
+        except Exception as e:
+            print(f"\nUpload failed: {e}")

data/pages/Admin.py

@@ -0,0 +1,459 @@
+import streamlit as st
+import streamlit_authenticator as stauth
+from pathlib import Path
+import sys
+import pandas as pd
+import subprocess
+from datetime import datetime
+import os
+from huggingface_upload import upload_all_to_huggingface
+
+# Allow imports of project modules
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from user_management import HuggingFaceUserManager, load_user_config
+
+st.set_page_config(page_title="Admin Panel", layout="wide", page_icon="🛠️")
+
+# CSS
+st.markdown("""
+<style>
+    .main .block-container { padding-top: 2rem; max-width: 1200px; }
+    h2 { color: #e0e0e0 !important; font-weight: 400 !important; font-size: 1.5rem !important; }
+</style>
+""", unsafe_allow_html=True)
+
+# CONFIG
+config, using_hf = load_user_config()
+
+if config is None:
+    st.error("Authentication configuration not found!")
+    st.stop()
+
+# AUTH SYSTEM
+authenticator = stauth.Authenticate(
+    config['credentials'],
+    config['cookie']['name'],
+    config['cookie']['key'],
+    config['cookie']['expiry_days']
+)
+
+try:
+    authenticator.login('main')
+except Exception as e:
+    st.error(f"Login error: {e}")
+
+name = st.session_state.get("name")
+authentication_status = st.session_state.get("authentication_status")
+username = st.session_state.get("username")
+
+if authentication_status == False:
+    st.error('Username/password is incorrect')
+    st.stop()
+
+if authentication_status == None:
+    st.warning('Please enter your username and password')
+    st.stop()
+
+# AUTH VIEW
+if authentication_status:
+
+    with st.sidebar:
+        st.markdown("---")
+        st.markdown(f"**Logged in as:** {name}")
+        st.markdown(f"**Username:** {username}")
+        authenticator.logout('Logout', 'sidebar')
+
+    ALLOWED_USERNAMES = set(config['credentials']['usernames'].keys())
+    if username not in ALLOWED_USERNAMES:
+        st.error(f"User '{username}' is not authorized.")
+        st.stop()
+
+    # HEADER
+    st.success(f"Welcome, {name}!")
+    st.markdown("---")
+    st.markdown("""
+    <div style='text-align: center; padding: 1rem 0 2rem 0;'>
+        <h1 style='color: #1f2937;'>Admin Panel</h1>
+        <p style='color: #6b7280;'>Cloud data sync controls</p>
+    </div>
+    """, unsafe_allow_html=True)
+    st.markdown("---")
+
+    # Tabs
+    tab1, tab2, tab3 = st.tabs(["Dashboard", "Data Pipeline", "User Management"])
+
+    # ------------------------------------------------------------------
+    # TAB 1 — Dashboard
+    # ------------------------------------------------------------------
+    with tab1:
+        st.subheader("Admin Dashboard")
+
+        users = config['credentials']['usernames']
+        admin_data = [
+            {
+                "Username": uname,
+                "Name": data.get("name"),
+                "Email": data.get("email"),
+                "Current User": "Admin" if uname == username else ""
+            }
+            for uname, data in users.items()
+        ]
+
+        st.dataframe(pd.DataFrame(admin_data), width="stretch", hide_index=True)
+
+    # ------------------------------------------------------------------
+    # TAB 2 — DATA PIPELINE
+    # ------------------------------------------------------------------
+    with tab2:
+        st.subheader("Data Pipeline")
+
+        if 'huggingface' not in st.secrets:
+            st.warning("Add HuggingFace credentials to `.streamlit/secrets.toml`")
+            st.stop()
+
+        from huggingface_upload import upload_to_huggingface, test_hf_connection
+
+        # --- Connection Test
+        st.markdown("Connection Status")
+        col1, col2 = st.columns(2)
+
+        with col1:
+            if st.button("Test HuggingFace Connection", width='stretch'):
+                ok, msg = test_hf_connection()
+                (st.success if ok else st.error)(msg)
+
+        with col2:
+            repo = st.secrets["huggingface"]["dataset_repo"]
+            st.info(f"Dataset: {repo}")
+
+        st.markdown("---")
+
+        # --- Full Data Update Section
+        st.subheader("Full Data Update")
+        st.info("Pull new data, process PDFs, generate embeddings, and upload to HuggingFace.")
+
+        # ➤ NEW UI CONTROL — Pull new data?
+        pull_new_data = st.radio(
+            "Pull new data from LegiScan?",
+            options=[
+                ("no", "No - Use existing local data"),
+                ("yes", "Yes - Pull fresh data (costs API quota)"),
+            ],
+            format_func=lambda x: x[1],
+            index=0,
+            key="pull_option"
+        )
+
+        # ➤ NEW UI CONTROL — overwrite known_bills.json?
+        overwrite_pdf = st.radio(
+            "After fixing PDF bills, overwrite data/known_bills.json?",
+            options=[
+                ("no", "No - keep original file"),
+                ("yes", "Yes - overwrite with cleaned PDF text"),
+            ],
+            format_func=lambda x: x[1],
+            index=0,
+            key="overwrite_option"
+        )
+
+        # Run full update
+        if st.button("Run Full Update & Upload", type="primary", width='stretch'):
+            status_container = st.container()
+
+            with status_container:
+                st.markdown("### Step 1: Running Data Pipeline")
+
+                with st.status("Processing data...", expanded=True) as status:
+
+                    try:
+                        update_cmd = [sys.executable, "update_data.py"]
+                        legiscan_answer = "y\n" if pull_new_data[0] == "yes" else "n\n"
+
+                        import os
+                        from dotenv import load_dotenv
+                        load_dotenv()
+
+                        env = os.environ.copy()
+
+                        # Pass OpenAI keys (existing logic)
+                        openai_key = (
+                            st.secrets.get("openai_api_key")
+                            or st.secrets.get("OPENAI_API_KEY")
+                            or env.get("openai_api_key")
+                            or env.get("OPENAI_API_KEY")
+                        )
+
+                        if openai_key:
+                            env["OPENAI_API_KEY"] = openai_key
+                            env["openai_api_key"] = openai_key
+                            st.success("OpenAI key found")
+                        else:
+                            st.warning("OpenAI API key missing!")
+
+                        # ➤ NEW: Pass PDF overwrite decision into environment
+                        env["FIX_PDF_OVERWRITE"] = (
+                            "yes" if overwrite_pdf[0] == "yes" else "no"
+                        )
+
+                        log_file = Path("pipeline_last_run.log")
+
+                        with log_file.open("w", encoding="utf-8") as lf:
+                            proc = subprocess.Popen(
+                                update_cmd,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT,
+                                stdin=subprocess.PIPE,
+                                text=True,
+                                bufsize=1,
+                                env=env,
+                            )
+
+                            # Send LegiScan yes/no
+                            try:
+                                proc.stdin.write(legiscan_answer)
+                                proc.stdin.write("n\n")  # continue-on-error prompt
+                                proc.stdin.flush()
+                                proc.stdin.close()
+                            except:
+                                pass
+
+                            # Stream output
+                            for line in proc.stdout:
+                                line = line.rstrip("\n")
+                                st.text(line)
+                                lf.write(line + "\n")
+
+                            rc = proc.wait()
+
+                        if rc == 0:
+                            status.update(label="Data pipeline completed", state="complete")
+                            st.success("Processing successful!")
+
+                            st.markdown("---")
+                            st.markdown("### Step 2: Uploading to HuggingFace")
+
+                            with st.spinner("Uploading..."):
+                                url = upload_to_huggingface()
+                                st.success("Uploaded to HuggingFace!")
+                                st.code(url)
+                                st.cache_data.clear()
+
+                        else:
+                            status.update(label="Pipeline failed", state="error")
+                            st.error(f"Pipeline exited with code {rc}")
+
+                    except Exception as e:
+                        st.error(f"Pipeline error: {e}")
+                        st.exception(e)
+
+        st.markdown("---")
+
+        with st.expander("Manual Upload Only"):
+            st.info("Use this only when skipping update_data.py")
+
+            if st.button("Upload Existing Data", width='stretch'):
+                with st.spinner("Uploading..."):
+                    url = upload_to_huggingface()
+                    st.success("Uploaded!")
+                    st.code(url)
+
+    
+    with tab3:
+        st.subheader("User Management")
+        
+        if using_hf:
+            st.success("Using HuggingFace for persistent user storage")
+            
+            try:
+                user_manager = HuggingFaceUserManager()
+                
+                st.markdown("Add New Admin")
+                
+                with st.form("add_user_form"):
+                    col1, col2 = st.columns(2)
+                    with col1:
+                        new_username = st.text_input("Username", key="new_username")
+                        new_email = st.text_input("Email", key="new_email")
+                    with col2:
+                        new_name = st.text_input("Full Name", key="new_name")
+                        new_password = st.text_input("Password", type="password", key="new_password")
+                    
+                    submit_add = st.form_submit_button("Add Admin", type="primary", width='stretch')
+                    
+                    if submit_add:
+                        if not all([new_username, new_email, new_name, new_password]):
+                            st.error("Please fill in all fields")
+                        else:
+                            with st.spinner("Adding user..."):
+                                import bcrypt
+                                hashed_password = bcrypt.hashpw(new_password.encode(), bcrypt.gensalt()).decode()
+                                
+                                success, message, commit_url = user_manager.add_user(
+                                    new_username, new_email, new_name, hashed_password
+                                )
+                                
+                                if success:
+                                    st.success(f"{message}")
+                                    st.cache_data.clear()
+                                    if commit_url:
+                                        with st.expander("View commit"):
+                                            st.code(commit_url)
+                                    st.rerun()
+                                else:
+                                    st.error(f"{message}")
+                
+                st.markdown("---")
+                
+                st.markdown("Edit Admin")
+                
+                users = config['credentials']['usernames']
+                usernames_list = list(users.keys())
+                
+                with st.form("edit_user_form"):
+                    user_to_edit = st.selectbox(
+                        "Select user to edit",
+                        options=usernames_list,
+                        key="edit_username"
+                    )
+                    
+                    current_user = users.get(user_to_edit, {})
+                    
+                    st.markdown("**Current Details:**")
+                    st.text(f"Email: {current_user.get('email', 'N/A')}")
+                    st.text(f"Name: {current_user.get('name', 'N/A')}")
+                    
+                    st.markdown("**New Details** (leave blank to keep current):")
+                    
+                    col1, col2 = st.columns(2)
+                    with col1:
+                        new_email = st.text_input("New Email", key="edit_email", placeholder="Leave blank to keep current")
+                        new_password = st.text_input("New Password", type="password", key="edit_password", placeholder="Leave blank to keep current")
+                    with col2:
+                        new_name = st.text_input("New Name", key="edit_name", placeholder="Leave blank to keep current")
+                    
+                    submit_edit = st.form_submit_button("Update Admin", type="primary", width='stretch')
+                    
+                    if submit_edit:
+                        if not any([new_email, new_name, new_password]):
+                            st.warning("Please enter at least one field to update")
+                        else:
+                            with st.spinner("Updating user..."):
+                                hashed_password = None
+                                if new_password:
+                                    import bcrypt
+                                    hashed_password = bcrypt.hashpw(new_password.encode(), bcrypt.gensalt()).decode()
+                                
+                                success, message, commit_url = user_manager.update_user(
+                                    user_to_edit,
+                                    new_email=new_email if new_email else None,
+                                    new_name=new_name if new_name else None,
+                                    new_password=hashed_password
+                                )
+                                
+                                if success:
+                                    st.success(f"{message}")
+                                    st.info("Refreshing user data...")
+                                    st.cache_data.clear()
+                                    if commit_url:
+                                        with st.expander("View commit"):
+                                            st.code(commit_url)
+                                    st.info("Please log out and log back in if you changed your own password")
+                                    st.rerun()
+                                else:
+                                    st.error(f"{message}")
+                
+                st.markdown("---")
+                
+                # Remove user
+                st.markdown("Remove Admin")
+                
+                users = config['credentials']['usernames']
+                usernames_list = list(users.keys())
+                
+                if len(usernames_list) > 1:
+                    with st.form("remove_user_form"):
+                        user_to_remove = st.selectbox(
+                            "Select user to remove",
+                            options=usernames_list,
+                            key="remove_username"
+                        )
+                        
+                        st.warning(f"This will permanently delete user: **{user_to_remove}**")
+                        
+                        confirm_remove = st.checkbox("I confirm I want to remove this user")
+                        submit_remove = st.form_submit_button("Remove Admin", type="secondary", width='stretch')
+                        
+                        if submit_remove:
+                            if not confirm_remove:
+                                st.error("Please confirm the removal")
+                            elif user_to_remove == username:
+                                st.error("You cannot remove yourself!")
+                            else:
+                                with st.spinner("Removing user..."):
+                                    success, message, commit_url = user_manager.remove_user(user_to_remove)
+                                    
+                                    if success:
+                                        st.success(f"✅ {message}")
+                                        st.cache_data.clear()
+                                        if commit_url:
+                                            with st.expander("View commit"):
+                                                st.code(commit_url)
+                                        st.rerun()
+                                    else:
+                                        st.error(f"{message}")
+                else:
+                    st.info("ℹCannot remove the last admin user")
+                
+                st.markdown("---")
+                
+                # Show current users
+                st.markdown("Current Admins")
+                for uname, udata in users.items():
+                    with st.expander(f"{udata.get('name', uname)} (@{uname})"):
+                        st.write(f"**Email:** {udata.get('email', 'N/A')}")
+                        st.write(f"**Username:** {uname}")
+                        st.write(f"**Admin Status:**Admin")
+                        
+                        if uname == username:
+                            st.info("This is you!")
+                
+            except Exception as e:
+                st.error(f"Error initializing user manager: {e}")
+                st.exception(e)
+        
+        else:
+            st.warning("Using secrets.toml (read-only)")
+            st.info("For persistent user management, add HuggingFace credentials to secrets.toml")
+            
+            with st.expander("How to add users manually"):
+                st.markdown("""
+                **To add new users when using secrets.toml:**
+                
+                1. **Generate password hash:**
+                ```bash
+                python generate_password_hash.py
+                ```
+                
+                2. **Add to secrets.toml:**
+                ```toml
+                [auth.credentials.usernames.newuser]
+                email = "user@vanderbilt.edu"
+                name = "New User"
+                password = "$2b$12$HASH_FROM_STEP_1"
+                ```
+                
+                3. **Update on HuggingFace Spaces** (re-upload secrets.toml)
+                
+                All registered users automatically get admin access.
+                """)
+            
+            st.markdown("---")
+            
+            st.markdown("Current Admins")
+            if 'credentials' in config and 'usernames' in config['credentials']:
+                users = config['credentials']['usernames']
+                for uname, udata in users.items():
+                    with st.expander(f"{udata.get('name', uname)} (@{uname})"):
+                        st.write(f"**Email:** {udata.get('email', 'N/A')}")
+                        st.write(f"**Username:** {uname}")
+                        st.write(f"**Admin Status:Admin")

data/update_data.py

@@ -0,0 +1,64 @@
+import subprocess
+import sys
+from huggingface_upload import upload_all_to_huggingface
+
+all_scripts = [
+    "data_updating_scripts/get_data.py",
+    "data_updating_scripts/fix_pdf_bills.py",
+    "data_updating_scripts/known_bills_status.py",
+    "data_updating_scripts/migrate_iapp_categories.py",
+    "data_updating_scripts/mark_no_text_bills.py",
+    "data_updating_scripts/generate_summaries.py",
+    "data_updating_scripts/generate_suggested_questions.py",
+    "data_updating_scripts/generate_reports.py",
+    "data_updating_scripts/eu_vectorstore.py",
+]
+
+print("Do you want to pull new data from LegiScan?")
+print("Enter 'y' or 'yes' to pull new data, or 'n' or 'no' to skip and use existing data:")
+response = input().strip().lower()
+
+if response in ["y", "yes"]:
+    print("\n✓ Will pull new data from LegiScan")
+    scripts_to_run = all_scripts
+elif response in ["n", "no"]:
+    print("\n✓ Skipping data pull, using existing data")
+    scripts_to_run = all_scripts[2:]
+else:
+    print(f"\n✗ Invalid response '{response}'. Please run the script again and enter 'y' or 'n'.")
+    sys.exit(1)
+
+print(f"\nWill run {len(scripts_to_run)} scripts:")
+for script in scripts_to_run:
+    print(f"  - {script}")
+
+print("\n" + "=" * 50)
+
+for script in scripts_to_run:
+    print(f"\n--- Running {script} ---")
+    print("=" * 50)
+
+    result = subprocess.run([sys.executable, script])
+
+    if result.returncode != 0:
+        print(f"\n✗ Script {script} failed with return code {result.returncode}")
+        print("Do you want to continue with the remaining scripts? (y/n):")
+        continue_response = input().strip().lower()
+        if continue_response not in ["y", "yes"]:
+            print("Stopping pipeline execution.")
+            sys.exit(1)
+    else:
+        print(f"✓ {script} completed successfully")
+
+print("\n" + "=" * 50)
+print("✓ Pipeline execution completed!")
+
+print("\nUploading all JSON datasets to HuggingFace…")
+
+try:
+    upload_all_to_huggingface()
+    print("✓ HuggingFace upload complete!")
+except KeyError as e:
+    print(f"✗ HuggingFace config error: {e}")
+except Exception as e:
+    print(f"✗ Upload failed: {e}")