Update app.py

app.py

@@ -1,10 +1,10 @@
-# app.py (Updated for a model with pre-normalized embeddings)
+# app.py
 
 import sys
 import subprocess
+import json
 from flask import Flask, render_template, request, flash, redirect, url_for, jsonify
 import torch
-import torch.nn.functional as F
 from transformers import AutoTokenizer, AutoModel
 import os
 import chromadb
@@ -13,19 +13,47 @@ from huggingface_hub import snapshot_download
 app = Flask(__name__)
 app.secret_key = os.urandom(24)
 
+# --- App Configuration ---
 CHROMA_PATH = "chroma_db"
 COLLECTION_NAME = "bible_verses"
-# *** CHANGE 1: USE A MODEL WITH NORMALIZED EMBEDDINGS ***
 MODEL_NAME = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
-# *** CHANGE 2: USE THE NEW REPO FOR THE NEW DATABASE ***
 DATASET_REPO = "broadfield-dev/bible-chromadb-multi-qa-mpnet"
 STATUS_FILE = "build_status.log"
+JSON_DIRECTORY = 'bible_json'
 
+# --- Globals and Helpers ---
 chroma_collection = None
 tokenizer = None
 embedding_model = None
 
-# Mean Pooling Function - Crucial for sentence-transformer models
+# *** ADD 1: BIBLE BOOK MAPPINGS FOR DATA RETRIEVAL ***
+BOOK_ID_TO_NAME = {
+    1: "Genesis", 2: "Exodus", 3: "Leviticus", 4: "Numbers", 5: "Deuteronomy", 6: "Joshua", 7: "Judges", 8: "Ruth", 9: "1 Samuel", 10: "2 Samuel", 11: "1 Kings", 12: "2 Kings", 13: "1 Chronicles", 14: "2 Chronicles", 15: "Ezra", 16: "Nehemiah", 17: "Esther", 18: "Job", 19: "Psalms", 20: "Proverbs", 21: "Ecclesiastes", 22: "Song of Solomon", 23: "Isaiah", 24: "Jeremiah", 25: "Lamentations", 26: "Ezekiel", 27: "Daniel", 28: "Hosea", 29: "Joel", 30: "Amos", 31: "Obadiah", 32: "Jonah", 33: "Micah", 34: "Nahum", 35: "Habakkuk", 36: "Zephaniah", 37: "Haggai", 38: "Zechariah", 39: "Malachi", 40: "Matthew", 41: "Mark", 42: "Luke", 43: "John", 44: "Acts", 45: "Romans", 46: "1 Corinthians", 47: "2 Corinthians", 48: "Galatians", 49: "Ephesians", 50: "Philippians", 51: "Colossians", 52: "1 Thessalonians", 53: "2 Thessalonians", 54: "1 Timothy", 55: "2 Timothy", 56: "Titus", 57: "Philemon", 58: "Hebrews", 59: "James", 60: "1 Peter", 61: "2 Peter", 62: "1 John", 63: "2 John", 64: "3 John", 65: "Jude", 66: "Revelation"
+}
+BOOK_NAME_TO_ID = {v: k for k, v in BOOK_ID_TO_NAME.items()}
+
+# *** ADD 2: HELPER FUNCTION TO READ VERSES FROM JSON ***
+def get_verses_from_json(version, book_name, chapter=None):
+    book_id = BOOK_NAME_TO_ID.get(book_name)
+    if not book_id: return None
+    
+    file_path = os.path.join(JSON_DIRECTORY, f"t_{version.lower()}.json")
+    if not os.path.exists(file_path): return None
+
+    with open(file_path, 'r') as f: data = json.load(f)
+    
+    rows = data.get("resultset", {}).get("row", [])
+    verses = []
+    for row in rows:
+        field = row.get("field", [])
+        if len(field) == 5:
+            row_book_id, row_chapter, row_verse, text = field[1], field[2], field[3], field[4]
+            if row_book_id == book_id and (chapter is None or row_chapter == chapter):
+                verses.append({'chapter': row_chapter, 'verse': row_verse, 'text': text.strip()})
+    
+    verses.sort(key=lambda v: (v['chapter'], v['verse']))
+    return verses
+
 def mean_pooling(model_output, attention_mask):
     token_embeddings = model_output[0]
     input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
@@ -39,28 +67,19 @@ def load_resources():
         if not os.path.exists(CHROMA_PATH) or not os.listdir(CHROMA_PATH):
             print(f"Local DB not found. Downloading from '{DATASET_REPO}'...")
             snapshot_download(repo_id=DATASET_REPO, repo_type="dataset", local_dir=CHROMA_PATH, local_dir_use_symlinks=False)
-            print("Database files downloaded.")
-        else:
-            print("Local database files found.")
         client = chromadb.PersistentClient(path=CHROMA_PATH)
-        collection = client.get_collection(name=COLLECTION_NAME)
-        if collection.count() == 0:
-            print("Warning: Database collection is empty.")
-            return False
-        chroma_collection = collection
-        print(f"Successfully connected to DB with {collection.count()} items.")
+        chroma_collection = client.get_collection(name=COLLECTION_NAME)
         tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
         embedding_model = AutoModel.from_pretrained(MODEL_NAME)
-        print(f"Embedding model '{MODEL_NAME}' loaded successfully.")
+        print(f"Resources loaded: DB has {chroma_collection.count()} items. Model is '{MODEL_NAME}'.")
         return True
     except Exception as e:
-        print(f"Could not load resources. DB may not be built or repo is empty.")
-        print(f"Error: {e}")
+        print(f"Could not load resources. DB may not be built. Error: {e}")
         return False
 
 resources_loaded = load_resources()
 
-# (home, build_rag_route, and status routes are unchanged)
+# --- Routes ---
 @app.route('/')
 def home():
     return render_template('index.html')
@@ -86,9 +105,9 @@ def status():
 def search():
     global resources_loaded
     if not resources_loaded:
-        resources_loaded = load_resources()
+        resources_loaded = load_resources() # Retry loading
         if not resources_loaded:
-            flash("Database not ready. Please wait for the build process to finish and then refresh the page.", "error")
+            flash("Database not ready. Please build the database from the Admin Panel.", "error")
             return redirect(url_for('home'))
 
     user_query = request.form['query']
@@ -98,29 +117,49 @@ def search():
     encoded_input = tokenizer([user_query], padding=True, truncation=True, return_tensors='pt')
     with torch.no_grad():
         model_output = embedding_model(**encoded_input)
-    
     query_embedding = mean_pooling(model_output, encoded_input['attention_mask'])
 
-    # *** REMOVED: NO LONGER NEED TO NORMALIZE THE QUERY EMBEDDING ***
-    # query_embedding = F.normalize(query_embedding, p=2, dim=1)
-
-    search_results = chroma_collection.query(
-        query_embeddings=query_embedding.cpu().tolist(),
-        n_results=10
-    )
+    search_results = chroma_collection.query(query_embeddings=query_embedding.cpu().tolist(), n_results=10)
 
     results_list = []
     documents, metadatas, distances = search_results['documents'][0], search_results['metadatas'][0], search_results['distances'][0]
 
     for i in range(len(documents)):
+        meta = metadatas[i]
+        # *** CHANGE 3: PASS NEW METADATA TO TEMPLATE FOR LINKING ***
         results_list.append({
             'score': distances[i],
             'text': documents[i],
-            'reference': metadatas[i].get('reference', 'N/A'),
-            'version': metadatas[i].get('version', 'N/A')
+            'reference': meta.get('reference', 'N/A'),
+            'version': meta.get('version', 'N/A'),
+            'book_name': meta.get('book_name', ''),
+            'chapter': meta.get('chapter', '')
         })
     
     return render_template('index.html', results=results_list, query=user_query)
 
+# *** ADD 3: NEW ROUTE FOR VIEWING A FULL CHAPTER ***
+@app.route('/chapter/<version>/<book_name>/<int:chapter_num>')
+def view_chapter(version, book_name, chapter_num):
+    verses = get_verses_from_json(version, book_name, chapter=chapter_num)
+    if not verses:
+        return "Chapter not found.", 404
+    return render_template('chapter.html', book_name=book_name, chapter_num=chapter_num, verses=verses, version=version)
+
+# *** ADD 4: NEW ROUTE FOR VIEWING A FULL BOOK ***
+@app.route('/book/<version>/<book_name>')
+def view_book(version, book_name):
+    all_verses = get_verses_from_json(version, book_name)
+    if not all_verses:
+        return "Book not found.", 404
+    
+    chapters = {}
+    for verse in all_verses:
+        chap_num = verse['chapter']
+        if chap_num not in chapters: chapters[chap_num] = []
+        chapters[chap_num].append(verse)
+        
+    return render_template('book.html', book_name=book_name, chapters=chapters)
+
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=7860)