Update app.py

app.py

@@ -31,106 +31,51 @@ os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')
 # Utility Functions
 # -------------------------------
 
-def extract_metadata(text: str) -> dict:
-    metadata = {}
-
-    # Extract the Title field
-    title_match = re.search(
-        r"Title:\s*(.*?)\s+(?=Website:|Twitter:|Instagram:|FaceBook:|Newsletter:)",
-        text,
-        re.IGNORECASE | re.DOTALL
-    )
-    if title_match:
-        metadata["title"] = title_match.group(1).strip()
-
-    # Extract the Ranking field and store raw ranking info
-    ranking_match = re.search(r"Ranking:\s*([^\n]+)", text, re.IGNORECASE)
-    if ranking_match:
-        ranking_value = ranking_match.group(1).strip()
-        metadata["raw_ranking"] = ranking_value  # Store full ranking information
-        if ranking_value.lower() == "winner":
-            metadata["LA2050 Grant Winner"] = ranking_value
-
-    # Extract the Year field (assuming a four-digit year)
-    year_match = re.search(r"Year:\s*(\d{4})", text, re.IGNORECASE)
-    if year_match:
-        metadata["year"] = year_match.group(1).strip()
-
-    # Extract the Organization field
-    org_match = re.search(
-        r"Organization:\s*(.*?)\s+(?=Goal:|Ranking:|Impact Metrics:)",
-        text,
-        re.IGNORECASE | re.DOTALL
-    )
-    if org_match:
-        metadata["organization"] = org_match.group(1).strip()
-
-    # Modified URL extraction: make http/https optional.
-    urls = re.findall(r"(Website|Volunteer|Newsletter):\s*((?:https?://)?\S+)", text)
-    for key, url in urls:
-        metadata[key.lower()] = url.strip()
-
-    # Adjust social handle extraction to capture full URLs.
-    social = re.findall(r"(Twitter|Instagram|FaceBook):\s*(\S+)", text)
-    for platform, handle in social:
-        if handle.startswith("http"):
-            metadata[platform.lower()] = handle.strip()
-        else:
-            metadata[f"{platform.lower()}_handle"] = f"https://{platform.lower()}.com/{handle.strip()}"
-
-    # Extract Working Areas in LA (if available)
-    working_match = re.search(r"Working Areas in LA:\s*(.*?)\s+(?=Summary:|$)", text, re.IGNORECASE | re.DOTALL)
-    if working_match:
-        metadata["working_areas_in_la"] = working_match.group(1).strip()
-
-    # Extract Zipcode (if available; assuming it is a 5-digit number)
-    zipcode_match = re.search(r"Zipcode:\s*(\d{5})", text, re.IGNORECASE)
-    if zipcode_match:
-        metadata["zipcode"] = zipcode_match.group(1).strip()
-
-    return metadata
-
-
-def load_and_process_data(file_path: str):
-    """
-    Loads JSON data from a file, extracts organization text and metadata,
-    and returns a list of Documents. Documents will have the ranking metadata
-    only if the organization is marked as a winner.
-    """
-    try:
-        data = json.loads(Path(file_path).read_text(encoding='utf-8'))
-        docs = []
-        for entry in data:
-            org_text = entry.get("OrganizationText", "")
-            if not org_text:
-                continue
-            metadata = extract_metadata(org_text)
-            # Insert winners at the beginning of the list
-            if metadata.get("LA2050 Grant Winner", "").lower() == "winner":
-                docs.insert(0, Document(page_content=org_text, metadata=metadata))
-            else:
-                docs.append(Document(page_content=org_text, metadata=metadata))
-        return docs
-    except Exception as e:
-        print(f"Error loading JSON: {e}")
-        return []
+def metadata_func(record,additional_fields=None):
+    return {
+        "title": record.get("Title", ""),
+        "organization": record.get("Organization", ""),
+        "LA 2050 Grant Status": record.get("Ranking", ""),
+        "impact": record.get("Impact Metrics", ""),
+        "year": record.get("Year", ""),
+        "urls": {
+            "website": record.get("Website", ""),
+            "twitter": record.get("Twitter", ""),
+            "instagram": record.get("Instagram", ""),
+            "facebook": record.get("FaceBook", ""),
+            "newsletter": record.get("Newsletter", ""),
+            "volunteer": record.get("Volunteer", ""),
+            "la2050": record.get("LA2050", "")
+        },
+        "social": {
+            "twitter": record.get("Twitter", ""),
+            "instagram": record.get("Instagram", ""),
+            "facebook": record.get("FaceBook", "")
+        },
+        "working_area": record.get("Working Areas in LA", ""),
+        "zipcode": record.get("Zipcode", "")
+    }
 
-# -------------------------------
-# Data Loading and Preprocessing
-# -------------------------------
+# Load the JSON data with custom metadata and content key
+loader = JSONLoader(
+    file_path='data.json',
+    jq_schema='.[]',
+    content_key='Summary',
+    metadata_func=metadata_func  # Pass the metadata_func function directly here
+)
+
+data = loader.load()
 
-file_path = './data.json'  # Ensure this file is available in your environment.
-docs = load_and_process_data(file_path)
 
 # Use a text splitter to create chunks from the documents.
 # (If you find that key fields are getting split, consider implementing a custom splitter.)
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=1600,  
+    chunk_size=760,  
     chunk_overlap=100,  
     add_start_index=True
 )
-all_splits = text_splitter.split_documents(docs)
+all_splits = text_splitter.split_documents(data)
 
 # -------------------------------
 # Set Up Retrievers