Update app.py

app.py

@@ -38,136 +38,58 @@ from pathlib import Path
 # Make sure to import your Document class from your LangChain module.
 from langchain_core.documents import Document  
 
-def extract_metadata(text: str) -> tuple[dict, str]:
+def extract_metadata(text: str) -> dict:
     metadata = {}
-    cleaned_text = text  # Start with the original text
     
-    # Extract and remove Title
+    # Extract the Title field
     title_match = re.search(
         r"Title:\s*(.*?)\s+(?=Website:|Twitter:|Instagram:|FaceBook:|Newsletter:)",
-        cleaned_text,
+        text,
         re.IGNORECASE | re.DOTALL
     )
     if title_match:
         metadata["title"] = title_match.group(1).strip()
-        cleaned_text = re.sub(
-            r"Title:\s*.*?(?=Website:|Twitter:|Instagram:|FaceBook:|Newsletter:)",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE | re.DOTALL
-        )
     
-    # Extract and remove Ranking (only if "winner")
+    # Extract the Organization field
+    org_match = re.search(
+        r"Organization:\s*(.*?)\s+(?=Goal:|Ranking:|Impact Metrics:)",
+        text,
+        re.IGNORECASE | re.DOTALL
+    )
+    if org_match:
+        metadata["organization"] = org_match.group(1).strip()
+    
+    # Extract the Ranking field with a more flexible pattern:
     ranking_match = re.search(
-        r"Ranking:\s*(.*?)\s+(?=Impact Metrics:|$)",
-        cleaned_text,
+        r"Ranking:\s*(.*?)\s*(?:Impact Metrics:|$)",
+        text,
         re.IGNORECASE | re.DOTALL
     )
     if ranking_match:
-        ranking_value = ranking_match.group(1).strip()
-        if ranking_value.lower() == "winner":
-            metadata["ranking"] = ranking_value
-        cleaned_text = re.sub(
-            r"Ranking:\s*.*?(?=Impact Metrics:|$)",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE | re.DOTALL
-        )
+        metadata["ranking"] = ranking_match.group(1).strip()
     
-    # Extract and remove Year
-    year_match = re.search(r"Year:\s*(\d{4})", cleaned_text, re.IGNORECASE)
+    # Extract the Year field (assuming a four-digit year)
+    year_match = re.search(r"Year:\s*(\d{4})", text, re.IGNORECASE)
     if year_match:
         metadata["year"] = year_match.group(1).strip()
-        cleaned_text = re.sub(r"Year:\s*\d{4}", "", cleaned_text, flags=re.IGNORECASE)
-    
-    # Extract and remove Organization
-    org_match = re.search(
-        r"Organization:\s*(.*?)\s+(?=Goal:|Ranking:|Impact Metrics:)",
-        cleaned_text,
-        re.IGNORECASE | re.DOTALL
-    )
-    if org_match:
-        metadata["organization"] = org_match.group(1).strip()
-        cleaned_text = re.sub(
-            r"Organization:\s*.*?(?=Goal:|Ranking:|Impact Metrics:)",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE | re.DOTALL
-        )
     
-    # Extract and remove URLs (Website, Volunteer, Newsletter)
-    urls = re.findall(r"(Website|Volunteer|Newsletter):\s*((?:https?://)?\S+)", cleaned_text)
+    # Extract URLs for Website, Volunteer, and Newsletter
+    urls = re.findall(r"(Website|Volunteer|Newsletter):\s*((?:https?://)?\S+)", text)
     for key, url in urls:
         metadata[key.lower()] = url.strip()
-        cleaned_text = re.sub(
-            rf"{key}:\s*{re.escape(url)}",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE
-        )
     
-    # Extract and remove social handles (Twitter, Instagram, FaceBook)
-    social = re.findall(r"(Twitter|Instagram|FaceBook):\s*(\S+)", cleaned_text)
+    # Extract social handles (Twitter, Instagram, FaceBook)
+    social = re.findall(r"(Twitter|Instagram|FaceBook):\s*(\S+)", text)
     for platform, handle in social:
         if handle.startswith("http"):
             metadata[platform.lower()] = handle.strip()
         else:
             metadata[f"{platform.lower()}_handle"] = f"https://{platform.lower()}.com/{handle.strip()}"
-        cleaned_text = re.sub(
-            rf"{platform}:\s*{re.escape(handle)}",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE
-        )
-    
-    # Extract and remove Working Areas in LA
-    working_match = re.search(
-        r"Working Areas in LA:\s*(.*?)\s+(?=Summary:|Ranking:|Impact Metrics:|$)",
-        cleaned_text,
-        re.IGNORECASE | re.DOTALL
-    )
-    if working_match:
-        metadata["working_areas"] = working_match.group(1).strip()
-        cleaned_text = re.sub(
-            r"Working Areas in LA:\s*.*?(?=Summary:|Ranking:|Impact Metrics:|$)",
-            "",
-            cleaned_text,
-            flags=re.IGNORECASE | re.DOTALL
-        )
     
-    # Extract and remove Zipcode (assuming 5-digit US zipcodes)
-    zipcode_match = re.search(r"Zipcode:\s*(\d{5})", cleaned_text, re.IGNORECASE)
-    if zipcode_match:
-        metadata["zipcode"] = zipcode_match.group(1).strip()
-        cleaned_text = re.sub(r"Zipcode:\s*\d{5}", "", cleaned_text, flags=re.IGNORECASE)
-    
-    # Clean up extra whitespace
-    cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip()
-    
-    # Create a metadata summary to append to the cleaned text.
-    meta_summary = ""
-    if "year" in metadata:
-        meta_summary += f"Year: {metadata['year']}. "
-    if "ranking" in metadata:
-        meta_summary += f"Ranking: {metadata['ranking']}. "
-    if "organization" in metadata:
-        meta_summary += f"Organization: {metadata['organization']}. "
-    if "working_areas" in metadata:
-        meta_summary += f"Working Areas in LA: {metadata['working_areas']}. "
-    if "zipcode" in metadata:
-        meta_summary += f"Zipcode: {metadata['zipcode']}. "
-    
-    combined_text = meta_summary + "\n" + cleaned_text if meta_summary else cleaned_text
-    
-    return metadata, combined_text
+    return metadata
 
 
 def load_and_process_data(file_path: str):
-    """
-    Loads JSON data from a file, extracts organization text and metadata (including working areas and zipcode),
-    cleans the text by removing redundant metadata, and returns a list of Documents.
-    Documents with a "winner" ranking are inserted at the beginning of the list.
-    """
     try:
         data = json.loads(Path(file_path).read_text(encoding='utf-8'))
         docs = []
@@ -175,16 +97,18 @@ def load_and_process_data(file_path: str):
             org_text = entry.get("OrganizationText", "")
             if not org_text:
                 continue
-            metadata, combined_text = extract_metadata(org_text)
+            metadata = extract_metadata(org_text)
+            # Optionally, prioritize winners
             if metadata.get("ranking", "").lower() == "winner":
-                docs.insert(0, Document(page_content=combined_text, metadata=metadata))
+                docs.insert(0, Document(page_content=org_text, metadata=metadata))
             else:
-                docs.append(Document(page_content=combined_text, metadata=metadata))
+                docs.append(Document(page_content=org_text, metadata=metadata))
         return docs
     except Exception as e:
         print(f"Error loading JSON: {e}")
         return []
 
+
 # -------------------------------
 # Data Loading and Preprocessing
 # -------------------------------