make hyperlink text webpage title, not URL text

ingestion.py

@@ -3,13 +3,11 @@ Document ingestion module for processing PDFs and URLs.
 """
 import os
 from typing import List, Dict
-from pathlib import Path
 import requests
 from bs4 import BeautifulSoup
 from pypdf import PdfReader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from sentence_transformers import SentenceTransformer
-import numpy as np
 import faiss
 import pickle
 
@@ -77,15 +75,15 @@ class DocumentIngestion:
         except Exception as e:
             raise Exception(f"Error reading PDF {file_path}: {str(e)}")
     
-    def read_url(self, url: str) -> str:
+    def read_url(self, url: str):
         """
-        Extract text from a URL.
-        
+        Extract text and page title from a URL.
+
         Args:
             url: URL to fetch and extract text from
-            
+
         Returns:
-            Extracted text content
+            Tuple of (text content, page title or None)
         """
         try:
             headers = {
@@ -93,22 +91,27 @@ class DocumentIngestion:
             }
             response = requests.get(url, headers=headers, timeout=10)
             response.raise_for_status()
-            
+
             soup = BeautifulSoup(response.content, 'html.parser')
-            
+
+            # Extract page title before stripping elements
+            page_title = None
+            if soup.title and soup.title.string:
+                page_title = soup.title.string.strip()
+
             # Remove script and style elements
             for script in soup(["script", "style"]):
                 script.decompose()
-            
+
             # Get text
             text = soup.get_text()
-            
+
             # Clean up whitespace
             lines = (line.strip() for line in text.splitlines())
             chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
             text = ' '.join(chunk for chunk in chunks if chunk)
-            
-            return text
+
+            return text, page_title
         except Exception as e:
             raise Exception(f"Error reading URL {url}: {str(e)}")
     
@@ -148,17 +151,20 @@ class DocumentIngestion:
         if urls:
             for url in urls:
                 try:
-                    text = self.read_url(url)
+                    text, page_title = self.read_url(url)
                     chunks = self.text_splitter.split_text(text)
-                    # Use a short label for URL (domain + path hint) as document_title
-                    try:
-                        from urllib.parse import urlparse
-                        parsed = urlparse(url)
-                        document_title = parsed.netloc or url
-                        if parsed.path and parsed.path != "/":
-                            document_title += " " + parsed.path.strip("/")[:50]
-                    except Exception:
-                        document_title = url
+                    # Use the page's <title> tag if available, otherwise fall back to domain + path
+                    if page_title:
+                        document_title = page_title
+                    else:
+                        try:
+                            from urllib.parse import urlparse
+                            parsed = urlparse(url)
+                            document_title = parsed.netloc or url
+                            if parsed.path and parsed.path != "/":
+                                document_title += " " + parsed.path.strip("/")[:50]
+                        except Exception:
+                            document_title = url
                     for i, chunk in enumerate(chunks):
                         all_texts.append(chunk)
                         all_metadata.append({