refactor UI and embedding of URL's

.claude/settings.local.json

@@ -3,7 +3,14 @@
     "allow": [
       "Bash(ls -la \"D:\\\\MAPS Lab\\\\Others\\\\AIVIZ-BOT\\\\processing\"\" && ls -la \"D:MAPS LabOthersAIVIZ-BOTllm_setup\"\")",
       "Bash(git add:*)",
-      "Bash(git commit:*)"
+      "Bash(git commit:*)",
+      "Bash(python3:*)",
+      "Bash(pip3 show:*)",
+      "Bash(pip show:*)",
+      "WebFetch(domain:www.gradio.app)",
+      "WebFetch(domain:github.com)",
+      "Bash(.venv/bin/pip install:*)",
+      "Bash(python -c:*)"
     ]
   }
 }

app.py

@@ -46,13 +46,180 @@ def echo(text, chat_history, request: gr.Request):
 def on_reset_button_click():
     llm_svc.store=LFUCache(capacity=50)
 
+# --- Maritime Theme ---
+maritime_blue = gr.themes.Color(
+    c50="#f0f9ff", c100="#e0f2fe", c200="#b9e6fe", c300="#7dd4fc",
+    c400="#38bdf8", c500="#0ea5e9", c600="#0284c7", c700="#0369a1",
+    c800="#075985", c900="#0c4a6e", c950="#082f49",
+    name="maritime-blue",
+)
+
+teal_accent = gr.themes.Color(
+    c50="#f0fdfa", c100="#ccfbf1", c200="#99f6e4", c300="#5eead4",
+    c400="#2dd4bf", c500="#14b8a6", c600="#0d9488", c700="#0f766e",
+    c800="#115e59", c900="#134e4a", c950="#042f2e",
+    name="teal-accent",
+)
+
+try:
+    stormy_theme = gr.themes.Ocean(
+        primary_hue=maritime_blue,
+        secondary_hue=teal_accent,
+        neutral_hue="slate",
+        spacing_size=gr.themes.sizes.spacing_md,
+        radius_size=gr.themes.sizes.radius_lg,
+        text_size=gr.themes.sizes.text_md,
+        font=(gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"),
+        font_mono=(gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "Consolas", "monospace"),
+    )
+except AttributeError:
+    stormy_theme = gr.themes.Soft(
+        primary_hue=maritime_blue,
+        secondary_hue=teal_accent,
+        neutral_hue="slate",
+        spacing_size=gr.themes.sizes.spacing_md,
+        radius_size=gr.themes.sizes.radius_lg,
+        text_size=gr.themes.sizes.text_md,
+        font=(gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"),
+        font_mono=(gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "Consolas", "monospace"),
+    )
+
+stormy_theme = stormy_theme.set(
+    body_background_fill="#f0f9ff",
+    body_background_fill_dark="#0c1929",
+    body_text_color="#0c4a6e",
+    body_text_color_dark="#e0f2fe",
+    block_background_fill="#ffffff",
+    block_background_fill_dark="#0f2942",
+    block_border_color="#b9e6fe",
+    block_border_color_dark="#0369a1",
+    button_primary_background_fill="linear-gradient(135deg, #0ea5e9, #0d9488)",
+    button_primary_background_fill_hover="linear-gradient(135deg, #38bdf8, #14b8a6)",
+    button_primary_background_fill_dark="linear-gradient(135deg, #0369a1, #0f766e)",
+    button_primary_text_color="#ffffff",
+    button_secondary_background_fill="#e0f2fe",
+    button_secondary_background_fill_hover="#b9e6fe",
+    button_secondary_background_fill_dark="#0f2942",
+    button_secondary_text_color="#0c4a6e",
+    button_secondary_text_color_dark="#7dd4fc",
+    input_background_fill="#f8fafc",
+    input_background_fill_dark="#0f2942",
+    input_border_color="#b9e6fe",
+    input_border_color_focus="#0ea5e9",
+    input_border_color_dark="#0369a1",
+    shadow_drop="0 2px 8px rgba(14, 165, 233, 0.08)",
+    shadow_drop_lg="0 4px 16px rgba(14, 165, 233, 0.12)",
+)
+
+custom_css = """
+.stormy-header {
+    text-align: center;
+    padding: 1.5rem 1rem 1rem 1rem;
+    background: linear-gradient(135deg, #0c4a6e 0%, #0ea5e9 50%, #0d9488 100%);
+    border-radius: 12px;
+    margin-bottom: 0.5rem;
+    color: white;
+}
+.stormy-header h1 {
+    font-size: 1.8rem;
+    margin: 0 0 0.25rem 0;
+    font-weight: 700;
+    color: #ffffff !important;
+}
+.stormy-header p {
+    font-size: 0.95rem;
+    margin: 0;
+    color: #e0f2fe !important;
+    opacity: 0.9;
+}
+.reset-btn {
+    max-width: 200px !important;
+}
+.stormy-footer {
+    text-align: center;
+    font-size: 0.8rem;
+    color: #64748b;
+    padding-top: 0.5rem;
+}
+"""
+
 if __name__ == '__main__':
     logging.info("Starting AIVIz Bot")
 
-    with gr.Blocks() as demo:
-        gr.ChatInterface(fn=echo, type="messages")
-        reset_button = gr.Button("Reset Chat Memory Cache")             
-        reset_button.click(on_reset_button_click)
+    with gr.Blocks(theme=stormy_theme, css=custom_css, title="Stormy - AISdb Assistant") as demo:
+
+        # Branding Header
+        gr.Markdown(
+            """
+            <div class="stormy-header">
+                <h1>Stormy - AISdb Assistant</h1>
+                <p>Your maritime data companion. Ask about AIS vessel tracking, data processing, machine learning, and more.</p>
+            </div>
+            """,
+            elem_id="header",
+        )
+
+        # Chat Interface
+        chatbot = gr.Chatbot(
+            placeholder=(
+                "<strong>Welcome aboard!</strong><br>"
+                "I'm Stormy, your AISdb documentation assistant.<br>"
+                "Ask me about vessel tracking, data queries, or machine learning with AIS data."
+            ),
+            height=500,
+            type="messages",
+            show_copy_button=True,
+        )
+
+        gr.ChatInterface(
+            fn=echo,
+            type="messages",
+            chatbot=chatbot,
+            textbox=gr.Textbox(
+                placeholder="Ask Stormy about AISdb...",
+                container=False,
+                scale=7,
+            ),
+            examples=[
+                "How do I get started with AISdb?",
+                "How can I query vessel tracks by MMSI?",
+                "What machine learning models work with AIS data?",
+                "How do I visualize ship trajectories on a map?",
+            ],
+        )
+
+        # Action Bar
+        with gr.Row():
+            with gr.Column(scale=3):
+                with gr.Accordion("About Stormy & AISdb", open=False):
+                    gr.Markdown(
+                        """
+                        **Stormy** is an AI assistant built on the AISdb (Automatic Identification System Database)
+                        documentation. It can help you with:
+
+                        - **Data Access**: Loading AIS data, creating databases, CSV export
+                        - **Querying**: SQL queries, filtering by MMSI, time ranges, geographic areas
+                        - **Processing**: Data cleaning, track interpolation, decimation
+                        - **Visualization**: Plotting vessel trajectories, hexagon discretization
+                        - **Machine Learning**: Seq2Seq models, autoencoders for AIS data
+                        - **Geospatial**: Haversine distance, shore distance, bathymetric data
+
+                        Powered by AISdb documentation from [aisviz.gitbook.io](https://aisviz.gitbook.io/documentation)
+                        and [MAPS Lab](https://mapslab.tech/).
+                        """
+                    )
+            with gr.Column(scale=1, min_width=200):
+                reset_button = gr.Button(
+                    "Reset Chat Memory",
+                    variant="secondary",
+                    size="sm",
+                    elem_classes=["reset-btn"],
+                )
+                reset_button.click(on_reset_button_click)
+
+        # Footer
+        gr.Markdown(
+            '<div class="stormy-footer">Built with Gradio & LangChain | AISdb Documentation Assistant</div>'
+        )
 
-    # Launch the interface
     demo.launch()

configs/config.py

@@ -68,7 +68,6 @@ URLS = ["https://aisviz.gitbook.io/documentation",
         ]
 CHUNK_SIZE = 768
 CHUNK_OVERLAP = 100
-TOTAL_RESULTS = 2389
 MAX_SIZE = 100
 EMBEDDINGS = HuggingFaceEmbeddings(
     model_name="sentence-transformers/all-mpnet-base-v2",

processing/documents.py

@@ -31,15 +31,21 @@ def format_documents(docs: list[Document]) -> str:
     return "\n\n".join(doc.page_content for doc in docs)
 
 
-def split_documents(documents: Iterable[Document]) -> list[Document]:
+def split_documents(documents: Iterable[Document], chunk_size: int = 768, chunk_overlap: int = 100) -> list[Document]:
     """
     Splits documents into smaller chunks.
 
     Args:
         documents (Iterable[Document]): The documents to split.
+        chunk_size (int): Maximum size of each chunk in characters.
+        chunk_overlap (int): Number of overlapping characters between chunks.
 
     Returns:
         list[Document]: A list of split documents.
     """
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
     return text_splitter.split_documents(documents)

processing/texts.py

@@ -1,6 +1,12 @@
+import re
+
+
 def clean_text(text: str) -> str:
     """
-    Clean the text by removing unwanted characters and formatting.
+    Clean the text by removing unwanted characters while preserving document structure.
     """
-    cleaned_text = text.replace("\n", " ").strip()
-    return cleaned_text
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)  # remove control chars (keep \n \t \r)
+    text = re.sub(r'\r\n?', '\n', text)           # normalize line endings
+    text = re.sub(r'\n{3,}', '\n\n', text)         # collapse excessive newlines
+    text = re.sub(r'[^\S\n]{3,}', ' ', text)       # collapse excessive spaces (not newlines)
+    return text.strip()

services/scraper.py

@@ -1,6 +1,7 @@
 from langchain.schema import Document
 
-from processing.documents import load_documents, format_documents, split_documents
+import configs.config as config
+from processing.documents import load_documents, split_documents
 from processing.texts import clean_text
 
 
@@ -17,10 +18,13 @@ class Service:
         for url in urls:
             try:
                 website_documents = load_documents(url)
-                formatted_content = format_documents(website_documents)
-                cleaned_content = clean_text(formatted_content)
-                documents.append(Document(page_content=cleaned_content))
+                for doc in website_documents:
+                    doc.page_content = clean_text(doc.page_content)
+                    doc.metadata["source"] = url
+                    documents.append(doc)
             except Exception as e:
                 raise Exception(f"Error processing {url}: {e}")
 
-        self.store.store_embeddings(split_documents(documents))
+        self.store.store_embeddings(
+            split_documents(documents, chunk_size=config.CHUNK_SIZE, chunk_overlap=config.CHUNK_OVERLAP)
+        )

stores/chroma.py

@@ -15,6 +15,9 @@ class ChromaDB:
     def store_embeddings(self, documents: list[Document]):
         """
         Store embeddings for the documents using HuggingFace embeddings and Chroma vectorstore.
+        Skips ingestion if the collection is already populated.
         """
+        if self.chroma._collection.count() > 0:
+            return
         self.chroma.add_documents(documents=documents, embeddings=self.embeddings,
                                   persist_directory=self._persistent_directory)