Upload 90 files

.dockerignore

@@ -0,0 +1,60 @@
+# Git files
+.git/
+.gitignore
+
+# Python cache and compiled files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+env/
+venv/
+.venv/
+
+# IDE and editor specific files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Secrets or local configuration (should be passed via environment variables)
+.env
+secrets/
+*.pem
+*.key
+
+# Log files
+*.log
+tensorus_audit.log # Specifically exclude this if it's generated locally
+
+# Test files and reports (unless you want to run tests in the image)
+tests/
+htmlcov/
+.pytest_cache/
+.coverage
+
+# OS-specific files
+.DS_Store
+Thumbs.db
+
+# Docker files themselves (no need to copy into the image)
+Dockerfile
+docker-compose.yml
+
+# Other project specific files/folders to exclude
+# E.g., notebooks/, docs/, etc.
+# data/ # If you have a local data folder not needed in the image
+# setup.sh # Exclude if not part of the Python API image
+# DEMO.md # Exclude if not part of the Python API image
+# LICENSE # Usually good to include, but can be excluded if desired
+# README.md # Usually good to include, but can be excluded if desired
+# pages/ # If these are Streamlit pages not served by this Docker image.
+
+# If `app.py` at root is a Streamlit launcher, and not part of this specific API service, exclude it too.
+# Assuming `app.py` from the root is not part of this specific service being Dockerized.
+app.py

.gitignore

@@ -0,0 +1,175 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+

DEMO.md

@@ -0,0 +1,203 @@
+# Tensorus Demo Script: Showcasing Agentic Tensor Management
+
+## Introduction
+
+This demo showcases the key capabilities of Tensorus, an agentic tensor database/data lake. We'll walk through data ingestion, storage, querying, tensor operations, and how to interact with the system via its UI and APIs.
+
+## Prerequisites
+
+*   Tensorus backend API running (`uvicorn api:app --reload --host 127.0.0.1 --port 7860`)
+*   Tensorus Streamlit UI running (`streamlit run app.py`)
+*   Tensorus MCP Server running (`python -m tensorus.mcp_server`)
+*   A terminal for API calls (e.g., using `curl`) or a tool like Postman.
+*   A web browser.
+
+## Demo Scenarios
+
+### Scenario 1: Automated Image Data Ingestion and Exploration
+
+**Goal:** Demonstrate how new image data is automatically ingested, stored, and can be explored.
+
+1.  **Prepare for Ingestion:**
+    *   Open the Streamlit UI in your browser (usually `http://localhost:8501`).
+    *   Navigate to the "Agents" or "Control Panel" page.
+    *   Ensure the "Data Ingestion Agent" is targeting the dataset `ingested_data_api` and monitoring the directory `temp_ingestion_source_api`. (You might need to configure this in `tensorus/api.py` if not dynamically configurable via UI).
+    *   Start the "Data Ingestion Agent" if it's not already running. View its logs in the UI to see it polling.
+    *   Verify the `temp_ingestion_source_api` directory (relative to where the API is run) is initially empty or clean it out.
+
+2.  **Simulate New Data Arrival:**
+    *   Download a sample image (e.g., a picture of a cat, `cat.jpg`) into the `temp_ingestion_source_api` directory on your local filesystem.
+    *   **Attractive Element:** Use a visually distinct and interesting image.
+
+3.  **Observe Ingestion:**
+    *   In the Streamlit UI, watch the Ingestion Agent's logs. You should see messages indicating it detected `cat.jpg`, processed it, and ingested it into `ingested_data_api`.
+
+4.  **Explore Ingested Data (UI):**
+    *   Navigate to the "Data Explorer" page in the Streamlit UI.
+    *   Select the `ingested_data_api` dataset from the dropdown.
+    *   You should see the newly ingested `cat.jpg` (or its metadata) listed.
+    *   Click to view its details (metadata, tensor shape, perhaps a preview if the UI supports it).
+    *   **Attractive Element:** The UI showing the image tensor's information clearly, perhaps even a thumbnail if the UI is advanced.
+
+5.  **Verify via API (Optional):**
+    *   Use `curl` or Postman to fetch the tensor details. First, list datasets to find `ingested_data_api`, then fetch its records, identify the `record_id` for `cat.jpg` from its metadata.
+        ```bash
+        # Example: List records in the dataset to find the ID (paged)
+        curl "http://127.0.0.1:7860/datasets/ingested_data_api/records?offset=0&limit=100"
+        # Then use the ID:
+        # curl http://127.0.0.1:7860/datasets/ingested_data_api/tensors/{record_id_of_cat_jpg}
+        ```
+    *   Confirm the API returns the tensor data and metadata.
+
+### Scenario 2: Natural Language Querying for Specific Data
+
+**Goal:** Show how NQL can be used to find specific tensors.
+
+1.  **Ensure Data Exists:**
+    *   Make sure the `cat.jpg` from Scenario 1 is present in `ingested_data_api`.
+    *   Optionally, add another image, e.g., `dog.png`, to `temp_ingestion_source_api` and let it be ingested. Give it distinctive metadata if possible, e.g., by modifying the `ingestion_agent.py` to add `{"animal_type": "dog"}` or by using the API to update metadata post-ingestion.
+
+2.  **Use NQL Chatbot (UI):**
+    *   Navigate to the "NQL Chatbot" or "Query Hub" page in the Streamlit UI.
+    *   Enter a query. Given the basic NQL, a query targeting the filename is most reliable:
+        `show all data from ingested_data_api where source_file contains "cat.jpg"`
+    *   If you added custom metadata like `{"animal_type": "cat"}` for the cat image, you could try:
+        `find records from ingested_data_api where animal_type = 'cat'`
+    *   **Attractive Element:** The chatbot interface and the directness of the natural language query yielding correct results.
+
+3.  **Observe Results:**
+    *   The UI should display the tensor(s) matching your query.
+
+### Scenario 3: Performing Tensor Operations via API
+
+**Goal:** Demonstrate applying a tensor operation to a stored tensor.
+
+1.  **Identify a Target Tensor:**
+    *   From Scenario 1 or 2, obtain the `record_id` of the `cat.jpg` tensor within the `ingested_data_api` dataset. Let's assume its `record_id` is `xyz123`.
+
+2.  **Perform a Tensor Operation (e.g., Transpose API):**
+    *   Image tensors are often (C, H, W) or (H, W, C). Let's assume it's (C, H, W) and we want to transpose H and W, which would be `dim0=1, dim1=2`.
+    *   Use `curl` or Postman:
+        ```bash
+        curl -X POST -H "Content-Type: application/json" -d '{
+          "input_tensor": {
+            "dataset_name": "ingested_data_api",
+            "record_id": "xyz123" 
+          },
+          "params": {
+            "dim0": 1, 
+            "dim1": 2  
+          },
+          "output_dataset_name": "ops_results",
+          "output_metadata": {"original_id": "xyz123", "operation": "transpose_height_width_demo"}
+        }' http://127.0.0.1:7860/ops/transpose
+        ```
+    *   **Attractive Element:** Showing the API call and the structured JSON response indicating success and the new transposed tensor's ID and details.
+
+3.  **Verify Result:**
+    *   The API response will contain the `record_id` and details of the new (transposed) tensor in the `ops_results` dataset.
+    *   Note the new shape in the response. If the original was (3, 128, 128), the new shape should be (3, 128, 128) after transposing height and width (assuming they were the same). If they were different, e.g. (3, 128, 200), the new shape would be (3, 200, 128).
+    *   Optionally, use the "Data Explorer" in the UI or another API call to fetch and inspect this new tensor.
+
+### Scenario 4: Interacting with the MCP Server (Conceptual)
+
+**Goal:** Explain how an external AI agent could leverage Tensorus via MCP.
+
+1.  **Show MCP Server Running:**
+    *   Point to the terminal where `python -m tensorus.mcp_server` is running and show its log output (e.g., "Tensorus MCP Server connected via stdio and ready.").
+
+2.  **Explain Available Tools (Conceptual):**
+    *   Briefly show the tool definitions in `tensorus/mcp_server.py` or refer to the README's "Available Tools" under "MCP Server Details".
+    *   Highlight a few tools like `tensorus_list_datasets`, `tensorus_ingest_tensor`, and `tensorus_apply_binary_operation`.
+
+3.  **Conceptual Client Interaction (Show code snippet from README):**
+    *   Show the example client-side JavaScript snippet from the `README.md`:
+        ```javascript
+        // Conceptual MCP client-side JavaScript
+        // Assuming 'client' is an initialized MCP client connected to the Tensorus MCP Server
+
+        async function example() {
+          // List available tools
+          const { tools } = await client.request({ method: 'tools/list' }, {});
+          console.log("Available Tensorus Tools:", tools.map(t => t.name));
+
+          // Create a new dataset
+          const createResponse = await client.request({ method: 'tools/call' }, {
+            name: 'tensorus_create_dataset',
+            arguments: { dataset_name: 'my_mcp_dataset_demo' }
+          });
+          console.log(JSON.parse(createResponse.content[0].text).message); // MCP server often returns JSON string in text
+        }
+        ```
+    *   **Attractive Element:** Emphasize that this allows *other* AI agents or LLMs to programmatically use Tensorus as a modular component in a larger intelligent system, promoting interoperability.
+
+### Scenario 5: Financial Time Series Forecasting with ARIMA
+
+**Goal:** Demonstrate end-to-end time series forecasting using generated financial data, Tensorus for storage, an ARIMA model for prediction, and visualization within a dedicated UI page.
+
+**Prerequisites Specific to this Demo:**
+*   Ensure `statsmodels` is installed. If you used the standard setup, install it via the optional models extras:
+    ```bash
+    pip install -e .[models]
+    ```
+    or install the `tensorus-models` package which includes it.
+
+**Steps:**
+
+1.  **Navigate to the Demo Page:**
+    *   Open the Streamlit UI (e.g., `http://localhost:8501`).
+    *   From the top navigation bar (or sidebar if the UI structure varies), find and click on the "Financial Forecast Demo" page (it might be titled "📈 Financial Forecast Demo" or similar).
+
+2.  **Generate and Ingest Data:**
+    *   On the "Financial Forecast Demo" page, locate the section "1. Data Generation & Ingestion."
+    *   Click the button labeled **"Generate & Ingest Sample Financial Data"**.
+    *   Wait for the spinner to complete. You should see:
+        *   A success message indicating data ingestion into a dataset like `financial_raw_data` in Tensorus.
+        *   A sample DataFrame (head) of the generated data (Date, Close, Volume).
+        *   A Plotly chart displaying the historical 'Close' prices that were just generated and ingested.
+    *   **Attractive Element:** Observe the immediate visualization of the generated time series.
+
+3.  **Configure and Run ARIMA Prediction:**
+    *   Go to the "3. ARIMA Model Prediction" section on the page.
+    *   You can adjust the ARIMA order (p, d, q) and the number of future predictions if you wish. Default values (e.g., p=5, d=1, q=0, predictions=30) are provided.
+    *   Click the button labeled **"Run ARIMA Prediction"**.
+    *   Wait for the spinner to complete. This step involves:
+        *   Loading the historical data from Tensorus.
+        *   Training/fitting the ARIMA model.
+        *   Generating future predictions.
+        *   Storing these predictions back into Tensorus (e.g., into `financial_predictions` dataset).
+
+4.  **View Prediction Results:**
+    *   Once the prediction is complete, scroll to the "4. Prediction Results" section.
+    *   You should see:
+        *   A Plotly chart displaying the original historical data with the ARIMA predictions plotted alongside/extending from it.
+        *   A table or list showing the actual predicted values for future dates.
+    *   **Attractive Element:** The clear visual comparison of the forecast against the historical data, showcasing the model's predictive attempt. The interactivity of Plotly charts (zoom, pan) enhances this.
+
+5.  **Interpretation (What this demonstrates):**
+    *   **Data Flow:** Generation -> Tensorus Storage (Raw Data) -> Retrieval for Modeling -> Prediction -> Tensorus Storage (Predictions) -> UI Visualization.
+    *   **Ease of Use:** A user-friendly interface to perform a complex task like time series forecasting.
+    *   **Modularity:** Integration of data generation, storage (Tensorus), modeling (statsmodels), and UI (Streamlit) components.
+    *   **Revised UI:** Notice the potentially improved layout and charting capabilities on this dedicated demo page.
+
+### Scenario 6: Dashboard Overview
+
+**Goal:** Show the main dashboard providing a system overview.
+
+1.  **Navigate to Dashboard:**
+    *   In the Streamlit UI, go to the main "Nexus Dashboard" page (usually the default page when you run `streamlit run app.py`).
+2.  **Review Metrics:**
+    *   Point out the key metrics displayed:
+        *   Total Tensors / Active Datasets (these might be placeholders or simple counts).
+        *   Agents Online / Status (showing the Ingestion Agent as 'running' if you started it).
+        *   API Status (should be 'Connected').
+        *   Simulated metrics like data ingestion rate, query latency, RL rewards, AutoML progress. Explain these are illustrative.
+    *   **Attractive Element:** A visually appealing dashboard. If any metrics are updating (even if simulated based on time), it adds to the dynamic feel.
+
+3.  **Activity Feed (if populated):**
+    *   Show the "Recent Agent Activity" feed. If the Ingestion Agent is running, it might populate this feed, or it might be placeholder data.
+    *   Explain that in a fully operational system, this feed would show real-time updates from all active agents.
+
+## Conclusion
+
+This demo has provided a glimpse into Tensorus's capabilities, including automated data handling by agents, flexible data storage, natural language querying, powerful tensor operations, and a user-friendly interface. The MCP server further extends its reach, allowing programmatic interaction from other AI systems, paving the way for more complex, collaborative AI workflows.

Dockerfile

@@ -0,0 +1,54 @@
+# Use an official Python runtime as a parent image
+FROM python:3.11-slim
+
+# Set environment variables for Python
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies that might be needed by Python packages
+# Example: build-essential for some packages, libpq-dev for psycopg2 from source (though -binary avoids this)
+# For psycopg2-binary, typically no extra system deps are needed for common platforms if using a compatible wheel.
+# RUN apt-get update && apt-get install -y --no-install-recommends build-essential libpq-dev && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file into the container
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application source code into the container
+# This structure assumes your main application code is within the 'tensorus' directory.
+COPY ./tensorus ./tensorus
+# If your app.py or main.py is at the root alongside Dockerfile, you'd copy it too:
+# COPY app.py . # Or specific main file if it's at the root.
+# Based on `CMD ["uvicorn", "tensorus.api.main:app"...]`, main:app is in tensorus/api/main.py.
+
+# Set default environment variables for the application
+# These can be overridden when running the container (e.g., via docker run -e or docker-compose.yml)
+ENV TENSORUS_STORAGE_BACKEND="in_memory"
+ENV TENSORUS_POSTGRES_HOST="db" # Default for docker-compose setup
+ENV TENSORUS_POSTGRES_PORT="5432"
+ENV TENSORUS_POSTGRES_USER="tensorus_user_dockerfile"
+ENV TENSORUS_POSTGRES_PASSWORD="tensorus_pass_dockerfile"
+ENV TENSORUS_POSTGRES_DB="tensorus_db_dockerfile"
+ENV TENSORUS_POSTGRES_DSN=""
+ENV TENSORUS_API_KEY_HEADER_NAME="X-API-KEY"
+ENV TENSORUS_VALID_API_KEYS="[]" # Example: "key1,key2" or "[\"key1\", \"key2\"]" - set to [] if no keys required
+ENV TENSORUS_AUTH_JWT_ENABLED="False" # Default JWT auth to disabled
+ENV TENSORUS_AUTH_JWT_ISSUER=""
+ENV TENSORUS_AUTH_JWT_AUDIENCE=""
+ENV TENSORUS_AUTH_JWT_ALGORITHM="RS256"
+ENV TENSORUS_AUTH_JWT_JWKS_URI=""
+ENV TENSORUS_AUTH_DEV_MODE_ALLOW_DUMMY_JWT="False"
+
+# Expose the port the app runs on
+EXPOSE 7860
+
+# Define the command to run the application
+# Use the package export from ``tensorus.api`` which exposes ``app`` in
+# ``tensorus/api/__init__.py``. This lets Uvicorn import the app directly
+# from the package without referencing the module path.
+CMD ["uvicorn", "tensorus.api:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Tensorus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,683 @@
 ---
-title: Mcp
-emoji: 🐠
-colorFrom: gray
-colorTo: purple
-sdk: docker
-pinned: false
 license: mit
-short_description: Tensorus MCP server
+title: Core
+sdk: docker
+emoji: 🐠
+colorFrom: blue
+colorTo: yellow
+short_description: Tensorus Core
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Tensorus: Agentic Tensor Database/Data Lake
+
+Tensorus is a specialized data platform focused on the management and agent-driven manipulation of tensor data. It offers a streamlined environment for storing, retrieving, and operating on tensors, laying the groundwork for advanced AI and machine learning workflows.
+
+The core purpose of Tensorus is to simplify and enhance how developers and AI agents interact with tensor datasets. By providing dedicated tools for tensor operations and a framework for agentic integration, Tensorus aims to accelerate tasks like automated data ingestion, reinforcement learning from stored experiences, and AutoML processes, ultimately enabling more efficient and intelligent data utilization in AI projects.
+
+## Key Features
+
+*   **Tensor Storage:** Efficiently store and retrieve PyTorch tensors with associated metadata.
+*   **Dataset Schemas:** Optional per-dataset schemas enforce required metadata fields and tensor shape/dtype.
+*   **Natural Query Language (NQL):** Query your tensor data using a simple, natural language-like syntax.
+*   **Agent Framework:** A foundation for building and integrating intelligent agents that interact with your data.
+    *   **Data Ingestion Agent:** Automatically monitors a directory for new files and ingests them as tensors.
+    *   **RL Agent:** A Deep Q-Network (DQN) agent that can learn from experiences stored in TensorStorage.
+    *   **AutoML Agent:** Performs hyperparameter optimization for a dummy model using random search.
+*   **API-Driven:** A FastAPI backend provides a RESTful API for interacting with Tensorus.
+*   **Streamlit UI:** A user-friendly Streamlit frontend for exploring data and controlling agents.
+*   **Tensor Operations:** A comprehensive library of robust tensor operations for common manipulations. See [Basic Tensor Operations](#basic-tensor-operations) for details.
+*   **Model System:** Optional model registry with example models provided in a
+    separate package. See [Tensorus Models](https://github.com/tensorus/models).
+*   **Metadata System:** Rich Pydantic schemas and storage backends for semantic, lineage, computational, quality, relational, and usage metadata.
+*   **Extensible:** Designed to be extended with more advanced agents, storage backends, and query capabilities.
+*   **Model Context Protocol (MCP) Server:** Provides a standardized interface for AI agents and LLMs to interact with Tensorus capabilities—including dataset management, tensor storage, and operations—using the Model Context Protocol. (See [MCP Server Details](#mcp-server-details) below).
+
+## Project Structure
+
+*   `app.py`: The main Streamlit frontend application (located at the project root).
+*   `pages/`: Directory containing individual Streamlit page scripts and shared UI utilities for the dashboard.
+    *   `pages/ui_utils.py`: Utility functions specifically for the Streamlit UI.
+    *   *(Other page scripts like `01_dashboard.py`, `02_control_panel.py`, etc., define the different views of the dashboard)*
+*   `tensorus/`: Directory containing the core `tensorus` library modules (this is the main installable package).
+    *   `tensorus/__init__.py`: Makes `tensorus` a Python package.
+    *   `tensorus/api.py`: The FastAPI application providing the backend API for Tensorus.
+    *   `tensorus/tensor_storage.py`: Core TensorStorage implementation for managing tensor data.
+    *   `tensorus/tensor_ops.py`: Library of functions for tensor manipulations.
+    *   `tensorus/nql_agent.py`: Agent for processing Natural Query Language queries.
+    *   `tensorus/ingestion_agent.py`: Agent for ingesting data from various sources.
+    *   `tensorus/rl_agent.py`: Agent for Reinforcement Learning tasks.
+    *   `tensorus/automl_agent.py`: Agent for AutoML processes.
+    *   `tensorus/dummy_env.py`: A simple environment for the RL agent demonstration.
+    *   *(Other Python files within `tensorus/` are part of the core library.)*
+*   `requirements.txt`: Lists the project's Python dependencies for development and local execution.
+*   `pyproject.toml`: Project metadata, dependencies for distribution, and build system configuration (e.g., for PyPI).
+*   `tensorus/mcp_server.py`: Python implementation of the Model Context Protocol (MCP) server using `fastmcp`.
+*   `README.md`: This file.
+*   `LICENSE`: Project license file.
+*   `.gitignore`: Specifies intentionally untracked files that Git should ignore.
+
+## Huggingface Demo
+
+You can try Tensorus online via Huggingface Spaces:
+
+*   **API Documentation:** [Swagger UI](https://tensorus-api.hf.space/docs) | [ReDoc](https://tensorus-api.hf.space/redoc)
+*   **Dashboard UI:** [Streamlit Dashboard](https://tensorus-dashboard.hf.space)
+
+## Tensorus Execution Cycle
+
+```mermaid
+graph TD
+    %% User Interface Layer
+    subgraph UI_Layer ["User Interaction"]
+        UI[Streamlit UI]
+    end
+
+    %% API Gateway Layer
+    subgraph API_Layer ["Backend Services"]
+        API[FastAPI Backend]
+        MCP["MCP Server (FastMCP Python)"]
+    end
+
+    %% Core Storage with Method Interface
+    subgraph Storage_Layer ["Core Storage - TensorStorage"]
+        TS[TensorStorage Core]
+        subgraph Storage_Methods ["Storage Interface"]
+            TS_insert[insert data metadata]
+            TS_query[query query_fn]
+            TS_get[get_by_id id]
+            TS_sample[sample n]
+            TS_update[update_metadata]
+        end
+        TS --- Storage_Methods
+    end
+
+    %% Agent Processing Layer
+    subgraph Agent_Layer ["Processing Agents"]
+        IA[Ingestion Agent]
+        NQLA[NQL Agent]
+        RLA[RL Agent]
+        AutoMLA[AutoML Agent]
+    end
+
+    %% Model System
+    subgraph Model_Layer ["Model System"]
+        Registry[Model Registry]
+        ModelsPkg[Models Package]
+    end
+
+    %% Tensor Operations Library
+    subgraph Ops_Layer ["Tensor Operations"]
+        TOps[TensorOps Library]
+    end
+
+    %% Primary UI & MCP Flow
+    UI -->|HTTP Requests| API
+    MCP -->|MCP Calls| API
+
+    %% API Orchestration
+    API -->|Command Dispatch| IA
+    API -->|Command Dispatch| NQLA
+    API -->|Command Dispatch| RLA
+    API -->|Command Dispatch| AutoMLA
+    API -->|Model Training| Registry
+    API -->|Direct Query| TS_query
+
+    %% Model System Interactions
+    Registry -->|Uses Models| ModelsPkg
+    Registry -->|Load/Save| TS
+    ModelsPkg -->|Tensor Ops| TOps
+
+    %% Agent Storage Interactions
+    IA -->|Data Ingestion| TS_insert
+
+    NQLA -->|Query Execution| TS_query
+    NQLA -->|Record Retrieval| TS_get
+
+    RLA -->|State Persistence| TS_insert
+    RLA -->|Experience Sampling| TS_sample
+    RLA -->|State Retrieval| TS_get
+
+    AutoMLA -->|Trial Storage| TS_insert
+    AutoMLA -->|Data Retrieval| TS_query
+
+    %% Computational Operations
+    NQLA -->|Vector Operations| TOps
+    RLA -->|Policy Evaluation| TOps
+    AutoMLA -->|Model Optimization| TOps
+
+    %% Indirect Storage Write-back
+    TOps -.->|Intermediate Results| TS_insert
+```
+
+## Getting Started
+
+### Prerequisites
+
+*   Python 3.10+
+*   PyTorch
+*   FastAPI
+*   Uvicorn
+*   Streamlit
+*   Pydantic v2
+*   Requests
+*   Pillow (for image preprocessing)
+*   Matplotlib (optional, for plotting RL rewards)
+
+### Installation
+
+1.  Clone the repository:
+
+    ```bash
+    git clone https://github.com/tensorus/tensorus.git
+    cd tensorus
+    ```
+
+2.  Create a virtual environment (recommended):
+
+    ```bash
+    python3 -m venv venv
+    source venv/bin/activate  # On Linux/macOS
+    venv\Scripts\activate  # On Windows
+    ```
+
+3.  Install dependencies using the provided setup script:
+
+    ```bash
+    ./setup.sh
+    ```
+    This installs Python requirements from `requirements.txt` and
+    `requirements-test.txt`, using CPU wheels for PyTorch and
+    pinning `httpx` to a compatible version. The test requirements
+    also install `fastapi>=0.110` for compatibility with Pydantic v2.
+    The script also installs test requirements for running the Python test suite.
+    Heavy machine-learning libraries (e.g. `xgboost`, `lightgbm`, `catboost`,
+    `statsmodels`, `torch-geometric`) are not installed by default. Install
+    them separately using `pip install tensorus[models]` or by installing the
+    `tensorus-models` package if you need the built-in models.
+    The audit logger writes to `tensorus_audit.log` by default. Override the
+    path with the `TENSORUS_AUDIT_LOG_PATH` environment variable if desired.
+
+### Running the API Server
+
+1.  Navigate to the project root directory (the directory containing the `tensorus` folder and `pyproject.toml`).
+2.  Ensure your virtual environment is activated if you are using one.
+3.  Start the FastAPI backend server using:
+
+    ```bash
+    uvicorn tensorus.api:app --host 0.0.0.0 --port 7860
+    ```
+
+    *   This command launches Uvicorn with the `app` instance defined in `tensorus/api.py`.
+    *   Access the API documentation at `http://localhost:7860/docs` or `http://localhost:7860/redoc`.
+    *   All dataset and agent endpoints are available once the server is running.
+
+### Running the Streamlit UI
+
+1.  In a separate terminal (with the virtual environment activated), navigate to the project root.
+2.  Start the Streamlit frontend:
+
+    ```bash
+    streamlit run app.py
+    ```
+
+    *   Access the UI in your browser at the URL provided by Streamlit (usually `http://localhost:8501`).
+
+### Running the MCP Server
+
+Tensorus provides a lightweight Python implementation of the Model Context Protocol server using `fastmcp`. It exposes the FastAPI endpoints as tools so you can run an MCP server without Node.js.
+
+**Starting the MCP Server:**
+
+1. Install dependencies (includes `fastmcp`):
+   ```bash
+   pip install -r requirements.txt
+   ```
+2. Ensure the FastAPI backend is running.
+3. Start the server from the repository root:
+   ```bash
+   python -m tensorus.mcp_server
+   ```
+   Add `--transport streamable-http` for a web endpoint. SSE transport is deprecated.
+   If you plan to run the MCP server tests in `tests/test_mcp_server.py` and `tests/test_mcp_client.py`, run `./setup.sh` beforehand to install all required packages.
+
+
+### Running the Agents (Examples)
+
+You can run the example agents directly from their respective files:
+
+*   **RL Agent:**
+
+    ```bash
+    python tensorus/rl_agent.py
+    ```
+
+*   **AutoML Agent:**
+
+    ```bash
+    python tensorus/automl_agent.py
+    ```
+
+*   **Ingestion Agent:**
+
+    ```bash
+    python tensorus/ingestion_agent.py
+    ```
+
+    *   Note: The Ingestion Agent will monitor the `temp_ingestion_source` directory (created automatically if it doesn't exist in the project root) for new files.
+
+### Docker Usage
+
+Tensorus can also be run inside a Docker container. Build the image from the project root:
+
+```bash
+docker build -t tensorus .
+```
+
+Run the container and expose the API server on port `7860`:
+
+```bash
+docker run -p 7860:7860 tensorus
+```
+
+The FastAPI documentation will then be available at `http://localhost:7860/docs`.
+
+If your system has NVIDIA GPUs and the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker) installed, you can pass `--gpus all` to `docker run` and modify `setup.sh` to install CUDA-enabled PyTorch wheels for GPU acceleration.
+
+### Deployment
+
+Tensorus adds several security headers by default. You can customize them with environment variables:
+
+* `TENSORUS_X_FRAME_OPTIONS` – value for the `X-Frame-Options` header (default `SAMEORIGIN`).
+* `TENSORUS_CONTENT_SECURITY_POLICY` – value for the `Content-Security-Policy` header (default `default-src 'self'`).
+
+If either variable is empty or set to `NONE`, the corresponding header is omitted.
+The default policy prevents ReDoc from loading its CDN assets. See [ReDoc and Content Security Policy](docs/api_guide.md#re-doc-and-content-security-policy) for an example policy that allows them.
+
+Example configuration:
+
+```bash
+# Allow embedding from a trusted site
+TENSORUS_X_FRAME_OPTIONS="ALLOW-FROM https://example.com"
+
+# Permit scripts from an external CDN
+TENSORUS_CONTENT_SECURITY_POLICY="default-src 'self'; script-src 'self' https://cdn.example.com"
+```
+
+### Test Suite Dependencies
+
+The Python tests rely on packages from both `requirements.txt` and
+`requirements-test.txt`. Make sure these dependencies are installed
+before running `pytest` by executing the provided setup script:
+
+```bash
+./setup.sh
+```
+
+### Running Tests
+
+Tensorus includes Python unit tests. To set up the environment and run them:
+
+1. Install all dependencies using:
+
+    ```bash
+    ./setup.sh
+    ```
+
+    This script installs packages from `requirements.txt` and `requirements-test.txt` (which pins `fastapi>=0.110` for Pydantic v2 support). It also pulls in `fastmcp` and other dependencies required by `tests/test_mcp_server.py` and `tests/test_mcp_client.py`.
+
+2. Run the Python test suite:
+
+    ```bash
+    pytest
+    ```
+
+    To specifically verify the Model Context Protocol components, run the MCP
+    server and client tests:
+
+    ```bash
+    pytest tests/test_mcp_server.py tests/test_mcp_client.py
+    ```
+
+
+## Using Tensorus
+
+### API Endpoints
+
+The API provides the following main endpoints:
+
+*   **Datasets:**
+    *   `POST /datasets/create`: Create a new dataset.
+    *   `POST /datasets/{name}/ingest`: Ingest a tensor into a dataset.
+    *   `GET /datasets/{name}/fetch`: Retrieve all records from a dataset.
+    *   `GET /datasets/{name}/records`: Retrieve a page of records. Supports `offset` (start index, default `0`) and `limit` (max results, default `100`).
+    *   `GET /datasets`: List all available datasets.
+*   **Querying:**
+    *   `POST /query`: Execute an NQL query.
+*   **Agents:**
+    *   `GET /agents`: List all registered agents.
+    *   `GET /agents/{agent_id}/status`: Get the status of a specific agent.
+    *   `POST /agents/{agent_id}/start`: Start an agent.
+    *   `POST /agents/{agent_id}/stop`: Stop an agent.
+    *   `GET /agents/{agent_id}/logs`: Get recent logs for an agent.
+*   **Metrics & Monitoring:**
+    *   `GET /metrics/dashboard`: Get aggregated dashboard metrics.
+
+### Dataset Schemas
+
+Datasets can optionally include a schema when created. The schema defines
+required metadata fields and expected tensor `shape` and `dtype`. Inserts that
+violate the schema will raise a validation error.
+
+Example:
+
+```python
+schema = {
+    "shape": [3, 10],
+    "dtype": "float32",
+    "metadata": {"source": "str", "value": "int"}
+}
+storage.create_dataset("my_ds", schema=schema)
+storage.insert("my_ds", torch.rand(3, 10), {"source": "sensor", "value": 5})
+```
+
+## Metadata System
+
+Tensorus includes a detailed metadata subsystem for describing tensors beyond their raw data. Each tensor has a `TensorDescriptor` and can be associated with optional semantic, lineage, computational, quality, relational, and usage metadata. The metadata storage backend is pluggable, supporting in-memory storage for quick testing or PostgreSQL for persistence. Search and aggregation utilities allow querying across these metadata fields. See [metadata_schemas.md](docs/metadata_schemas.md) for schema details.
+
+### Streamlit UI
+
+
+The Streamlit UI provides a user-friendly interface for:
+
+*   **Dashboard:** View basic system metrics and agent status.
+*   **Agent Control:** Start, stop, and view logs for agents.
+*   **NQL Chat:** Enter natural language queries and view results.
+*   **Data Explorer:** Browse datasets, preview data, and perform tensor operations.
+
+## Natural Query Language (NQL)
+
+Tensorus ships with a simple regex‑based Natural Query Language for retrieving
+tensors by metadata. You can issue NQL queries via the API or from the "NQL
+Chat" page in the Streamlit UI.
+
+### Enabling LLM rewriting
+
+Set `NQL_USE_LLM=true` before starting the API server or Streamlit UI to enable
+experimental LLM rewriting of natural language queries. Optionally specify a
+model with `NQL_LLM_MODEL=<model-name>` (e.g., `google/flan-t5-base`). This
+feature relies on the heavy `transformers` dependency and may trigger a model
+download the first time it runs, which can take some time.
+
+## Agent Details
+
+### Data Ingestion Agent
+
+*   **Functionality:** Monitors a source directory for new files, preprocesses them into tensors, and inserts them into TensorStorage.
+*   **Supported File Types:** CSV, PNG, JPG, JPEG, TIF, TIFF (can be extended).
+*   **Preprocessing:** Uses default functions for CSV and images (resize, normalize).
+*   **Configuration:**
+    *   `source_directory`: The directory to monitor.
+    *   `polling_interval_sec`: How often to check for new files.
+    *   `preprocessing_rules`: A dictionary mapping file extensions to custom preprocessing functions.
+
+### RL Agent
+
+*   **Functionality:** A Deep Q-Network (DQN) agent that learns from experiences stored in TensorStorage.
+*   **Environment:** Uses a `DummyEnv` for demonstration.
+*   **Experience Storage:** Stores experiences (state, action, reward, next_state, done) in TensorStorage.
+*   **Training:** Implements epsilon-greedy exploration and target network updates.
+*   **Configuration:**
+    *   `state_dim`: Dimensionality of the environment state.
+    *   `action_dim`: Number of discrete actions.
+    *   `hidden_size`: Hidden layer size for the DQN.
+    *   `lr`: Learning rate.
+    *   `gamma`: Discount factor.
+    *   `epsilon_*`: Epsilon-greedy parameters.
+    *   `target_update_freq`: Target network update frequency.
+    *   `batch_size`: Experience batch size.
+    *   `experience_dataset`: Dataset name for experiences.
+    *   `state_dataset`: Dataset name for state tensors.
+
+### AutoML Agent
+
+*   **Functionality:** Performs hyperparameter optimization using random search.
+*   **Model:** Trains a simple `DummyMLP` model.
+*   **Search Space:** Configurable hyperparameter search space (learning rate, hidden size, activation).
+*   **Evaluation:** Trains and evaluates models on synthetic data.
+*   **Results:** Stores trial results (parameters, score) in TensorStorage.
+*   **Configuration:**
+    *   `search_space`: Dictionary defining the hyperparameter search space.
+    *   `input_dim`: Input dimension for the model.
+    *   `output_dim`: Output dimension for the model.
+    *   `task_type`: Type of task ('regression' or 'classification').
+    *   `results_dataset`: Dataset name for storing results.
+
+
+### Tensorus Models
+
+The collection of example models previously bundled with Tensorus now lives in
+a separate repository: [tensorus/models](https://github.com/tensorus/models).
+Install it with:
+
+```bash
+pip install tensorus-models
+```
+
+When the package is installed, Tensorus will automatically import it. Set the
+environment variable `TENSORUS_MINIMAL_IMPORT=1` before importing Tensorus to
+skip this optional dependency and keep startup lightweight.
+
+## Basic Tensor Operations
+
+This section details the core tensor manipulation functionalities provided by `tensor_ops.py`. These operations are designed to be robust, with built-in type and shape checking where appropriate.
+
+#### Arithmetic Operations
+
+*   `add(t1, t2)`: Element-wise addition of two tensors, or a tensor and a scalar.
+*   `subtract(t1, t2)`: Element-wise subtraction of two tensors, or a tensor and a scalar.
+*   `multiply(t1, t2)`: Element-wise multiplication of two tensors, or a tensor and a scalar.
+*   `divide(t1, t2)`: Element-wise division of two tensors, or a tensor and a scalar. Includes checks for division by zero.
+*   `power(t1, t2)`: Raises each element in `t1` to the power of `t2`. Supports tensor or scalar exponents.
+*   `log(tensor)`: Element-wise natural logarithm with warnings for non-positive values.
+
+#### Matrix and Dot Operations
+
+*   `matmul(t1, t2)`: Matrix multiplication of two tensors, supporting various dimensionalities (e.g., 2D matrices, batched matrix multiplication).
+*   `dot(t1, t2)`: Computes the dot product of two 1D tensors.
+*   `outer(t1, t2)`: Computes the outer product of two 1‑D tensors.
+*   `cross(t1, t2, dim=-1)`: Computes the cross product along the specified dimension (size must be 3).
+*   `matrix_eigendecomposition(matrix_A)`: Returns eigenvalues and eigenvectors of a square matrix.
+*   `matrix_trace(matrix_A)`: Computes the trace of a 2-D matrix.
+*   `tensor_trace(tensor_A, axis1=0, axis2=1)`: Trace of a tensor along two axes.
+*   `svd(matrix)`: Singular value decomposition of a matrix, returns `U`, `S`, and `Vh`.
+*   `qr_decomposition(matrix)`: QR decomposition returning `Q` and `R`.
+*   `lu_decomposition(matrix)`: LU decomposition returning permutation `P`, lower `L`, and upper `U` matrices.
+*   `cholesky_decomposition(matrix)`: Cholesky factor of a symmetric positive-definite matrix.
+*   `matrix_inverse(matrix)`: Inverse of a square matrix.
+*   `matrix_determinant(matrix)`: Determinant of a square matrix.
+*   `matrix_rank(matrix)`: Rank of a matrix.
+
+#### Reduction Operations
+
+*   `sum(tensor, dim=None, keepdim=False)`: Computes the sum of tensor elements over specified dimensions.
+*   `mean(tensor, dim=None, keepdim=False)`: Computes the mean of tensor elements over specified dimensions. Tensor is cast to float for calculation.
+*   `min(tensor, dim=None, keepdim=False)`: Finds the minimum value in a tensor, optionally along a dimension. Returns values and indices if `dim` is specified.
+*   `max(tensor, dim=None, keepdim=False)`: Finds the maximum value in a tensor, optionally along a dimension. Returns values and indices if `dim` is specified.
+*   `variance(tensor, dim=None, unbiased=False, keepdim=False)`: Variance of tensor elements.
+*   `covariance(matrix_X, matrix_Y=None, rowvar=True, bias=False, ddof=None)`: Covariance matrix estimation.
+*   `correlation(matrix_X, matrix_Y=None, rowvar=True)`: Correlation coefficient matrix.
+
+#### Reshaping and Slicing
+
+*   `reshape(tensor, shape)`: Changes the shape of a tensor without changing its data.
+*   `transpose(tensor, dim0, dim1)`: Swaps two dimensions of a tensor.
+*   `permute(tensor, dims)`: Permutes the dimensions of a tensor according to the specified order.
+*   `flatten(tensor, start_dim=0, end_dim=-1)`: Flattens a range of dimensions into a single dimension.
+*   `squeeze(tensor, dim=None)`: Removes dimensions of size 1, or a specific dimension if provided.
+*   `unsqueeze(tensor, dim)`: Inserts a dimension of size 1 at the given position.
+
+#### Concatenation and Splitting
+
+*   `concatenate(tensors, dim=0)`: Joins a sequence of tensors along an existing dimension.
+*   `stack(tensors, dim=0)`: Joins a sequence of tensors along a new dimension.
+
+#### Advanced Operations
+
+*   `einsum(equation, *tensors)`: Applies Einstein summation convention to the input tensors based on the provided equation string.
+*   `compute_gradient(func, tensor)`: Returns the gradient of a scalar `func` with respect to `tensor`.
+*   `compute_jacobian(func, tensor)`: Computes the Jacobian matrix of a vector function.
+*   `convolve_1d(signal_x, kernel_w, mode='valid')`: 1‑D convolution using `torch.nn.functional.conv1d`.
+*   `convolve_2d(image_I, kernel_K, mode='valid')`: 2‑D convolution using `torch.nn.functional.conv2d`.
+ *   `frobenius_norm(tensor)`: Calculates the Frobenius norm.
+ *   `l1_norm(tensor)`: Calculates the L1 norm (sum of absolute values).
+
+## Tensor Decomposition Operations
+
+Tensorus includes a library of higher‑order tensor factorizations in
+`tensor_decompositions.py`. These operations mirror the algorithms
+available in TensorLy and related libraries.
+
+* **CP Decomposition** – Canonical Polyadic factorization returning
+  weights and factor matrices.
+* **NTF‑CP Decomposition** – Non‑negative CP using
+  `non_negative_parafac`.
+* **Tucker Decomposition** – Standard Tucker factorization for specified
+  ranks.
+* **Non‑negative Tucker / Partial Tucker** – Variants with HOOI and
+  non‑negative constraints.
+* **HOSVD** – Higher‑order SVD (Tucker with full ranks).
+* **Tensor Train (TT)** – Sequence of TT cores representing the tensor.
+* **TT‑SVD** – TT factorization via SVD initialization.
+* **Tensor Ring (TR)** – Circular variant of TT.
+* **Hierarchical Tucker (HT)** – Decomposition using a dimension tree.
+* **Block Term Decomposition (BTD)** – Sum of Tucker‑1 terms for 3‑way
+  tensors.
+* **t‑SVD** – Tensor singular value decomposition based on the
+  t‑product.
+
+Examples of how to call these methods are provided in
+[`tensorus/tensor_decompositions.py`](tensorus/tensor_decompositions.py).
+
+## MCP Server Details
+
+The Tensorus Model Context Protocol (MCP) Server allows external AI agents, LLM-based applications, and other MCP-compatible clients to interact with Tensorus functionalities in a standardized way. It acts as a bridge, translating MCP requests into calls to the Tensorus Python API.
+
+### Overview
+
+*   **Protocol:** Implements the [Model Context Protocol](https://modelcontextprotocol.io/introduction).
+*   **Language:** Python, using the `fastmcp` library.
+*   **Communication:** Typically uses stdio for communication with a single client.
+*   **Interface:** Exposes Tensorus capabilities as a set of "tools" that an MCP client can list and call.
+
+### Available Tools
+
+The MCP server provides tools for various Tensorus functionalities. Below is an overview. For detailed input schemas and descriptions, an MCP client can call the standard `tools/list` method on the server, or you can inspect the tool definitions in `tensorus/mcp_server.py`.
+
+*   **Dataset Management:**
+    *   `tensorus_list_datasets`: Lists all available datasets.
+    *   `tensorus_create_dataset`: Creates a new dataset.
+    *   `tensorus_delete_dataset`: Deletes an existing dataset.
+*   **Tensor Management:**
+    *   `tensorus_ingest_tensor`: Ingests a new tensor (with data provided as JSON) into a dataset.
+    *   `tensorus_get_tensor_details`: Retrieves the data and metadata for a specific tensor.
+    *   `tensorus_delete_tensor`: Deletes a specific tensor from a dataset.
+    *   `tensorus_update_tensor_metadata`: Updates the metadata of a specific tensor.
+*   **Tensor Operations:** These tools allow applying operations from the `TensorOps` library to tensors stored in Tensorus.
+    *   `tensorus_apply_unary_operation`: Applies operations like `log`, `reshape`, `transpose`, `permute`, `sum`, `mean`, `min`, `max`.
+    *   `tensorus_apply_binary_operation`: Applies operations like `add`, `subtract`, `multiply`, `divide`, `power`, `matmul`, `dot`.
+    *   `tensorus_apply_list_operation`: Applies operations like `concatenate` and `stack` that take a list of input tensors.
+    *   `tensorus_apply_einsum`: Applies Einstein summation.
+
+*Note on Tensor Operations via MCP:* Input tensors are referenced by their `dataset_name` and `record_id`. The result is typically stored as a new tensor, and the MCP tool returns details of this new result tensor (like its `record_id`).
+
+### Example Client Interaction (Conceptual)
+
+```javascript
+// Conceptual MCP client-side JavaScript
+// Assuming 'client' is an initialized MCP client connected to the Tensorus MCP Server
+
+async function example() {
+  // List available tools
+  const { tools } = await client.request({ method: 'tools/list' }, {});
+  console.log("Available Tensorus Tools:", tools.map(t => t.name)); // Log only names for brevity
+
+  // Create a new dataset
+  const createResponse = await client.request({ method: 'tools/call' }, {
+    name: 'tensorus_create_dataset',
+    arguments: { dataset_name: 'my_mcp_dataset' }
+  });
+  console.log(JSON.parse(createResponse.content[0].text).message);
+
+  // Ingest a tensor
+  const ingestResponse = await client.request({ method: 'tools/call' }, {
+    name: 'tensorus_ingest_tensor',
+    arguments: {
+      dataset_name: 'my_mcp_dataset',
+      tensor_shape: [2, 2],
+      tensor_dtype: 'float32',
+      tensor_data: [[1.0, 2.0], [3.0, 4.0]],
+      metadata: { source: 'mcp_client_example' }
+    }
+  });
+  // Assuming the Python API returns { success, message, data: { record_id, ... } }
+  // And MCP server stringifies this whole object in the text content
+  const ingestData = JSON.parse(ingestResponse.content[0].text);
+  console.log("Ingest success:", ingestData.success, "Record ID:", ingestData.data.record_id);
+}
+```
+
+You can also interact with the server using the included Python helper:
+
+```python
+from tensorus.mcp_client import TensorusMCPClient
+
+async def example_py():
+    async with TensorusMCPClient("http://localhost:7860/mcp") as client:
+        tools = await client.list_datasets()
+        print(tools)
+```
+
+## Completed Features
+
+*   **Tensor Storage:** Efficiently stores and retrieves PyTorch tensors with associated metadata, including in-memory and optional file-based persistence. Supports dataset creation, tensor ingestion, querying, sampling, and metadata updates.
+*   **Natural Query Language (NQL):** Provides a basic regex-based natural language interface for querying tensor data, supporting retrieval and simple filtering.
+*   **Agent Framework:** Includes several operational agents:
+    *   **Data Ingestion Agent:** Monitors local directories for CSV and image files, preprocesses them, and ingests them into TensorStorage.
+    *   **RL Agent:** Implements a DQN agent that learns from experiences (stored in TensorStorage) in a dummy environment.
+    *   **AutoML Agent:** Performs random search hyperparameter optimization for a dummy MLP model, storing trial results in TensorStorage.
+*   **API-Driven:** A comprehensive FastAPI backend offers RESTful endpoints for dataset management, NQL querying, tensor operations, and agent control (live for Ingestion Agent, simulated for RL/AutoML).
+*   **Streamlit UI:** A multi-page user interface for dashboard overview, agent control, NQL interaction, data exploration, and API interaction.
+*   **Tensor Operations:** A library of robust tensor operations (arithmetic, matrix ops, reductions, reshaping, etc.) accessible via the API.
+*   **Model Context Protocol (MCP) Server:** A Python server built with `fastmcp` exposes Tensorus capabilities (storage and operations) via the Model Context Protocol.
+*   **Extensible Design:** The project is structured with modular components, facilitating future extensions.
+
+## Future Implementation
+
+*   **Enhanced NQL:** Integrate a local or remote LLM for more robust natural language understanding.
+*   **Advanced Agents:** Develop more sophisticated agents for specific tasks (e.g., anomaly detection, forecasting).
+*   **Persistent Storage Backend:** Replace/augment current file-based persistence with more robust database or cloud storage solutions (e.g., PostgreSQL, S3, MinIO).
+*   **Scalability & Performance:**
+    *   Implement tensor chunking for very large tensors.
+    *   Optimize query performance with indexing.
+    *   Asynchronous operations for agents and API calls.
+*   **Security:** Implement authentication and authorization mechanisms for the API and UI.
+*   **Real-World Integration:**
+    *   Connect Ingestion Agent to more data sources (e.g., cloud storage, databases, APIs).
+    *   Integrate RL Agent with real-world environments or more complex simulations.
+*   **Advanced AutoML:**
+    *   Implement sophisticated search algorithms (e.g., Bayesian Optimization, Hyperband).
+    *   Support for diverse model architectures and custom models.
+*   **Model Management:** Add capabilities for saving, loading, versioning, and deploying trained models (from RL/AutoML).
+*   **Streaming Data Support:** Enhance Ingestion Agent to handle real-time streaming data.
+*   **Resource Management:** Add tools and controls for monitoring and managing the resource consumption (CPU, memory) of agents.
+*   **Improved UI/UX:** Continuously refine the Streamlit UI for better usability and richer visualizations.
+*   **Comprehensive Testing:** Expand unit, integration, and end-to-end tests.
+
+## Contributing
+
+Contributions are welcome! Please feel free to open issues or submit pull requests.
+
+## License
+
+MIT License

app.py

@@ -0,0 +1,378 @@
+# app.py
+"""
+Streamlit frontend application for the Tensorus platform.
+New UI structure with top navigation and Nexus Dashboard.
+"""
+
+import streamlit as st
+import json
+import time
+import requests # Needed for ui_utils functions if integrated
+import logging # Needed for ui_utils functions if integrated
+import torch # Needed for integrated tensor utils
+from typing import List, Dict, Any, Optional, Union, Tuple # Needed for integrated tensor utils
+from pages.pages_shared_utils import get_api_status, get_agent_status, get_datasets # Updated imports
+
+# Work around a Streamlit bug where inspecting `torch.classes` during module
+# watching can raise a `RuntimeError`. Removing the module from `sys.modules`
+# prevents Streamlit's watcher from trying to access it.
+import sys
+if "torch.classes" in sys.modules:
+    del sys.modules["torch.classes"]
+
+# --- Page Configuration ---
+st.set_page_config(
+    page_title="Tensorus Platform",
+    page_icon="🧊",
+    layout="wide",
+    initial_sidebar_state="collapsed" # Collapse sidebar as nav is now at top
+)
+
+# --- Configure Logging (Optional but good practice) ---
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# --- Integrated Tensor Utilities (Preserved) ---
+
+def _validate_tensor_data(data: List[Any], shape: List[int]):
+    """
+    Validates if the nested list structure of 'data' matches the 'shape'.
+    Raises ValueError on mismatch. (Optional validation)
+    """
+    if not shape:
+        if not isinstance(data, (int, float)): raise ValueError("Scalar tensor data must be a single number.")
+        return True
+    if not isinstance(data, list): raise ValueError(f"Data for shape {shape} must be a list.")
+    expected_len = shape[0]
+    if len(data) != expected_len: raise ValueError(f"Dimension 0 mismatch: Expected {expected_len}, got {len(data)} for shape {shape}.")
+    if len(shape) > 1:
+        for item in data: _validate_tensor_data(item, shape[1:])
+    elif len(shape) == 1:
+        if not all(isinstance(x, (int, float)) for x in data): raise ValueError("Innermost list elements must be numbers.")
+    return True
+
+def list_to_tensor(shape: List[int], dtype_str: str, data: Union[List[Any], int, float]) -> torch.Tensor:
+    """
+    Converts a Python list (potentially nested) or scalar into a PyTorch tensor
+    with the specified shape and dtype.
+    """
+    try:
+        dtype_map = {
+            'float32': torch.float32, 'float': torch.float,
+            'float64': torch.float64, 'double': torch.double,
+            'int32': torch.int32, 'int': torch.int,
+            'int64': torch.int64, 'long': torch.long,
+            'bool': torch.bool
+        }
+        torch_dtype = dtype_map.get(dtype_str.lower())
+        if torch_dtype is None: raise ValueError(f"Unsupported dtype string: {dtype_str}")
+
+        tensor = torch.tensor(data, dtype=torch_dtype)
+
+        if list(tensor.shape) != shape:
+            logger.debug(f"Initial tensor shape {list(tensor.shape)} differs from target {shape}. Attempting reshape.")
+            try:
+                tensor = tensor.reshape(shape)
+            except RuntimeError as reshape_err:
+                raise ValueError(f"Created tensor shape {list(tensor.shape)} != requested {shape} and reshape failed: {reshape_err}") from reshape_err
+
+        return tensor
+    except (TypeError, ValueError) as e:
+        logger.error(f"Error converting list to tensor: {e}. Shape: {shape}, Dtype: {dtype_str}")
+        raise ValueError(f"Failed tensor conversion: {e}") from e
+    except Exception as e:
+        logger.exception(f"Unexpected error during list_to_tensor: {e}")
+        raise ValueError(f"Unexpected tensor conversion error: {e}") from e
+
+def tensor_to_list(tensor: torch.Tensor) -> Tuple[List[int], str, List[Any]]:
+    """
+    Converts a PyTorch tensor back into its shape, dtype string, and nested list representation.
+    """
+    if not isinstance(tensor, torch.Tensor):
+        raise TypeError("Input must be a torch.Tensor")
+    shape = list(tensor.shape)
+    dtype_str = str(tensor.dtype).split('.')[-1]
+    data = tensor.tolist()
+    return shape, dtype_str, data
+
+# --- Helper functions for dashboard (can be expanded) ---
+def get_total_tensors_placeholder():
+    # For now, as this endpoint is hypothetical for this task
+    return "N/A"
+
+@st.cache_data(ttl=300)
+def get_active_datasets_placeholder():
+    datasets = get_datasets()
+    if datasets: # get_datasets returns [] on error or if no datasets
+        return str(len(datasets))
+    return "Error"
+
+@st.cache_data(ttl=60)
+def get_agents_online_placeholder():
+    agent_data = get_agent_status()
+    if agent_data:
+        try:
+            # Assuming agent_data is a dict like {'agent_id': {'status': 'running', ...}}
+            # or {'agent_id': {'running': True, ...}}
+            online_agents = sum(1 for agent in agent_data.values()
+                                if agent.get('running') is True or str(agent.get('status', '')).lower() == 'running')
+            total_agents = len(agent_data)
+            return f"{online_agents}/{total_agents} Online"
+        except Exception as e:
+            logger.error(f"Error processing agent data for dashboard: {e}")
+            return "Error"
+    return "N/A" # If agent_data is None
+
+# --- CSS Styles ---
+# Renaming app.py's specific CSS loader to avoid confusion with the shared one.
+def load_app_specific_css():
+    # This function now only loads styles specific to the Nexus Dashboard content in app.py
+    # General styles (body, .stApp, nav, common-card, etc.) are in pages_shared_utils.load_shared_css()
+    st.markdown("""
+<style>
+    /* Nexus Dashboard Specific Styles */
+    .dashboard-title { /* Custom title for the dashboard */
+        color: #e0e0ff; /* Light purple/blue, matching shared h1 */
+        text-align: center;
+        margin-top: 1rem; /* Standardized margin */
+        margin-bottom: 2rem;
+        font-size: 2.8em; /* Slightly larger for main dashboard title */
+        font-weight: bold;
+    }
+
+    /* Metric Cards Container for Dashboard */
+    .metric-card-container {
+        display: flex;
+        flex-wrap: wrap;
+        justify-content: space-around; 
+        gap: 20px; 
+        padding: 0 1rem; 
+    }
+    
+    /* Individual Metric Card - inherits from .common-card (defined in shared_utils) */
+    /* This specific .metric-card class is for dashboard cards if they need further specialization */
+    .metric-card { 
+        /* Inherits background, border, padding, shadow, transition from .common-card */
+        flex: 1 1 220px; /* Adjusted flex basis */
+        min-width: 200px; 
+        max-width: 300px;
+        text-align: center;
+    }
+    /* .metric-card:hover is inherited from .common-card:hover */
+    
+    .metric-card .icon { /* Specific styling for icons within dashboard metric cards */
+        font-size: 2.8em; /* Slightly larger icon for dashboard */
+        margin-bottom: 0.5rem; /* Tighter spacing */
+        /* color is inherited from .common-card .icon or can be overridden here */
+    }
+    .metric-card h3 { /* Metric card titles */
+        /* color, font-size, margin-bottom, font-weight inherited from .common-card h3 */
+        /* No specific overrides here unless needed for dashboard metric cards */
+    }
+    .metric-card p.metric-value { /* Specific class for the main value display */
+        font-size: 2em; /* Larger font for the metric value */
+        font-weight: bold;
+        color: #ffffff; /* White color for emphasis */
+        margin-top: 0.25rem; /* Adjust as needed */
+        margin-bottom: 0;
+    }
+
+    /* Specific status icon colors for API status card in dashboard */
+    .metric-card.api-status-connected .icon { color: #50C878; } /* Emerald Green */
+    .metric-card.api-status-disconnected .icon { color: #FF6961; } /* Pastel Red */
+
+    /* Activity Feed Styles for Dashboard */
+    .activity-feed-container { /* Container for the activity feed section */
+        margin-top: 2.5rem;
+        padding: 0 1.5rem; 
+    }
+    /* .activity-feed-container h2 is covered by shared h2 styles */
+    
+    .activity-item { /* Individual item in the feed */
+        background-color: #1e2a47; /* Slightly lighter than common-card, for variety */
+        padding: 0.85rem 1.25rem; /* Adjusted padding */
+        border-radius: 6px;
+        margin-bottom: 0.6rem; /* Slightly more space */
+        font-size: 0.95em;
+        border-left: 4px solid #3a6fbf; /* Accent Blue border */
+        box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    .activity-item .timestamp { /* Timestamp within an activity item */
+        color: #8080af; /* Muted purple for timestamp */
+        font-weight: bold;
+        font-size: 0.85em; /* Smaller timestamp */
+        margin-right: 0.75em; /* More space after timestamp */
+    }
+    .activity-item strong { /* Agent name or key part of activity */
+        color: #b0b0df; /* Tertiary heading color for emphasis */
+    }
+</style>
+""", unsafe_allow_html=True)
+
+# --- Page Functions ---
+
+def nexus_dashboard_content():
+    # Uses .dashboard-title for its main heading
+    st.markdown('<h1 class="dashboard-title">Tensorus Nexus</h1>', unsafe_allow_html=True)
+
+    # System Health & Key Metrics
+    # Uses .metric-card-container for the overall layout
+    st.markdown('<div class="metric-card-container">', unsafe_allow_html=True)
+
+    # Card 1: Total Tensors
+    total_tensors_val = get_total_tensors_placeholder()
+    st.markdown(f"""
+    <div class="common-card metric-card">
+        <div class="icon">⚙️</div>
+        <h3>Total Tensors</h3>
+        <p class="metric-value">{total_tensors_val}</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # Card 2: Active Datasets
+    active_datasets_val = get_active_datasets_placeholder()
+    st.markdown(f"""
+    <div class="common-card metric-card">
+        <div class="icon">📚</div>
+        <h3>Active Datasets</h3>
+        <p class="metric-value">{active_datasets_val}</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # Card 3: Agents Online
+    agents_online_val = get_agents_online_placeholder()
+    st.markdown(f"""
+    <div class="common-card metric-card">
+        <div class="icon">🤖</div>
+        <h3>Agents Online</h3>
+        <p class="metric-value">{agents_online_val}</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # Card 4: API Status
+    @st.cache_data(ttl=30)
+    def cached_get_api_status():
+        return get_api_status()
+
+    api_ok, _ = cached_get_api_status()
+    api_status_text_val = "Connected" if api_ok else "Disconnected"
+    # Add specific class for API status icon coloring based on shared status styles
+    api_status_icon_class = "api-status-connected" if api_ok else "api-status-disconnected"
+    api_icon_char = "✔️" if api_ok else "❌"
+    st.markdown(f"""
+    <div class="common-card metric-card {api_status_icon_class}">
+        <div class="icon">{api_icon_char}</div>
+        <h3>API Status</h3>
+        <p class="metric-value">{api_status_text_val}</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.markdown('</div>', unsafe_allow_html=True) # Close metric-card-container
+
+    # Agent Activity Feed
+    # Uses .activity-feed-container and h2 (which is styled by shared CSS)
+    st.markdown('<div class="activity-feed-container">', unsafe_allow_html=True)
+    st.markdown('<h2>Recent Agent Activity</h2>', unsafe_allow_html=True)
+
+    # Placeholder activity items
+    activity_items = [
+        {"timestamp": "2023-10-27 10:05:15", "agent": "IngestionAgent", "action": "added 'img_new.png' to 'raw_images'"},
+        {"timestamp": "2023-10-27 10:02:30", "agent": "RLAgent", "action": "completed training cycle, reward: 75.2"},
+        {"timestamp": "2023-10-27 09:55:48", "agent": "MonitoringAgent", "action": "detected high CPU usage on node 'compute-01'"},
+        {"timestamp": "2023-10-27 09:45:10", "agent": "IngestionAgent", "action": "processed batch of 100 sensor readings"},
+    ]
+
+    for item in activity_items:
+        st.markdown(f"""
+        <div class="activity-item">
+            <span class="timestamp">[{item['timestamp']}]</span>
+            <strong>{item['agent']}:</strong> {item['action']}
+        </div>
+        """, unsafe_allow_html=True)
+    st.markdown('</div>', unsafe_allow_html=True) # Close activity-feed-container
+
+
+# --- Main Application ---
+# Import the shared CSS loader
+try:
+    from pages.pages_shared_utils import load_css as load_shared_css
+except ImportError:
+    st.error("Failed to import shared CSS loader. Page styling will be incomplete.")
+    def load_shared_css(): pass # Dummy function
+
+def main():
+    load_shared_css() # Load shared styles first
+    load_app_specific_css() # Then load app-specific styles (for dashboard)
+
+    # Initialize session state for current page if not set
+    if 'current_page' not in st.session_state:
+        st.session_state.current_page = "Nexus Dashboard"
+
+    # --- Top Navigation Bar ---
+    nav_items = {
+        "Nexus Dashboard": "Nexus Dashboard",
+        "Agents": "Agents",
+        "Explorer": "Explorer",
+        "Query Hub": "Query Hub",
+        "API Docs": "API Docs"
+    }
+
+    nav_html_parts = [f'<div class="topnav-container"><span class="logo">🧊 Tensorus</span><nav>']
+    for page_id, page_name in nav_items.items():
+        active_class = "active" if st.session_state.current_page == page_id else ""
+        # Use st.button an invisible character as key to make Streamlit rerun and update session state
+        # This is a common workaround for making nav links update state
+        # We'll use a more robust JavaScript approach if this is problematic, but st.query_params is better
+        
+        # Using query_params for navigation state is more robust
+        # Check if query_params for page is set, if so, it overrides session_state
+        query_params = st.query_params.to_dict()
+        if "page" in query_params and query_params["page"] in nav_items:
+            st.session_state.current_page = query_params["page"]
+            # Clear the query param after use to avoid it sticking on manual refresh
+            # However, for deeplinking, we might want to keep it.
+            # For now, let's allow it to persist. To clear: st.query_params.clear()
+            
+        # Construct the link with query_params
+        nav_html_parts.append(
+            f'<a href="?page={page_id}" class="{active_class}" target="_self">{page_name}</a>'
+        )
+
+    nav_html_parts.append('</nav></div>')
+    st.markdown("".join(nav_html_parts), unsafe_allow_html=True)
+    
+    # Handle page selection clicks (alternative to query_params if that proves problematic)
+    # This part is tricky with pure st.markdown links.
+    # The query_params approach is generally preferred for web-like navigation.
+
+    # --- Content Area ---
+    # The main app.py now acts as a router to other pages or displays dashboard content directly.
+    if st.session_state.current_page == "Nexus Dashboard":
+        nexus_dashboard_content()
+    elif st.session_state.current_page == "Agents":
+        st.switch_page("pages/control_panel_v2.py")
+    elif st.session_state.current_page == "Explorer":
+        st.switch_page("pages/data_explorer_v2.py")
+    elif st.session_state.current_page == "Query Hub":
+        st.switch_page("pages/nql_chatbot_v2.py")
+    elif st.session_state.current_page == "API Docs":
+        st.switch_page("pages/api_playground_v2.py")
+    else:
+        # Default to Nexus Dashboard if current_page is unrecognized
+        st.session_state.current_page = "Nexus Dashboard"
+        nexus_dashboard_content()
+        # It's good practice to trigger a rerun if state was corrected
+        st.rerun()
+
+
+if __name__ == "__main__":
+    # --- Initialize Old Session State Keys (to avoid errors if they are still used by preserved code) ---
+    # This should be phased out as those sections are rebuilt.
+    if 'agent_status' not in st.session_state: st.session_state.agent_status = None
+    if 'datasets' not in st.session_state: st.session_state.datasets = []
+    if 'selected_dataset' not in st.session_state: st.session_state.selected_dataset = None
+    if 'dataset_preview' not in st.session_state: st.session_state.dataset_preview = None
+    if 'explorer_result' not in st.session_state: st.session_state.explorer_result = None
+    if 'nql_response' not in st.session_state: st.session_state.nql_response = None
+
+    main()

docker-compose.yml

@@ -0,0 +1,55 @@
+version: '3.8'
+
+services:
+  app:
+    build: .
+    ports:
+      - "7860:7860"
+    volumes:
+      # Persist audit logs generated by the app container to the host
+      # The path inside the container is /app/tensorus_audit.log as defined in tensorus/audit.py (implicitly if WORKDIR is /app)
+      - ./tensorus_audit.log:/app/tensorus_audit.log
+    environment:
+      # Python settings
+      PYTHONUNBUFFERED: 1
+      # Tensorus App Settings
+      TENSORUS_STORAGE_BACKEND: postgres
+      TENSORUS_POSTGRES_HOST: db  # Service name of the postgres container
+      TENSORUS_POSTGRES_PORT: 5432 # Default postgres port, container to container
+      TENSORUS_POSTGRES_USER: tensorus_user_compose
+      TENSORUS_POSTGRES_PASSWORD: tensorus_password_compose
+      TENSORUS_POSTGRES_DB: tensorus_db_compose
+      # TENSORUS_POSTGRES_DSN: "" # Can be set if preferred over individual params
+      TENSORUS_API_KEY_HEADER_NAME: "X-API-KEY"
+      TENSORUS_VALID_API_KEYS: "compose_key1,another_secure_key" # Comma-separated or JSON list of keys
+      TENSORUS_AUTH_JWT_ENABLED: "False"
+      # TENSORUS_AUTH_JWT_ISSUER: "your_issuer_here"
+      # TENSORUS_AUTH_JWT_AUDIENCE: "tensorus_api_audience"
+      # TENSORUS_AUTH_JWT_ALGORITHM: "RS256"
+      # TENSORUS_AUTH_JWT_JWKS_URI: "your_jwks_uri_here"
+      TENSORUS_AUTH_DEV_MODE_ALLOW_DUMMY_JWT: "False"
+    depends_on:
+      db:
+        condition: service_healthy # Wait for db to be healthy (Postgres specific healthcheck needed in db service)
+        # For simpler startup without healthcheck, just `depends_on: - db` is fine,
+        # but app might start before DB is ready. Entrypoint script in app can handle retries then.
+
+  db:
+    image: postgres:15-alpine
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/ # Persist data
+    environment:
+      POSTGRES_USER: tensorus_user_compose # Must match app's TENSORUS_POSTGRES_USER
+      POSTGRES_PASSWORD: tensorus_password_compose # Must match app's TENSORUS_POSTGRES_PASSWORD
+      POSTGRES_DB: tensorus_db_compose # Must match app's TENSORUS_POSTGRES_DB
+    ports:
+      - "5433:5432" # Expose Postgres on host port 5433 to avoid conflict if local PG runs on 5432
+    healthcheck: # Basic healthcheck for Postgres
+      test: ["CMD-SHELL", "pg_isready -U tensorus_user_compose -d tensorus_db_compose"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  postgres_data: # Defines the named volume for data persistence
+    driver: local

docs/api_guide.md

@@ -0,0 +1,49 @@
+# API Guide
+
+The Tensorus Metadata System provides a comprehensive RESTful API for managing and interacting with tensor metadata.
+
+## Interactive API Documentation (Swagger UI)
+
+The API is self-documenting using OpenAPI. Once the application is running (e.g., locally at `http://localhost:7860`), you can access the interactive Swagger UI at:
+
+*   **`/docs`**: [http://localhost:7860/docs](http://localhost:7860/docs)
+
+This interface allows you to explore all available endpoints, view their request and response schemas, and even try out API calls directly from your browser.
+
+## Alternative API Documentation (ReDoc)
+
+An alternative ReDoc interface is also available at:
+
+*   **`/redoc`**: [http://localhost:7860/redoc](http://localhost:7860/redoc)
+
+### ReDoc and Content Security Policy
+
+Tensorus sets the header `Content-Security-Policy: default-src 'self'` by default. This strict policy blocks the external fonts, stylesheets, and scripts that ReDoc loads from CDNs, so the ReDoc page will appear blank unless the policy is relaxed.
+
+To permit these assets you can override the policy via the `TENSORUS_CONTENT_SECURITY_POLICY` environment variable:
+
+```bash
+TENSORUS_CONTENT_SECURITY_POLICY="default-src 'self'; script-src 'self' https://cdn.jsdelivr.net; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; font-src 'self' https://fonts.gstatic.com; img-src 'self' https://fastapi.tiangolo.com"
+```
+
+Set the variable to an empty string or `NONE` to remove the header entirely.
+
+## Main API Categories
+
+The API is organized into several categories based on functionality:
+
+*   **Tensor Descriptors:** Core operations for creating, reading, updating, deleting, and listing tensor descriptors.
+*   **Semantic Metadata (Per Tensor):** Managing human-readable names, descriptions, etc., associated with specific tensors, nested under `/tensor_descriptors/{tensor_id}/semantic/`.
+*   **Extended Metadata (Per Tensor):** CRUD operations for detailed metadata types, nested under `/tensor_descriptors/{tensor_id}/`:
+    *   Lineage Metadata (`/lineage`)
+    *   Computational Metadata (`/computational`)
+    *   Quality Metadata (`/quality`)
+    *   Relational Metadata (`/relational`)
+    *   Usage Metadata (`/usage`)
+*   **Versioning & Lineage:** Endpoints for creating tensor versions and managing lineage relationships at a higher level.
+*   **Search & Aggregation:** Advanced querying, text-based search across metadata, and metadata aggregation.
+*   **Import/Export:** Endpoints for exporting and importing tensor metadata in JSON format.
+*   **Management:** Health checks and system metrics.
+*   **Authentication:** Write operations (POST, PUT, PATCH, DELETE) are protected by API keys. See [Installation and Configuration](./installation.md) for details on setting API keys. The API key should be passed in the HTTP header specified by `TENSORUS_API_KEY_HEADER_NAME` (default: `X-API-KEY`).
+
+Please refer to the interactive `/docs` for detailed information on each endpoint, including request parameters, request bodies, and response structures.

docs/index.md

@@ -0,0 +1,22 @@
+# Welcome to the Tensorus Metadata System
+
+The Tensorus Metadata System is a comprehensive solution for managing metadata
+associated with tensor data in an Agentic Tensor Database. It enables advanced
+capabilities such as semantic search, lineage tracking, and version control.
+
+This documentation provides an overview of the system, how to install and
+configure it, and details about its API and metadata schemas.
+
+## Key Features
+
+*   **Rich Metadata Schemas:** Describe tensors with detailed structural, semantic, lineage, computational, quality, relational, and usage metadata.
+*   **Flexible Storage:** Supports in-memory storage for quick testing and a PostgreSQL backend for persistent, scalable storage.
+*   **Powerful API:** A comprehensive RESTful API for CRUD operations, advanced querying, search, aggregation, versioning, and lineage tracking.
+*   **Data Export/Import:** Utilities to export and import metadata in a standard JSON format.
+*   **Security:** Basic API key authentication for write operations and audit logging.
+*   **Monitoring:** Health check and basic metrics endpoints.
+*   **Enterprise Ready (Conceptual):** Designed with enterprise features like JWT authentication in mind.
+*   **Analytics:** Example APIs for deriving insights from metadata (e.g., stale tensors, complex tensors, co-occurring tags).
+*   **Dockerized:** Easy to deploy using Docker and Docker Compose.
+
+Navigate through the documentation using the sidebar to learn more.

docs/installation.md

@@ -0,0 +1,89 @@
+# Installation and Configuration
+
+## Installation
+
+### Using Docker (Recommended for Local Development/Testing)
+
+The easiest way to get Tensorus and its PostgreSQL backend running locally is by using Docker Compose:
+
+1.  Ensure you have Docker and Docker Compose installed.
+2.  Clone the repository (if applicable) or ensure you have the `docker-compose.yml` and application files.
+3.  From the project root, run:
+    ```bash
+    docker-compose up --build
+    ```
+4.  The API will typically be available at `http://localhost:7860`.
+
+### Manual Installation (From Source)
+
+1.  Ensure you have Python 3.10+ installed.
+2.  Clone the repository.
+3.  Create and activate a virtual environment:
+    ```bash
+    python -m venv venv
+    source venv/bin/activate  # On Windows: venv\Scripts\activate
+    ```
+4.  Install dependencies:
+    ```bash
+    pip install -r requirements.txt
+    ```
+    Heavy machine-learning libraries used by the optional models are not
+    installed by default. Install them with the `[models]` extra when needed:
+    ```bash
+    pip install -e .[models]
+    ```
+    If you intend to run the test suite, also install the test requirements:
+    ```bash
+    pip install -r requirements-test.txt
+    ```
+    This ensures `fastapi>=0.110` is installed so the API is compatible with
+    Pydantic v2.
+5.  *(Optional)* Install the example models package. The built-in models that
+    were previously part of this repository now live at
+    [https://github.com/tensorus/models](https://github.com/tensorus/models):
+    ```bash
+    pip install tensorus-models
+    ```
+6.  Set up the necessary environment variables (see Configuration below).
+7.  Run the application using Uvicorn:
+    ```bash
+    uvicorn tensorus.api:app --host 0.0.0.0 --port 7860
+    ```
+
+    This exposes all dataset and agent endpoints at `http://localhost:7860`.
+
+## Configuration
+
+Tensorus is configured via environment variables. Key variables include:
+
+*   `TENSORUS_STORAGE_BACKEND`: Specifies the storage backend.
+    *   `in_memory` (default): Uses in-memory storage (data is not persisted).
+    *   `postgres`: Uses PostgreSQL.
+*   `TENSORUS_POSTGRES_HOST`: Hostname for PostgreSQL (e.g., `localhost` or `db` if using Docker Compose).
+*   `TENSORUS_POSTGRES_PORT`: Port for PostgreSQL (e.g., `5432`).
+*   `TENSORUS_POSTGRES_USER`: Username for PostgreSQL.
+*   `TENSORUS_POSTGRES_PASSWORD`: Password for PostgreSQL.
+*   `TENSORUS_POSTGRES_DB`: Database name for PostgreSQL.
+*   `TENSORUS_POSTGRES_DSN`: Alternative DSN connection string for PostgreSQL.
+*   `TENSORUS_VALID_API_KEYS`: List of valid API keys. Values can be a comma-separated string (e.g., `key1,key2,anotherkey`) or a JSON array. If no keys are required, set this to `[]`.
+*   `TENSORUS_API_KEY_HEADER_NAME`: HTTP header name for the API key (default: `X-API-KEY`).
+*   `TENSORUS_MINIMAL_IMPORT`: Set to any value to skip importing the optional
+    `tensorus-models` package for a lightweight installation.
+
+### JWT Authentication (Conceptual - For Future Use)
+*   `TENSORUS_AUTH_JWT_ENABLED`: `True` or `False` (default `False`).
+*   `TENSORUS_AUTH_JWT_ISSUER`: URL of the JWT issuer.
+*   `TENSORUS_AUTH_JWT_AUDIENCE`: Expected audience for JWTs.
+*   `TENSORUS_AUTH_JWT_ALGORITHM`: Algorithm (default `RS256`).
+*   `TENSORUS_AUTH_JWT_JWKS_URI`: URI to fetch JWKS.
+*   `TENSORUS_AUTH_DEV_MODE_ALLOW_DUMMY_JWT`: `True` to allow dummy JWTs for development if JWT auth is enabled (default `False`).
+
+Refer to the `docker-compose.yml` for example environment variable settings when running with Docker. For manual setup, export these variables in your shell or use a `.env` file (if your setup supports it, though direct environment variables are primary).
+
+## Running Tests
+
+Tensorus includes Python unit tests. After installing the dependencies you can run them with:
+
+```bash
+pytest
+```

docs/metadata_schemas.md

@@ -0,0 +1,57 @@
+# Metadata Schemas Overview
+
+The Tensorus Metadata System employs a rich set of Pydantic schemas to define the structure and validation rules for metadata. These schemas ensure data consistency and provide a clear contract for API interactions.
+
+## Core Schema: TensorDescriptor
+
+The `TensorDescriptor` is the fundamental metadata unit. It captures essential information about a tensor's structure, storage, and identity. Key fields include:
+
+*   `tensor_id`: Unique UUID for the tensor.
+*   `dimensionality`: Number of dimensions.
+*   `shape`: Size of each dimension.
+*   `data_type`: Data type of tensor elements (e.g., `float32`, `int64`).
+*   `storage_format`: How the tensor is physically stored (e.g., `raw`, `numpy_npz`).
+*   `creation_timestamp`, `last_modified_timestamp`.
+*   `owner`, `access_control` (which itself is a nested model detailing read/write permissions).
+*   `byte_size`, `checksum` (e.g., MD5, SHA256 of the tensor data).
+*   `compression_info` (details if tensor data is compressed).
+*   `tags`: A list of arbitrary string tags for categorization.
+*   `metadata`: A flexible dictionary for any other custom key-value metadata.
+
+## Semantic Metadata
+
+Associated with a `TensorDescriptor` (one-to-many, identified by `name` per tensor), this schema describes the meaning and context of the tensor data. Key fields:
+
+*   `tensor_id`: Links to the parent `TensorDescriptor`.
+*   `name`: The specific name of this semantic annotation (e.g., "primary_class_label", "object_bounding_boxes", "feature_description_set").
+*   `description`: Detailed explanation of this semantic annotation.
+*   *(Note: The original broader concept of a single SemanticMetadata object per tensor with fields like `domain`, `purpose`, etc., has evolved. These broader concepts might be captured in `TensorDescriptor.tags`, `TensorDescriptor.metadata`, or specific named `SemanticMetadata` entries.)*
+
+## Extended Metadata Schemas
+
+These provide more detailed and specialized information, typically one-to-one with a `TensorDescriptor`:
+
+*   **`LineageMetadata`**: Tracks origin (`source`), parent tensors (`parent_tensors`), a history of transformations (`transformation_history`), version string (`version`), version control details (`version_control`), and other provenance information (`provenance`).
+    *   `LineageSource`: Details the origin (e.g., file, API, computation).
+    *   `ParentTensorLink`: Links to parent tensors and describes the relationship.
+    *   `TransformationStep`: Describes an operation in the transformation history.
+    *   `VersionControlInfo`: Git-like versioning details for the source or tensor itself.
+
+*   **`ComputationalMetadata`**: Describes how the tensor was computed, including the `algorithm` used, `parameters`, reference to a `computational_graph_ref`, `execution_environment` details, `computation_time_seconds`, and `hardware_info`.
+
+*   **`QualityMetadata`**: Captures information about data quality. Includes:
+    *   `statistics` (`QualityStatistics` model: min, max, mean, std_dev, median, variance, percentiles, histogram).
+    *   `missing_values` (`MissingValuesInfo` model: count, percentage, imputation strategy).
+    *   `outliers` (`OutlierInfo` model: count, percentage, detection method).
+    -   `noise_level`, `confidence_score`, `validation_results` (custom checks), `drift_score`.
+
+*   **`RelationalMetadata`**: Describes relationships to other tensors (`related_tensors` via `RelatedTensorLink`), membership in `collections`, explicit `dependencies` on other tensors, and `dataset_context`.
+
+*   **`UsageMetadata`**: Tracks how the tensor is used:
+    *   `access_history` (list of `UsageAccessRecord` detailing who/what accessed it, when, and how).
+    *   `usage_frequency` (auto-calculated from history or can be set).
+    *   `last_accessed_at` (auto-calculated from history or can be set).
+    *   `application_references` (list of applications or models that use this tensor).
+    *   `purpose` (dictionary describing purposes, e.g. for specific model training).
+
+For the exact field definitions, types, optionality, default values, and validation rules for each schema, please refer to the source code in the `tensorus/metadata/schemas.py` module or consult the schemas provided in the interactive API documentation at `/docs`. The I/O schemas for export/import are defined in `tensorus/metadata/schemas_iodata.py`.

mkdocs.yml

@@ -0,0 +1,11 @@
+site_name: Tensorus Metadata System
+site_description: Documentation for the Tensorus Metadata System.
+nav:
+    - Home: index.md
+    - Installation: installation.md
+    - API Guide: api_guide.md
+    - Metadata Schemas: metadata_schemas.md
+theme: readthedocs
+# Optional: Add repo_url, edit_uri for "Edit on GitHub" links if relevant
+# repo_url: https://github.com/your_username/tensorus
+# edit_uri: edit/main/docs/

pages/1_Dashboard.py

@@ -0,0 +1,125 @@
+# pages/1_Dashboard.py (Modifications for Step 3)
+
+import streamlit as st
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import time
+# Updated imports to use API-backed functions
+from .ui_utils import get_dashboard_metrics, list_all_agents, get_agent_status # MODIFIED
+
+st.set_page_config(page_title="Tensorus Dashboard", layout="wide")
+
+st.title("📊 Operations Dashboard")
+st.caption("Overview of Tensorus datasets and agent activity from API.")
+
+# --- Fetch Data ---
+# Use st.cache_data for API calls that don't need constant updates
+# or manage refresh manually. For simplicity, call directly on rerun/button click.
+metrics_data = None
+agent_list = None # Fetch full agent list for detailed status display
+
+# Button to force refresh
+if st.button("🔄 Refresh Dashboard Data"):
+    # Clear previous cache if any or just proceed to refetch
+    metrics_data = get_dashboard_metrics()
+    agent_list = list_all_agents()
+    st.session_state['dashboard_metrics'] = metrics_data # Store in session state
+    st.session_state['dashboard_agents'] = agent_list
+    st.rerun() # Rerun the script to reflect fetched data
+else:
+    # Try to load from session state or fetch if not present
+    if 'dashboard_metrics' not in st.session_state:
+         st.session_state['dashboard_metrics'] = get_dashboard_metrics()
+    if 'dashboard_agents' not in st.session_state:
+         st.session_state['dashboard_agents'] = list_all_agents()
+
+    metrics_data = st.session_state['dashboard_metrics']
+    agent_list = st.session_state['dashboard_agents']
+
+
+# --- Display Metrics ---
+st.subheader("System Metrics")
+if metrics_data:
+    col1, col2, col3 = st.columns(3)
+    col1.metric("Total Datasets", metrics_data.get('dataset_count', 'N/A'))
+    col2.metric("Total Records (Est.)", f"{metrics_data.get('total_records_est', 0):,}")
+    # Agent status summary from metrics
+    agent_summary = metrics_data.get('agent_status_summary', {})
+    running_agents = agent_summary.get('running', 0) + agent_summary.get('starting', 0)
+    col3.metric("Running Agents", running_agents)
+
+    st.divider()
+
+    # --- Performance Metrics Row ---
+    st.subheader("Performance Indicators (Simulated)")
+    pcol1, pcol2, pcol3, pcol4 = st.columns(4)
+    pcol1.metric("Ingestion Rate (rec/s)", f"{metrics_data.get('data_ingestion_rate', 0.0):.1f}")
+    pcol2.metric("Avg Query Latency (ms)", f"{metrics_data.get('avg_query_latency_ms', 0.0):.1f}")
+    pcol3.metric("Latest RL Reward", f"{metrics_data.get('rl_latest_reward', 'N/A')}")
+    pcol4.metric("Best AutoML Score", f"{metrics_data.get('automl_best_score', 'N/A')}")
+
+else:
+    st.warning("Could not fetch dashboard metrics from the API.")
+
+st.divider()
+
+# --- Agent Status Details ---
+st.subheader("Agent Status")
+if agent_list:
+    num_agents = len(agent_list)
+    cols = st.columns(max(1, num_agents)) # Create columns for agents
+
+    for i, agent_info in enumerate(agent_list):
+        agent_id = agent_info.get('id')
+        with cols[i % len(cols)]: # Distribute agents into columns
+            with st.container(border=True):
+                st.markdown(f"**{agent_info.get('name', 'Unknown Agent')}** (`{agent_id}`)")
+                # Fetch detailed status for more info if needed, or use basic status from list
+                # status_details = get_agent_status(agent_id) # Can make page slower
+                status = agent_info.get('status', 'unknown')
+                status_color = "green" if status in ["running", "starting"] else ("orange" if status in ["stopping"] else ("red" if status in ["error"] else "grey"))
+                st.markdown(f"Status: :{status_color}[**{status.upper()}**]")
+
+                # Display config from the list info
+                with st.expander("Config"):
+                    st.json(agent_info.get('config', {}), expanded=False)
+else:
+    st.warning("Could not fetch agent list from the API.")
+
+st.divider()
+
+# --- Performance Monitoring Chart (Using simulated data from metrics for now) ---
+st.subheader("Performance Monitoring (Placeholder Graph)")
+if metrics_data:
+    # Create some fake historical data for plotting based on current metrics
+    history_len = 20
+    # Use session state to persist some history for smoother simulation
+    if 'sim_history' not in st.session_state:
+        st.session_state['sim_history'] = pd.DataFrame({
+            'Ingestion Rate': np.random.rand(history_len) * metrics_data.get('data_ingestion_rate', 10),
+            'Query Latency': np.random.rand(history_len) * metrics_data.get('avg_query_latency_ms', 100),
+            'RL Reward': np.random.randn(history_len) * 5 + (metrics_data.get('rl_latest_reward', 0) or 0)
+        })
+
+    # Update history with latest point
+    latest_data = pd.DataFrame({
+         'Ingestion Rate': [metrics_data.get('data_ingestion_rate', 0.0)],
+         'Query Latency': [metrics_data.get('avg_query_latency_ms', 0.0)],
+         'RL Reward': [metrics_data.get('rl_latest_reward', 0) or 0] # Handle None
+    })
+    st.session_state['sim_history'] = pd.concat([st.session_state['sim_history'].iloc[1:], latest_data], ignore_index=True)
+
+
+    # Use Plotly for better interactivity
+    try:
+        fig = px.line(st.session_state['sim_history'], title="Simulated Performance Metrics Over Time")
+        fig.update_layout(legend_title_text='Metrics')
+        st.plotly_chart(fig, use_container_width=True)
+    except Exception as e:
+        st.warning(f"Could not display performance chart: {e}")
+
+else:
+     st.info("Performance metrics unavailable.")
+
+st.caption(f"Dashboard data timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metrics_data.get('timestamp', time.time())) if metrics_data else time.time())}")

pages/2_Control_Panel.py

@@ -0,0 +1,138 @@
+# pages/2_Control_Panel.py (Modifications for Step 3)
+
+import streamlit as st
+import time
+import json
+# Use the updated API-backed functions
+from .ui_utils import (
+    list_all_agents,
+    get_agent_status,
+    get_agent_logs,
+    start_agent,
+    stop_agent,
+    update_agent_config,
+)
+
+st.set_page_config(page_title="Agent Control Panel", layout="wide")
+
+st.title("🕹️ Multi-Agent Control Panel")
+st.caption("Manage and monitor Tensorus agents via API.")
+
+# Fetch agent list from API
+agent_list = list_all_agents()
+
+if not agent_list:
+    st.error("Could not fetch agent list from API. Please ensure the backend is running and reachable.")
+    st.stop()
+
+# Create a mapping from name to ID for easier selection
+# Handle potential duplicate names if necessary, though IDs should be unique
+agent_options = {agent['name']: agent['id'] for agent in agent_list}
+# Add ID to name if names aren't unique (optional robustness)
+# agent_options = {f"{agent['name']} ({agent['id']})": agent['id'] for agent in agent_list}
+
+
+selected_agent_name = st.selectbox("Select Agent:", options=agent_options.keys())
+
+if selected_agent_name:
+    selected_agent_id = agent_options[selected_agent_name]
+    st.divider()
+    st.subheader(f"Control: {selected_agent_name} (`{selected_agent_id}`)")
+
+    # Use session state to store fetched status and logs for the selected agent
+    # This avoids refetching constantly unless a refresh is triggered
+    agent_state_key = f"agent_status_{selected_agent_id}"
+    agent_logs_key = f"agent_logs_{selected_agent_id}"
+
+    # Button to force refresh status and logs
+    if st.button(f"🔄 Refresh Status & Logs##{selected_agent_id}"): # Unique key per agent
+        st.session_state[agent_state_key] = get_agent_status(selected_agent_id)
+        st.session_state[agent_logs_key] = get_agent_logs(selected_agent_id)
+        st.rerun() # Rerun to display refreshed data
+
+    # Fetch status if not in session state or refresh button wasn't just clicked
+    if agent_state_key not in st.session_state:
+        st.session_state[agent_state_key] = get_agent_status(selected_agent_id)
+
+    status_info = st.session_state[agent_state_key]
+
+    if status_info:
+        status = status_info.get('status', 'unknown')
+        status_color = "green" if status in ["running", "starting"] else ("orange" if status in ["stopping"] else ("red" if status in ["error"] else "grey"))
+        st.markdown(f"Current Status: :{status_color}[**{status.upper()}**]")
+        last_log_ts = status_info.get('last_log_timestamp')
+        if last_log_ts:
+            st.caption(f"Last Log Entry (approx.): {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last_log_ts))}")
+    else:
+        st.error(f"Could not retrieve status for agent '{selected_agent_name}'.")
+
+    # Control Buttons (Now call API functions)
+    col1, col2, col3 = st.columns([1, 1, 5])
+    is_running = status_info and status_info.get('status') == 'running'
+    is_stopped = status_info and status_info.get('status') == 'stopped'
+
+    with col1:
+        start_disabled = not is_stopped # Disable if not stopped
+        if st.button("▶️ Start", key=f"start_{selected_agent_id}", disabled=start_disabled):
+            if start_agent(selected_agent_id): # API call returns success/fail
+                # Trigger refresh after short delay to allow backend state change (optimistic)
+                time.sleep(1.0)
+                # Clear state cache and rerun
+                if agent_state_key in st.session_state: del st.session_state[agent_state_key]
+                if agent_logs_key in st.session_state: del st.session_state[agent_logs_key]
+                st.rerun()
+    with col2:
+        stop_disabled = not is_running # Disable if not running
+        if st.button("⏹️ Stop", key=f"stop_{selected_agent_id}", disabled=stop_disabled):
+            if stop_agent(selected_agent_id): # API call returns success/fail
+                time.sleep(1.0)
+                if agent_state_key in st.session_state: del st.session_state[agent_state_key]
+                if agent_logs_key in st.session_state: del st.session_state[agent_logs_key]
+                st.rerun()
+
+    st.divider()
+
+    # Configuration & Logs
+    tab1, tab2 = st.tabs(["Configuration", "Logs"])
+
+    with tab1:
+        if status_info and 'config' in status_info:
+            current_config = status_info['config']
+            st.write("Current configuration:")
+            st.json(current_config)
+
+            with st.expander("Edit configuration"):
+                form = st.form(key=f"cfg_form_{selected_agent_id}")
+                config_text = form.text_area(
+                    "Configuration JSON",
+                    value=json.dumps(current_config, indent=2),
+                    height=200,
+                )
+                submitted = form.form_submit_button("Update")
+                if submitted:
+                    try:
+                        new_cfg = json.loads(config_text)
+                        if update_agent_config(selected_agent_id, new_cfg):
+                            if agent_state_key in st.session_state:
+                                del st.session_state[agent_state_key]
+                            time.sleep(0.5)
+                            st.rerun()
+                    except json.JSONDecodeError as e:
+                        form.error(f"Invalid JSON: {e}")
+        else:
+            st.warning("Configuration not available.")
+
+    with tab2:
+        st.write("Recent logs (fetched from API):")
+        # Fetch logs if not in session state
+        if agent_logs_key not in st.session_state:
+            st.session_state[agent_logs_key] = get_agent_logs(selected_agent_id)
+
+        logs = st.session_state[agent_logs_key]
+        if logs is not None:
+            st.code("\n".join(logs), language="log")
+        else:
+            st.error("Could not retrieve logs.")
+
+else:
+    st.info("Select an agent from the dropdown above.")

pages/3_NQL_Chatbot.py

@@ -0,0 +1,94 @@
+# pages/3_NQL_Chatbot.py
+
+import streamlit as st
+from .ui_utils import execute_nql_query # MODIFIED
+import pandas as pd # Added import for pandas
+
+st.set_page_config(page_title="NQL Chatbot", layout="wide")
+
+st.title("💬 Natural Query Language (NQL) Chatbot")
+st.caption("Query Tensorus datasets using natural language.")
+st.info("Backend uses Regex-based NQL Agent. LLM integration is future work.")
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+        if "results" in message and message["results"] is not None and not message["results"].empty: # Check if DataFrame is not None and not empty
+            st.dataframe(message["results"], use_container_width=True) # Display results as dataframe
+        elif "error" in message:
+            st.error(message["error"])
+
+
+# React to user input
+if prompt := st.chat_input("Enter your query (e.g., 'get all data from my_dataset')"):
+    # Display user message in chat message container
+    st.chat_message("user").markdown(prompt)
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+
+    # Get assistant response from NQL Agent API
+    with st.spinner("Processing query..."):
+        nql_response = execute_nql_query(prompt)
+
+        response_content = ""
+        results_df = None
+        error_msg = None
+
+        if nql_response:
+            response_content = nql_response.get("message", "Error processing response.")
+            if nql_response.get("success"):
+                results_list = nql_response.get("results")
+                if results_list:
+                    # Convert results list (containing dicts with 'metadata', 'shape', etc.) to DataFrame
+                    # Extract relevant fields for display
+                    display_data = []
+                    for res in results_list:
+                        row = {
+                            "record_id": res["metadata"].get("record_id"),
+                            "shape": str(res.get("shape")), # Convert shape list to string
+                            "dtype": res.get("dtype"),
+                            **res["metadata"] # Flatten metadata into columns
+                        }
+                         # Remove potentially large 'tensor' data from direct display
+                        row.pop('tensor', None)
+                        # Avoid duplicate metadata keys if also present at top level
+                        row.pop('shape', None)
+                        row.pop('dtype', None)
+                        row.pop('record_id', None)
+                        display_data.append(row)
+
+                    if display_data:
+                         results_df = pd.DataFrame(display_data)
+
+                # Augment message if results found
+                count = nql_response.get("count")
+                if count is not None:
+                    response_content += f" Found {count} record(s)."
+
+            else:
+                # NQL agent indicated failure (parsing or execution)
+                error_msg = response_content # Use the message as the error
+
+        else:
+            # API call itself failed (connection error, etc.)
+            response_content = "Failed to get response from the NQL agent."
+            error_msg = response_content
+
+    # Display assistant response in chat message container
+    message_data = {"role": "assistant", "content": response_content}
+    with st.chat_message("assistant"):
+        st.markdown(response_content)
+        if results_df is not None and not results_df.empty: # Check if DataFrame is not None and not empty
+             st.dataframe(results_df, use_container_width=True)
+             message_data["results"] = results_df # Store for history display if needed (might be large)
+        elif error_msg:
+             st.error(error_msg)
+             message_data["error"] = error_msg
+
+    # Add assistant response to chat history
+    st.session_state.messages.append(message_data)

pages/4_Data_Explorer.py

@@ -0,0 +1,167 @@
+# pages/4_Data_Explorer.py
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+from .ui_utils import list_datasets, fetch_dataset_data # MODIFIED
+import torch # Needed if we want to recreate tensors for inspection/plotting
+
+st.set_page_config(page_title="Data Explorer", layout="wide")
+
+st.title("🔍 Interactive Data Explorer")
+st.caption("Browse, filter, and visualize Tensorus datasets.")
+
+# --- Dataset Selection ---
+datasets = list_datasets()
+if not datasets:
+    st.warning("No datasets found or API connection failed. Cannot explore data.")
+    st.stop() # Stop execution if no datasets
+
+selected_dataset = st.selectbox("Select Dataset:", datasets)
+
+# --- Data Fetching & Filtering ---
+if selected_dataset:
+    st.subheader(f"Exploring: {selected_dataset}")
+
+    PAGE_SIZE = 20
+    page_key = f"page_{selected_dataset}"
+    if page_key not in st.session_state:
+        st.session_state[page_key] = 0
+
+    page = st.session_state[page_key]
+    offset = page * PAGE_SIZE
+    records = fetch_dataset_data(selected_dataset, offset=offset, limit=PAGE_SIZE)
+
+    prev_disabled = page == 0
+    next_disabled = records is None or len(records) < PAGE_SIZE
+
+    col_prev, col_next = st.columns(2)
+    with col_prev:
+        if st.button("Previous", disabled=prev_disabled, key="prev_btn"):
+            st.session_state[page_key] = max(0, page - 1)
+            st.experimental_rerun()
+    with col_next:
+        if st.button("Next", disabled=next_disabled, key="next_btn"):
+            st.session_state[page_key] = page + 1
+            st.experimental_rerun()
+
+    if records is None:
+        st.error("Failed to fetch data for the selected dataset.")
+        st.stop()
+    elif not records:
+        st.info("Selected dataset is empty.")
+        st.stop()
+
+    start_idx = offset + 1
+    end_idx = offset + len(records)
+    st.info(f"Displaying records {start_idx} - {end_idx} (page {page + 1})")
+
+    # Create DataFrame from metadata for filtering/display
+    metadata_list = [r['metadata'] for r in records]
+    df_meta = pd.DataFrame(metadata_list)
+
+    # --- Metadata Filtering UI ---
+    st.sidebar.header("Filter by Metadata")
+    filter_cols = st.sidebar.multiselect("Select metadata columns to filter:", options=df_meta.columns.tolist())
+
+    filtered_df = df_meta.copy()
+    for col in filter_cols:
+        unique_values = filtered_df[col].dropna().unique().tolist()
+        if pd.api.types.is_numeric_dtype(filtered_df[col]):
+            # Numeric filter (slider)
+            min_val, max_val = float(filtered_df[col].min()), float(filtered_df[col].max())
+            if min_val < max_val:
+                selected_range = st.sidebar.slider(f"Filter {col}:", min_val, max_val, (min_val, max_val))
+                filtered_df = filtered_df[filtered_df[col].between(selected_range[0], selected_range[1])]
+            else:
+                st.sidebar.caption(f"{col}: Single numeric value ({min_val}), no range filter.")
+
+        elif len(unique_values) > 0 and len(unique_values) <= 20: # Limit dropdown options
+            # Categorical filter (multiselect)
+             selected_values = st.sidebar.multiselect(f"Filter {col}:", options=unique_values, default=unique_values)
+             if selected_values: # Only filter if some values are selected
+                  filtered_df = filtered_df[filtered_df[col].isin(selected_values)]
+             else: # If user deselects everything, show nothing
+                  filtered_df = filtered_df[filtered_df[col].isnull()] # Hack to get empty DF matching columns
+
+        else:
+            st.sidebar.text_input(f"Filter {col} (Text contains):", key=f"text_{col}")
+            search_term = st.session_state.get(f"text_{col}", "").lower()
+            if search_term:
+                # Ensure column is string type before using .str.contains
+                filtered_df = filtered_df[filtered_df[col].astype(str).str.lower().str.contains(search_term, na=False)]
+
+
+    st.divider()
+    st.subheader("Filtered Data View")
+    st.write(f"{len(filtered_df)} records matching filters.")
+    st.dataframe(filtered_df, use_container_width=True)
+
+    # --- Tensor Preview & Visualization ---
+    st.divider()
+    st.subheader("Tensor Preview")
+
+    if not filtered_df.empty:
+        # Allow selecting a record ID from the filtered results
+        record_ids = filtered_df['record_id'].tolist()
+        selected_record_id = st.selectbox("Select Record ID to Preview Tensor:", record_ids)
+
+        if selected_record_id:
+            # Find the full record data corresponding to the selected ID
+            selected_record = next((r for r in records if r['metadata'].get('record_id') == selected_record_id), None)
+
+            if selected_record:
+                st.write("Metadata:")
+                st.json(selected_record['metadata'])
+
+                shape = selected_record.get("shape")
+                dtype = selected_record.get("dtype")
+                data_list = selected_record.get("data")
+
+                st.write(f"Tensor Info: Shape={shape}, Dtype={dtype}")
+
+                try:
+                     # Recreate tensor for potential plotting/display
+                     # Be careful with large tensors in Streamlit UI!
+                     # We might only want to show info or small slices.
+                     if shape and dtype and data_list is not None:
+                          tensor = torch.tensor(data_list, dtype=getattr(torch, dtype, torch.float32)) # Use getattr for dtype
+                          st.write("Tensor Data (first few elements):")
+                          st.code(f"{tensor.flatten()[:10].numpy()}...") # Show flattened start
+
+                          # --- Simple Visualizations ---
+                          if tensor.ndim == 1 and tensor.numel() > 1:
+                               st.line_chart(tensor.numpy())
+                          elif tensor.ndim == 2 and tensor.shape[0] > 1 and tensor.shape[1] > 1 :
+                               # Simple heatmap using plotly (requires plotly)
+                               try:
+                                   fig = px.imshow(tensor.numpy(), title="Tensor Heatmap", aspect="auto")
+                                   st.plotly_chart(fig, use_container_width=True)
+                               except Exception as plot_err:
+                                   st.warning(f"Could not generate heatmap: {plot_err}")
+                          elif tensor.ndim == 3 and tensor.shape[0] in [1, 3]: # Basic image check (C, H, W) or (1, H, W)
+                               try:
+                                   # Permute if needed (e.g., C, H, W -> H, W, C for display)
+                                   if tensor.shape[0] in [1, 3]:
+                                       display_tensor = tensor.permute(1, 2, 0).squeeze() # H, W, C or H, W
+                                       # Clamp/normalize data to display range [0, 1] or [0, 255] - basic attempt
+                                       display_tensor = (display_tensor - display_tensor.min()) / (display_tensor.max() - display_tensor.min() + 1e-6)
+                                       st.image(display_tensor.numpy(), caption="Tensor as Image (Attempted)", use_column_width=True)
+                               except ImportError:
+                                    st.warning("Pillow needed for image display (`pip install Pillow`)")
+                               except Exception as img_err:
+                                    st.warning(f"Could not display tensor as image: {img_err}")
+                          else:
+                               st.info("No specific visualization available for this tensor shape/dimension.")
+
+                     else:
+                          st.warning("Tensor data, shape, or dtype missing in the record.")
+
+                except Exception as tensor_err:
+                    st.error(f"Error processing tensor data for preview: {tensor_err}")
+            else:
+                 st.warning("Selected record details not found (this shouldn't happen).")
+        else:
+            st.info("Select a record ID above to preview its tensor.")
+    else:
+        st.info("No records match the current filters.")

pages/5_API_Playground.py

@@ -0,0 +1,57 @@
+# pages/5_API_Playground.py
+
+import streamlit as st
+import streamlit.components.v1 as components
+from .ui_utils import TENSORUS_API_URL, get_api_status # MODIFIED
+
+st.set_page_config(page_title="API Playground", layout="wide")
+
+st.title("🚀 API Playground & Documentation Hub")
+st.caption("Explore and interact with the Tensorus REST API.")
+
+# Check if API is running
+api_running = get_api_status()
+
+if not api_running:
+    st.error(
+        f"The Tensorus API backend does not seem to be running at {TENSORUS_API_URL}. "
+        "Please start the backend (`uvicorn api:app --reload`) to use the API Playground."
+    )
+    st.stop() # Stop execution if API is not available
+else:
+    st.success(f"Connected to API backend at {TENSORUS_API_URL}")
+
+st.markdown(
+    f"""
+    This section provides live, interactive documentation for the Tensorus API,
+    powered by FastAPI's OpenAPI integration. You can explore endpoints,
+    view schemas, and even try out API calls directly in your browser.
+
+    * **Swagger UI:** A graphical interface for exploring and testing API endpoints.
+    * **ReDoc:** Alternative documentation format, often preferred for reading.
+
+    Select a view below:
+    """
+)
+
+# Use tabs to embed Swagger and ReDoc
+tab1, tab2 = st.tabs(["Swagger UI", "ReDoc"])
+
+# Construct the documentation URLs based on the API base URL
+swagger_url = f"{TENSORUS_API_URL}/docs"
+redoc_url = f"{TENSORUS_API_URL}/redoc"
+
+with tab1:
+    st.subheader("Swagger UI")
+    st.markdown(f"Explore the API interactively. [Open in new tab]({swagger_url})")
+    # Embed Swagger UI using an iframe
+    components.iframe(swagger_url, height=800, scrolling=True)
+
+with tab2:
+    st.subheader("ReDoc")
+    st.markdown(f"View the API documentation. [Open in new tab]({redoc_url})")
+    # Embed ReDoc using an iframe
+    components.iframe(redoc_url, height=800, scrolling=True)
+
+st.divider()
+st.caption("Note: Ensure the Tensorus API backend is running to interact with the playground.")

pages/6_Financial_Forecast_Demo.py

@@ -0,0 +1,236 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import plotly.graph_objects as go # Using graph_objects for more control
+import time
+import torch
+
+# Assuming tensorus is in PYTHONPATH or installed
+from tensorus.tensor_storage import TensorStorage
+from tensorus.financial_data_generator import generate_financial_data
+from tensorus.time_series_predictor import load_time_series_from_tensorus, train_arima_and_predict, store_predictions_to_tensorus
+
+# Attempt to load shared CSS
+try:
+    from pages.pages_shared_utils import load_css
+    LOAD_CSS_AVAILABLE = True
+except ImportError:
+    LOAD_CSS_AVAILABLE = False
+    def load_css(): # Dummy function
+        st.markdown("<style>/* No shared CSS found */</style>", unsafe_allow_html=True)
+        # st.warning("Could not load shared CSS from pages.pages_shared_utils.") # Keep UI cleaner
+
+# Page Configuration
+st.set_page_config(page_title="Financial Forecast Demo", layout="wide")
+if LOAD_CSS_AVAILABLE:
+    load_css()
+
+# --- Constants ---
+RAW_DATASET_NAME = "financial_raw_data"
+PREDICTION_DATASET_NAME = "financial_predictions"
+TIME_SERIES_NAME = "synthetic_stock_close"
+# Use a demo-specific storage path to avoid conflicts with main app
+TENSOR_STORAGE_PATH = "tensor_data_financial_demo"
+
+
+def get_tensor_storage_instance():
+    """Initializes and returns a TensorStorage instance for the demo."""
+    return TensorStorage(storage_path=TENSOR_STORAGE_PATH)
+
+def show_financial_demo_page():
+    """
+    Main function to display the Streamlit page for the financial demo.
+    """
+    st.title("📈 Financial Time Series Forecasting Demo")
+    st.markdown("""
+    This demo showcases time series forecasting using an ARIMA model on synthetically generated 
+    financial data. Data is generated, stored in Tensorus, loaded for model training, 
+    and predictions are then stored back and visualized.
+    """)
+
+    # Initialize session state variables
+    if 'raw_financial_df' not in st.session_state:
+        st.session_state.raw_financial_df = None
+    if 'loaded_historical_series' not in st.session_state:
+        st.session_state.loaded_historical_series = None
+    if 'predictions_series' not in st.session_state:
+        st.session_state.predictions_series = None
+    if 'arima_p' not in st.session_state: # For number_input persistence
+        st.session_state.arima_p = 5
+    if 'arima_d' not in st.session_state:
+        st.session_state.arima_d = 1
+    if 'arima_q' not in st.session_state:
+        st.session_state.arima_q = 0
+    if 'n_predictions' not in st.session_state:
+        st.session_state.n_predictions = 30
+
+
+    # --- Section 1: Data Generation & Ingestion ---
+    st.header("1. Data Generation & Ingestion")
+    if st.button("Generate & Ingest Sample Financial Data", key="generate_data_button"):
+        with st.spinner("Generating and ingesting data..."):
+            try:
+                storage = get_tensor_storage_instance()
+                
+                # Generate sample financial data
+                df = generate_financial_data(days=365, initial_price=150, trend_slope=0.1, seasonality_amplitude=20)
+                st.session_state.raw_financial_df = df # Store for immediate plotting
+
+                # Ensure dataset exists
+                if RAW_DATASET_NAME not in storage.list_datasets():
+                    storage.create_dataset(RAW_DATASET_NAME)
+                    st.info(f"Dataset '{RAW_DATASET_NAME}' created.")
+                else:
+                    st.info(f"Dataset '{RAW_DATASET_NAME}' already exists. Using existing.")
+
+                # Prepare data for TensorStorage
+                series_to_store = torch.tensor(df['Close'].values, dtype=torch.float32)
+                dates_for_metadata = df['Date'].dt.strftime('%Y-%m-%d').tolist()
+                metadata = {
+                    "name": TIME_SERIES_NAME, 
+                    "dates": dates_for_metadata,
+                    "source": "financial_demo_generator",
+                    "description": f"Daily closing prices for {TIME_SERIES_NAME}"
+                }
+
+                record_id = storage.insert(RAW_DATASET_NAME, series_to_store, metadata)
+                if record_id:
+                    st.success(f"Data for '{TIME_SERIES_NAME}' ingested into '{RAW_DATASET_NAME}' with Record ID: {record_id}")
+                    # Also load it into loaded_historical_series for consistency in the workflow
+                    st.session_state.loaded_historical_series = pd.Series(df['Close'].values, index=pd.to_datetime(df['Date']), name=TIME_SERIES_NAME)
+                    st.session_state.predictions_series = None # Clear previous predictions
+                else:
+                    st.error("Failed to ingest data into Tensorus.")
+            except Exception as e:
+                st.error(f"Error during data generation/ingestion: {e}")
+
+    # --- Section 2: Historical Data Visualization ---
+    st.header("2. Historical Data Visualization")
+    if st.button("Load and View Historical Data from Tensorus", key="load_historical_button"):
+        with st.spinner(f"Loading '{TIME_SERIES_NAME}' from Tensorus..."):
+            try:
+                storage = get_tensor_storage_instance()
+                loaded_series = load_time_series_from_tensorus(
+                    storage, 
+                    RAW_DATASET_NAME, 
+                    series_metadata_field="name", 
+                    series_name=TIME_SERIES_NAME, 
+                    date_metadata_field="dates"
+                )
+                if loaded_series is not None:
+                    st.session_state.loaded_historical_series = loaded_series
+                    st.session_state.predictions_series = None # Clear previous predictions
+                    st.success(f"Successfully loaded '{TIME_SERIES_NAME}' from Tensorus.")
+                else:
+                    st.warning(f"Could not find or load '{TIME_SERIES_NAME}' from Tensorus dataset '{RAW_DATASET_NAME}'. Generate data first if it's not there.")
+            except Exception as e:
+                st.error(f"Error loading historical data: {e}")
+
+    if st.session_state.loaded_historical_series is not None:
+        st.subheader(f"Historical Data: {st.session_state.loaded_historical_series.name}")
+        fig_hist = go.Figure()
+        fig_hist.add_trace(go.Scatter(
+            x=st.session_state.loaded_historical_series.index, 
+            y=st.session_state.loaded_historical_series.values, 
+            mode='lines', 
+            name='Historical Close'
+        ))
+        fig_hist.update_layout(title="Historical Stock Prices", xaxis_title="Date", yaxis_title="Price")
+        st.plotly_chart(fig_hist, use_container_width=True)
+    else:
+        st.info("Generate or load data to view historical prices.")
+
+
+    # --- Section 3: ARIMA Model Prediction ---
+    st.header("3. ARIMA Model Prediction")
+    
+    col1, col2, col3, col4 = st.columns(4)
+    with col1:
+        st.number_input("ARIMA Order (p)", min_value=0, max_value=10, key="arima_p", help="Autoregressive order.")
+    with col2:
+        st.number_input("ARIMA Order (d)", min_value=0, max_value=3, key="arima_d", help="Differencing order.")
+    with col3:
+        st.number_input("ARIMA Order (q)", min_value=0, max_value=10, key="arima_q", help="Moving average order.")
+    with col4:
+        st.number_input("Number of Future Predictions", min_value=1, max_value=180, key="n_predictions", help="Number of future days to predict.")
+
+    if st.button("Run ARIMA Prediction", key="run_arima_button"):
+        if st.session_state.loaded_historical_series is None:
+            st.error("No historical data loaded. Please generate or load data first (Sections 1 or 2).")
+        else:
+            with st.spinner("Running ARIMA prediction... This might take a moment."):
+                try:
+                    p, d, q = st.session_state.arima_p, st.session_state.arima_d, st.session_state.arima_q
+                    n_preds = st.session_state.n_predictions
+                    
+                    predictions = train_arima_and_predict(
+                        series=st.session_state.loaded_historical_series,
+                        arima_order=(p, d, q),
+                        n_predictions=n_preds
+                    )
+                    
+                    if predictions is not None:
+                        st.session_state.predictions_series = predictions
+                        st.success(f"ARIMA prediction complete for {n_preds} steps.")
+                        
+                        # Store predictions
+                        storage = get_tensor_storage_instance()
+                        model_details_dict = {"type": "ARIMA", "order": (p,d,q), "parameters_estimated": True} # Example details
+                        
+                        pred_record_id = store_predictions_to_tensorus(
+                            storage, 
+                            predictions, 
+                            PREDICTION_DATASET_NAME, 
+                            original_series_name=st.session_state.loaded_historical_series.name, 
+                            model_details=model_details_dict
+                        )
+                        if pred_record_id:
+                           st.info(f"Predictions stored in Tensorus dataset '{PREDICTION_DATASET_NAME}' with Record ID: {pred_record_id}")
+                        else:
+                           st.warning("Failed to store predictions in Tensorus.")
+                    else:
+                        st.error("Failed to generate ARIMA predictions. Check model parameters, data, or logs.")
+                except Exception as e:
+                    st.error(f"Error during ARIMA prediction: {e}")
+
+    # --- Section 4: Prediction Results & Visualization ---
+    st.header("4. Prediction Results")
+    if st.session_state.predictions_series is not None and st.session_state.loaded_historical_series is not None:
+        st.subheader("Historical Data and Predictions")
+        
+        fig_combined = go.Figure()
+        fig_combined.add_trace(go.Scatter(
+            x=st.session_state.loaded_historical_series.index, 
+            y=st.session_state.loaded_historical_series.values, 
+            mode='lines', 
+            name='Historical Close',
+            line=dict(color='blue')
+        ))
+        fig_combined.add_trace(go.Scatter(
+            x=st.session_state.predictions_series.index, 
+            y=st.session_state.predictions_series.values, 
+            mode='lines', 
+            name='Predicted Close',
+            line=dict(color='red', dash='dash')
+        ))
+        fig_combined.update_layout(
+            title=f"{st.session_state.loaded_historical_series.name}: Historical vs. Predicted", 
+            xaxis_title="Date", 
+            yaxis_title="Price"
+        )
+        st.plotly_chart(fig_combined, use_container_width=True)
+
+        st.subheader("Predicted Values (Next {} Days)".format(len(st.session_state.predictions_series)))
+        # Display predictions as a DataFrame, ensure index is named 'Date' if it's a DatetimeIndex
+        pred_df_display = st.session_state.predictions_series.to_frame()
+        if isinstance(pred_df_display.index, pd.DatetimeIndex):
+            pred_df_display.index.name = "Date"
+        st.dataframe(pred_df_display)
+
+    elif st.session_state.loaded_historical_series is not None:
+        st.info("Run ARIMA Prediction in Section 3 to see forecasted results.")
+    else:
+        st.info("Generate or load data and run predictions to see results here.")
+
+if __name__ == "__main__":
+    show_financial_demo_page()

pages/api_playground_v2.py

@@ -0,0 +1,171 @@
+# pages/api_playground_v2.py
+
+import streamlit as st
+import streamlit.components.v1 as components
+
+# Import from the shared utils for pages
+try:
+    from pages.pages_shared_utils import (
+        load_css as load_shared_css,
+        API_BASE_URL, # Use this instead of TENSORUS_API_URL
+        get_api_status
+    )
+except ImportError:
+    st.error("Critical Error: Could not import `pages_shared_utils`. Page cannot function.")
+    def load_shared_css(): pass
+    API_BASE_URL = "http://127.0.0.1:7860" # Fallback
+    def get_api_status(): 
+        st.error("`get_api_status` unavailable.")
+        return False, {"error": "Setup issue"} # Simulate API down
+    st.stop()
+
+st.set_page_config(page_title="API Playground (V2)", layout="wide")
+load_shared_css() # Load common CSS from shared utilities
+
+# Custom CSS for API Playground page
+# These styles are specific to this page and enhance the shared theme.
+st.markdown("""
+<style>
+    /* API Playground specific styles */
+    /* Using classes for titles and captions allows for more specific styling if needed,
+       while still inheriting base styles from shared CSS (h1, .stCaption etc.) */
+    .main-title { 
+        color: #e0e0ff !important; /* Ensure it uses the desired dashboard title color */
+        text-align: center;
+        margin-bottom: 0.5rem; 
+        font-weight: bold;
+    }
+    .main-caption { 
+        color: #c0c0ff !important; /* Consistent caption color */
+        text-align: center;
+        margin-bottom: 2rem;
+    }
+
+    /* Status message styling - these could be potentially moved to shared_utils if used elsewhere */
+    /* For now, keeping them page-specific as they are styled slightly differently from .status-indicator */
+    .status-message-shared { /* Base for status messages on this page */
+        padding: 0.75rem;
+        border-radius: 5px;
+        margin-bottom: 1rem;
+        text-align: center; /* Center align text in status messages */
+    }
+    .status-success { /* Success status message, inherits from .status-message-shared */
+        color: #4CAF50; /* Green text */
+        background-color: rgba(76, 175, 80, 0.1); /* Light green background */
+        border: 1px solid rgba(76, 175, 80, 0.3); /* Light green border */
+    }
+    .status-error { /* Error status message, inherits from .status-message-shared */
+        color: #F44336; /* Red text */
+        background-color: rgba(244, 67, 54, 0.1); /* Light red background */
+        border: 1px solid rgba(244, 67, 54, 0.3); /* Light red border */
+    }
+    .status-error code, .status-success code { /* Style <code> tags within status messages */
+        background-color: rgba(0,0,0,0.1);
+        padding: 0.1em 0.3em;
+        border-radius: 3px;
+    }
+
+    /* Styling for the Streamlit Tabs component */
+    .tab-container .stTabs [data-baseweb="tab-list"] { 
+        background-color: transparent; /* Remove default Streamlit tab bar background if any */
+        gap: 5px; /* Increased gap between tab headers */
+        border-bottom: 1px solid #2a3f5c; /* Match other dividers */
+    }
+    .tab-container .stTabs [data-baseweb="tab"] { 
+        background-color: #18223f; /* Match common-card background for inactive tabs */
+        color: #c0c0ff; /* Standard text color for inactive tabs */
+        border-radius: 5px 5px 0 0; /* Rounded top corners */
+        padding: 0.75rem 1.5rem;
+        border: 1px solid #2a3f5c; /* Border for inactive tabs */
+        border-bottom: none; /* Remove bottom border as it's handled by tab-list */
+        margin-bottom: -1px; /* Overlap with tab-list border */
+    }
+    .tab-container .stTabs [data-baseweb="tab"][aria-selected="true"] { 
+        background-color: #2a2f4c; /* Slightly lighter for active tab, similar to nav hover */
+        color: #ffffff; /* White text for active tab */
+        font-weight: bold;
+        border-color: #3a6fbf; /* Accent color for active tab border */
+    }
+
+    /* Container for the iframes displaying Swagger/ReDoc */
+    .iframe-container {
+        border: 1px solid #2a3f5c; /* Consistent border */
+        border-radius: 0 5px 5px 5px; /* Rounded corners, except top-left to align with tabs */
+        overflow: hidden; /* Ensures iframe respects border radius */
+        margin-top: -1px; /* Align with tab-list border */
+        height: 800px; /* Default height */
+    }
+</style>
+""", unsafe_allow_html=True)
+
+# Page Title and Caption, using custom classes for styling
+st.markdown('<h1 class="main-title">🚀 API Playground & Documentation Hub</h1>', unsafe_allow_html=True)
+st.markdown('<p class="main-caption">Explore and interact with the Tensorus REST API directly.</p>', unsafe_allow_html=True)
+
+
+# Check if API is running using the shared utility function.
+api_ok, api_info = get_api_status()
+
+# Display API status message.
+if not api_ok:
+    # If API is not reachable, display an error message and stop page execution.
+    st.markdown(
+        f"""
+        <div class="status-message-shared status-error">
+            <strong>API Connection Error:</strong> The Tensorus API backend does not seem to be running or reachable at <code>{API_BASE_URL}</code>.
+            <br>Please ensure the backend (<code>uvicorn api:app --reload</code>) is active to use the API Playground.
+        </div>
+        """, unsafe_allow_html=True
+    )
+    st.stop() # Halt further rendering of the page.
+else:
+    # If API is reachable, display a success message with API version.
+    api_version = api_info.get("version", "N/A") # Get API version from status info.
+    st.markdown(
+        f"""
+        <div class="status-message-shared status-success">
+            Successfully connected to Tensorus API v{api_version} at <code>{API_BASE_URL}</code>.
+        </div>
+        """, unsafe_allow_html=True
+    )
+
+# Introductory text for the API Playground.
+st.markdown(
+    """
+    This section provides live, interactive documentation for the Tensorus API,
+    powered by FastAPI's OpenAPI integration. You can explore endpoints,
+    view schemas, and even try out API calls directly in your browser.
+    """
+)
+
+# Use tabs to embed Swagger UI and ReDoc for API documentation.
+# Wrapping tabs in a div to apply specific tab styling if needed.
+st.markdown('<div class="tab-container">', unsafe_allow_html=True) 
+tab1, tab2 = st.tabs(["Swagger UI", "ReDoc"])
+st.markdown('</div>', unsafe_allow_html=True)
+
+
+# Construct the URLs for Swagger and ReDoc based on the API_BASE_URL from shared utils.
+swagger_url = f"{API_BASE_URL}/docs"
+redoc_url = f"{API_BASE_URL}/redoc"
+
+# Swagger UI Tab
+with tab1:
+    st.subheader("Swagger UI") # Styled by shared CSS
+    st.markdown(f"Explore the API interactively. [Open Swagger UI in new tab]({swagger_url})")
+    # Embed Swagger UI using an iframe within a styled container.
+    st.markdown('<div class="iframe-container">', unsafe_allow_html=True)
+    components.iframe(swagger_url, height=800, scrolling=True)
+    st.markdown('</div>', unsafe_allow_html=True)
+
+# ReDoc Tab
+with tab2:
+    st.subheader("ReDoc") # Styled by shared CSS
+    st.markdown(f"View the API documentation. [Open ReDoc in new tab]({redoc_url})")
+    # Embed ReDoc using an iframe within a styled container.
+    st.markdown('<div class="iframe-container">', unsafe_allow_html=True)
+    components.iframe(redoc_url, height=800, scrolling=True)
+    st.markdown('</div>', unsafe_allow_html=True)
+
+st.divider() # Visual separator.
+st.caption("Note: The API backend must be running and accessible to fully utilize the interactive documentation.")

pages/control_panel_v2.py

@@ -0,0 +1,300 @@
+# pages/control_panel_v2.py
+
+import streamlit as st
+import time
+
+# Import from the newly created shared utils for pages
+try:
+    from pages.pages_shared_utils import (
+        get_agent_status,
+        start_agent,
+        stop_agent,
+        load_css as load_shared_css
+    )
+except ImportError:
+    st.error("Critical Error: Could not import `pages_shared_utils`. Page cannot function.")
+    def get_agent_status(): st.error("`get_agent_status` unavailable."); return None
+    def start_agent(agent_id): st.error(f"Start action for {agent_id} unavailable."); return False
+    def stop_agent(agent_id): st.error(f"Stop action for {agent_id} unavailable."); return False
+    def load_shared_css(): pass
+    st.stop()
+
+
+st.set_page_config(page_title="Agent Control Tower (V2)", layout="wide")
+load_shared_css() # Load shared styles first. This is crucial for base styling.
+
+# Custom CSS for Agent Cards on this page.
+# These styles complement or override the .common-card style from shared_utils.
+# Note: .common-card styles (background, border, padding, shadow) are inherited.
+st.markdown("""
+<style>
+    /* Agent Card specific styling */
+    .agent-card { /* This class is applied along with .common-card */
+        /* Example: if agent cards needed a slightly different background or padding than common-card */
+        /* background-color: #1c2849; */ /* Slightly different background if needed */
+    }
+    .agent-card-header { /* Header section within an agent card */
+        display: flex;
+        align-items: center;
+        margin-bottom: 1rem; 
+        padding-bottom: 0.75rem;
+        border-bottom: 1px solid #2a3f5c; /* Consistent border color */
+    }
+    .agent-card-header .icon { /* Icon in the header */
+        font-size: 2em; 
+        margin-right: 0.75rem;
+        color: #9090ff; /* Specific icon color for agent cards */
+    }
+    .agent-card-header h3 { /* Agent name in the header */
+        /* Inherits .common-card h3 styles primarily, ensuring consistency */
+        color: #d0d0ff !important; /* Override if a brighter color is needed than common-card h3 */
+        font-size: 1.4em !important;
+        margin: 0 !important; /* Remove any default margins */
+    }
+    .agent-card .status-text { 
+        font-weight: bold !important; /* Ensure status text is bold */
+        display: inline-block; /* Allows padding and consistent look */
+        padding: 0.2em 0.5em;
+        border-radius: 4px;
+        font-size: 0.9em;
+    }
+    /* Specific status colors for agent cards, using shared .status-indicator naming convention for consistency */
+    .agent-card .status-running { color: #ffffff; background-color: #4CAF50;} 
+    .agent-card .status-stopped { color: #e0e0e0; background-color: #525252;} 
+    .agent-card .status-error   { color: #ffffff; background-color: #F44336;} 
+    .agent-card .status-unknown { color: #333333; background-color: #BDBDBD;} 
+
+    .agent-card p.description { /* Agent description text */
+        font-size: 0.95em; 
+        color: #c0c0e0; 
+        margin-bottom: 0.75rem; 
+    }
+    .agent-card .metrics { /* Styling for the metrics block within an agent card */
+        font-size: 0.9em;
+        color: #b0b0d0;
+        margin-top: 0.75rem;
+        padding: 0.6rem; /* Increased padding */
+        background-color: rgba(0,0,0,0.15); /* Slightly darker background for metrics section */
+        border-radius: 4px;
+        border-left: 3px solid #3a6fbf; /* Accent blue border */
+    }
+    .agent-card .metrics p { margin-bottom: 0.3rem; } /* Spacing for lines within metrics block */
+
+    .agent-card .actions { /* Container for action buttons */
+        margin-top: 1.25rem; 
+        display: flex;
+        gap: 0.75rem; /* Space between buttons */
+        justify-content: flex-start; /* Align buttons to the start */
+    }
+    /* Buttons within .actions will inherit general .stButton styling from shared_css */
+</style>
+""", unsafe_allow_html=True)
+
+st.title("🕹️ Agent Control Tower") 
+st.caption("Manage and monitor your Tensorus intelligent agents.")
+
+# AGENT_DETAILS_MAP provides static information (name, icon, description) for each agent type.
+# This map is used to render the UI elements for agents known to the frontend.
+# Metrics template helps in displaying placeholder metrics consistently.
+AGENT_DETAILS_MAP = {
+    "ingestion": {"name": "Ingestion Agent", "icon": "📥", "description": "Monitors file systems and ingests new data tensors into the platform.", "metrics_template": "Files Processed: {files_processed}"},
+    "rl_trainer": {"name": "RL Training Agent", "icon": "🧠", "description": "Trains reinforcement learning models using available experiences and data.", "metrics_template": "Training Cycles: {episodes_trained}, Avg. Reward: {avg_reward}"},
+    "automl_search": {"name": "AutoML Search Agent", "icon": "✨", "description": "Performs automated machine learning model searches and hyperparameter tuning.", "metrics_template": "Search Trials: {trials_completed}, Best Model Score: {best_score}"},
+    "nql_query": {"name": "NQL Query Agent", "icon": "🗣️", "description": "Processes Natural Query Language (NQL) requests against tensor data.", "metrics_template": "Queries Handled: {queries_processed}"}
+}
+
+# Create two tabs: one for the roster of agents, another for visualizing interactions.
+tab1, tab2 = st.tabs(["📊 Agent Roster", "🔗 Agent Interaction Visualizer"])
+
+# --- Agent Roster Tab ---
+with tab1:
+    st.header("Live Agent Roster") # Standard header, styled by shared CSS
+    
+    # Button to manually refresh agent statuses from the backend.
+    if st.button("🔄 Refresh Agent Statuses"):
+        # Update session state cache with fresh data from API.
+        # Using a page-specific cache key to avoid conflicts.
+        st.session_state.agent_statuses_cache_control_panel_v2 = get_agent_status() 
+        # Streamlit automatically reruns the script on button press.
+
+    # Cache for agent statuses to avoid redundant API calls on every interaction.
+    # Initializes if key is not present or if cache is None (e.g., after an error).
+    cache_key = 'agent_statuses_cache_control_panel_v2'
+    if cache_key not in st.session_state or st.session_state[cache_key] is None:
+        st.session_state[cache_key] = get_agent_status() # Initial fetch if not in cache
+    
+    agents_api_data = st.session_state[cache_key] # Retrieve cached data.
+
+    # Check if agent data was successfully fetched.
+    if not agents_api_data:
+        st.error("Could not fetch agent statuses. Ensure the backend API is running and `pages_shared_utils.py` is correctly configured.")
+    else:
+        num_columns = 2  # Define number of columns for agent card layout.
+        cols = st.columns(num_columns)
+        
+        # Iterate through a predefined list of agent IDs expected to be available.
+        # This ensures consistent ordering and display of known agents.
+        agent_ids_to_display = list(AGENT_DETAILS_MAP.keys())
+
+        for i, agent_id in enumerate(agent_ids_to_display):
+            # Retrieve static details (icon, name, description) from the predefined map.
+            # Provides a fallback if an agent_id from the API isn't in the map (e.g., new agent not yet in UI map).
+            agent_static_details = AGENT_DETAILS_MAP.get(agent_id, {
+                "name": agent_id.replace("_", " ").title() + " Agent (Unknown)", # Default name formatting
+                "icon": "❓", # Default icon for unknown agent types
+                "description": "Details not defined in AGENT_DETAILS_MAP.",
+                "metrics_template": "Status: {status}" # Basic fallback metric
+            })
+            # Retrieve live runtime information (status, specific metrics) from the API data.
+            agent_runtime_info = agents_api_data.get(agent_id, {}) # Default to empty dict if agent not in API response
+
+            with cols[i % num_columns]: # Distribute agents into the defined columns.
+                # Each agent is displayed in a styled card.
+                # The card uses 'common-card' class from shared CSS and page-specific 'agent-card'.
+                st.markdown(f'<div class="common-card agent-card" id="agent-card-{agent_id}">', unsafe_allow_html=True)
+                
+                # Agent card header: Icon and Name.
+                st.markdown(f"""
+                    <div class="agent-card-header">
+                        <span class="icon">{agent_static_details['icon']}</span>
+                        <h3>{agent_static_details['name']}</h3>
+                    </div>
+                """, unsafe_allow_html=True)
+
+                # Agent status: Fetched live, styled based on status string.
+                # Prioritize 'status' field, fallback to 'running' boolean if 'status' is missing.
+                current_status = agent_runtime_info.get('status', agent_runtime_info.get('running', 'unknown'))
+                if isinstance(current_status, bool): # Convert boolean status to string representation
+                    current_status = "running" if current_status else "stopped"
+                status_class = f"status-{current_status.lower()}" # CSS class for styling the status text
+                st.markdown(f'<p>Status: <span class="status-text {status_class}">{current_status.upper()}</span></p>', unsafe_allow_html=True)
+                
+                # Agent description from the static map.
+                st.markdown(f"<p class='description'>{agent_static_details['description']}</p>", unsafe_allow_html=True)
+
+                # Placeholder for agent-specific metrics.
+                # Actual metrics would be populated from `agent_runtime_info` if the backend provides them.
+                # Fallback to "N/A" if a metric key is not present in runtime_info.
+                metrics_values = { 
+                    "files_processed": agent_runtime_info.get("files_processed", "N/A"),
+                    "episodes_trained": agent_runtime_info.get("episodes_trained", "N/A"),
+                    "avg_reward": agent_runtime_info.get("avg_reward", "N/A"),
+                    "trials_completed": agent_runtime_info.get("trials_completed", "N/A"),
+                    "best_score": agent_runtime_info.get("best_score", "N/A"),
+                    "queries_processed": agent_runtime_info.get("queries_processed", "N/A"),
+                    "status": current_status # Fallback for generic status display in metrics
+                }
+                metrics_str = agent_static_details['metrics_template'].format(**metrics_values)
+                st.markdown(f'<div class="metrics"><p>{metrics_str}</p></div>', unsafe_allow_html=True)
+
+                # Action buttons container.
+                st.markdown('<div class="actions">', unsafe_allow_html=True)
+                actions_cols = st.columns([1,1,1]) # Columns for button layout within the actions div.
+                
+                with actions_cols[0]: # Details button (currently a placeholder).
+                    st.button("Details", key=f"details_{agent_id}_v2", help="View detailed agent information (coming soon).", disabled=True, use_container_width=True)
+                
+                with actions_cols[1]: # Start/Stop button, conditional on agent status.
+                    if current_status == "running":
+                        if st.button("Stop", key=f"stop_{agent_id}_v2", type="secondary", use_container_width=True):
+                            result = stop_agent(agent_id) # API call from shared_utils
+                            st.toast(result.get("message", f"Stop request sent for {agent_id}."))
+                            time.sleep(1.0) # Brief pause to allow backend to process the request.
+                            st.session_state[cache_key] = get_agent_status() # Refresh status cache.
+                            st.rerun() # Rerun page to reflect updated status.
+                    else: # Agent is stopped, errored, or in an unknown state.
+                        if st.button("Start", key=f"start_{agent_id}_v2", type="primary", use_container_width=True):
+                            result = start_agent(agent_id) # API call from shared_utils
+                            st.toast(result.get("message", f"Start request sent for {agent_id}."))
+                            time.sleep(1.0)
+                            st.session_state[cache_key] = get_agent_status() # Refresh.
+                            st.rerun()
+                
+                with actions_cols[2]: # Configure button (currently a placeholder).
+                    st.button("Configure", key=f"config_{agent_id}_v2", help="Configure agent settings (coming soon).", disabled=True, use_container_width=True)
+                
+                st.markdown('</div>', unsafe_allow_html=True) # Close actions div.
+                st.markdown('</div>', unsafe_allow_html=True) # Close agent-card div.
+
+# --- Agent Interaction Visualizer Tab ---
+with tab2: # Agent Interaction Visualizer Tab
+    st.header("Agent Interaction Visualizer") # Standard header
+    st.subheader("Conceptual Agent-Data Flow")
+
+    # DOT language definition for the Graphviz chart.
+    # This describes the nodes (agents, data stores, sources) and edges (data flow) of the system.
+    graphviz_code = """
+    digraph AgentDataFlow {
+        // General graph attributes
+        rankdir=LR; /* Layout direction: Left to Right */
+        bgcolor="#0a0f2c"; /* Background color to match the app theme */
+        node [shape=record, style="filled,rounded", fillcolor="#18223f", /* Default node style */
+              fontname="Arial", fontsize=11, fontcolor="#e0e0e0", /* Default font attributes */
+              color="#3a6fbf", penwidth=1.5]; /* Default border color and width */
+        edge [fontname="Arial", fontsize=10, fontcolor="#c0c0ff", /* Default edge style */
+              color="#7080ff", penwidth=1.2]; /* Default edge color and width */
+
+        // Cluster for Data Sources
+        subgraph cluster_data_sources {
+            label="External Sources"; /* Cluster title */
+            style="rounded";
+            color="#4A5C85"; /* Cluster border color */
+            bgcolor="#101828"; /* Cluster background (slightly different from main) */
+            fontcolor="#D0D0FF"; /* Cluster title color */
+            
+            fs [label="FileSystem | (e.g., S3, Local Disk)", shape=folder, fillcolor="#3E5F8A"]; // File system node
+            user_queries [label="User Queries | (via UI/API)", shape=ellipse, fillcolor="#5DADE2"]; // User queries node
+        }
+
+        // Cluster for Intelligent Agents
+        subgraph cluster_agents_group {
+            label="Intelligent Agents";
+            style="rounded"; color="#4A5C85"; bgcolor="#101828"; fontcolor="#D0D0FF";
+            
+            ingestion_agent [label="{IngestionAgent | 📥 | Ingests raw data}", shape=Mrecord, fillcolor="#E74C3C"]; // Red
+            nql_agent [label="{NQLAgent | 🗣️ | Processes natural language queries}", shape=Mrecord, fillcolor="#9B59B6"]; // Purple
+            rl_agent [label="{RLAgent | 🧠 | Trains RL models, generates experiences}", shape=Mrecord, fillcolor="#2ECC71"]; // Green
+            automl_agent [label="{AutoMLAgent | ✨ | Conducts AutoML searches}", shape=Mrecord, fillcolor="#F1C40F"]; // Yellow
+        }
+        
+        // Cluster for Data Stores
+        subgraph cluster_data_stores {
+            label="Tensorus Data Stores";
+            style="rounded"; color="#4A5C85"; bgcolor="#101828"; fontcolor="#D0D0FF";
+            
+            ingested_data_api [label="Ingested Data Store | (Primary Tensor Collection)", shape=cylinder, fillcolor="#7F8C8D", height=1.5]; // Grey
+            rl_states [label="RL States | (Tensor Collection)", shape=cylinder, fillcolor="#95A5A6"]; // Lighter Grey
+            rl_experiences [label="RL Experiences | (Metadata/Tensor Collection)", shape=cylinder, fillcolor="#95A5A6"];
+            automl_results [label="AutoML Results | (Tensor/Metadata Collection)", shape=cylinder, fillcolor="#95A5A6"];
+        }
+
+        // Edges defining data flow between nodes
+        fs -> ingestion_agent [label=" new files/streams"];
+        ingestion_agent -> ingested_data_api [label=" stores tensors & metadata"];
+        
+        user_queries -> nql_agent [label=" NQL query"];
+        nql_agent -> ingested_data_api [label=" reads data"];
+        nql_agent -> rl_experiences [label=" (can also query specific stores like experiences)", style=dashed]; /* Dashed for optional/secondary path */
+        
+        ingested_data_api -> rl_agent [label=" reads training data/states"];
+        rl_agent -> rl_states [label=" stores/updates policy states"];
+        rl_agent -> rl_experiences [label=" stores new experiences"];
+        rl_experiences -> rl_agent [label=" reads past experiences for training"]; // Loop back for learning
+        
+        ingested_data_api -> automl_agent [label=" reads datasets for AutoML"];
+        automl_agent -> automl_results [label=" stores trial results & models"];
+    }
+    """
+    try:
+        # Render the Graphviz chart.
+        st.graphviz_chart(graphviz_code)
+        st.caption("This is a conceptual representation of data flows. Actual interactions can be more complex and configurable.")
+    except Exception as e:
+        # Handle potential errors if Graphviz is not installed or there's an issue with the DOT code.
+        st.error(f"Could not render Graphviz chart: {e}")
+    st.markdown("Please ensure Graphviz is installed and accessible in your environment's PATH (e.g., `sudo apt-get install graphviz` on Debian/Ubuntu).")
+
+# Initialize session state key if not already present for this page.
+# This helps prevent errors if the page is loaded before the cache is populated.
+if 'agent_statuses_cache_control_panel_v2' not in st.session_state:
+    st.session_state.agent_statuses_cache_control_panel_v2 = None

pages/data_explorer_v2.py

@@ -0,0 +1,224 @@
+# pages/data_explorer_v2.py
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import torch
+
+try:
+    from pages.pages_shared_utils import (
+        load_css as load_shared_css,
+        get_datasets,
+        get_dataset_preview
+    )
+except ImportError:
+    st.error("Critical Error: Could not import `pages_shared_utils`. Page cannot function.")
+    def load_shared_css(): pass
+    def get_datasets(): st.error("`get_datasets` unavailable."); return []
+    def get_dataset_preview(dataset_name, limit=5): st.error("`get_dataset_preview` unavailable."); return None
+    st.stop()
+
+# Configure page settings
+st.set_page_config(page_title="Tensor Explorer V2", layout="wide")
+load_shared_css() # Apply shared Nexus theme styles
+
+st.title("🔍 Tensor Explorer (V2)")
+st.caption("Browse, filter, and visualize Tensorus datasets with agent context.")
+
+# --- Dataset Selection ---
+# Fetch the list of available datasets from the backend using shared utility.
+available_datasets = get_datasets() 
+if not available_datasets:
+    st.warning("No datasets found. Ensure the backend API is running and accessible.")
+    st.stop() # Halt execution if no datasets are available for exploration.
+
+# Dropdown for user to select a dataset.
+selected_dataset_name = st.selectbox("Select Dataset:", available_datasets)
+
+# --- Data Fetching & Initial Processing ---
+# Proceed only if a dataset is selected.
+if selected_dataset_name:
+    st.subheader(f"Exploring: {selected_dataset_name}") # Styled by shared CSS
+
+    MAX_RECORDS_DISPLAY = 100 # Define max records to fetch for the initial preview.
+    # Fetch dataset preview information (includes sample records) using shared utility.
+    dataset_info = get_dataset_preview(selected_dataset_name, limit=MAX_RECORDS_DISPLAY)
+
+    # Validate fetched data.
+    if dataset_info is None or "preview" not in dataset_info:
+        st.error(f"Failed to fetch data preview for '{selected_dataset_name}'. The backend might be down or the dataset is invalid.")
+        st.stop()
+    
+    records_preview = dataset_info.get("preview", []) # List of records (tensor data + metadata).
+    total_records_in_dataset = dataset_info.get("record_count", len(records_preview)) # Total records in the dataset.
+
+    if not records_preview:
+        st.info(f"Dataset '{selected_dataset_name}' is empty or no preview data available.")
+        st.stop()
+
+    st.info(f"Displaying {len(records_preview)} of {total_records_in_dataset} records from '{selected_dataset_name}'.")
+
+    # Prepare data for DataFrame display: extract and flatten metadata.
+    # This makes metadata fields directly accessible as columns in the DataFrame for filtering and display.
+    processed_records_for_df = []
+    for record in records_preview:
+        meta = record.get('metadata', {}).copy() # Make a copy to avoid modifying original record.
+        meta['tensor_id'] = record.get('id', 'N/A') # Use 'id' from record as 'tensor_id'.
+        meta['shape'] = str(record.get('shape', 'N/A')) # Store shape as string for display.
+        meta['dtype'] = record.get('dtype', 'N/A')
+        # Ensure 'created_by' field exists, defaulting to 'Unknown' if not present. This is for filtering.
+        meta['created_by'] = meta.get('created_by', 'Unknown')
+        processed_records_for_df.append(meta)
+
+    df_display_initial = pd.DataFrame(processed_records_for_df) # DataFrame for filtering and display.
+
+    # --- Sidebar Filtering UI ---
+    st.sidebar.header("Filter Options") # Styled by shared CSS
+
+    # Filter by Source Agent ('created_by' field).
+    st.sidebar.subheader("Filter by Source Agent") # Styled by shared CSS
+    # Predefined agent sources, augmented with any unique sources found in the data for comprehensive filtering.
+    agent_sources_default = ["IngestionAgent", "RLAgent", "AutoMLAgent", "Unknown"] 
+    if 'created_by' in df_display_initial.columns:
+        available_agents_in_data = df_display_initial['created_by'].unique().tolist()
+        filter_options_agents = sorted(list(set(agent_sources_default + available_agents_in_data)))
+    else:
+        filter_options_agents = agent_sources_default # Fallback if 'created_by' column is missing.
+
+    selected_agent_filters = st.sidebar.multiselect(
+        "Show data created by:",
+        options=filter_options_agents,
+        default=[] # No agents selected by default; shows all data initially.
+    )
+
+    # Filter by Other Metadata Fields.
+    st.sidebar.subheader("Filter by Other Metadata Fields") # Styled by shared CSS
+    # Allow filtering on any metadata column except those already specifically handled or less useful for direct filtering.
+    potential_filter_cols = [col for col in df_display_initial.columns if col not in ['tensor_id', 'created_by', 'shape', 'dtype']]
+    
+    filter_cols_metadata = st.sidebar.multiselect(
+        "Select metadata fields to filter:",
+        options=potential_filter_cols
+    )
+
+    # Apply filters to the DataFrame. Start with a copy of the initial DataFrame.
+    filtered_df_final = df_display_initial.copy()
+
+    # Apply agent source filter if any agents are selected.
+    if selected_agent_filters:
+        if 'created_by' in filtered_df_final.columns:
+            filtered_df_final = filtered_df_final[filtered_df_final['created_by'].isin(selected_agent_filters)]
+        else: 
+            # This case should ideally not be reached if 'created_by' is always added or defaulted.
+            st.sidebar.warning("'created_by' field not found for agent filtering.")
+            filtered_df_final = filtered_df_final.iloc[0:0] # Show no results if filter is active but field is missing.
+            
+    # Apply other metadata filters based on user selections.
+    for col in filter_cols_metadata:
+        unique_values = filtered_df_final[col].dropna().unique().tolist()
+        # Numeric filter: use a range slider if there are multiple unique numeric values.
+        if pd.api.types.is_numeric_dtype(filtered_df_final[col]) and filtered_df_final[col].nunique() > 1:
+            min_val, max_val = float(filtered_df_final[col].min()), float(filtered_df_final[col].max())
+            selected_range = st.sidebar.slider(f"Filter {col}:", min_val, max_val, (min_val, max_val), key=f"slider_{col}")
+            filtered_df_final = filtered_df_final[filtered_df_final[col].between(selected_range[0], selected_range[1])]
+        # Categorical filter (with limited unique values for dropdown): use multiselect.
+        elif len(unique_values) > 0 and len(unique_values) <= 25: # Threshold for using multiselect.
+             default_selection = [] # Default to no specific selection (i.e., don't filter unless user picks).
+             selected_values = st.sidebar.multiselect(f"Filter {col}:", options=unique_values, default=default_selection, key=f"multi_{col}")
+             if selected_values: # Only apply filter if user has made selections.
+                filtered_df_final = filtered_df_final[filtered_df_final[col].isin(selected_values)]
+        # Text search for other columns or columns with too many unique values.
+        else:
+            search_term = st.sidebar.text_input(f"Search in {col} (contains):", key=f"text_{col}").lower()
+            if search_term:
+                filtered_df_final = filtered_df_final[filtered_df_final[col].astype(str).str.lower().str.contains(search_term, na=False)]
+
+    # --- Display Filtered Data ---
+    st.divider() # Visual separator.
+    st.subheader("Filtered Data View") # Styled by shared CSS
+    st.write(f"{len(filtered_df_final)} records matching filters.") # Display count of filtered records.
+    
+    # Define preferred column order for the displayed DataFrame.
+    cols_to_display_order = ['tensor_id', 'created_by', 'shape', 'dtype']
+    remaining_cols = [col for col in df_display_initial.columns if col not in cols_to_display_order]
+    # Ensure only existing columns are included in the final display order.
+    final_display_columns = [col for col in cols_to_display_order if col in filtered_df_final.columns] + \
+                            [col for col in remaining_cols if col in filtered_df_final.columns]
+    
+    st.dataframe(filtered_df_final[final_display_columns], use_container_width=True, hide_index=True)
+
+    # --- Tensor Preview & Visualization ---
+    st.divider()
+    st.subheader("Tensor Preview") # Styled by shared CSS
+
+    if not filtered_df_final.empty:
+        available_tensor_ids = filtered_df_final['tensor_id'].tolist()
+        # Dropdown to select a tensor ID from the filtered results for preview.
+        selected_tensor_id = st.selectbox("Select Tensor ID to Preview:", available_tensor_ids, key="tensor_preview_select")
+
+        if selected_tensor_id:
+            # Retrieve the full record (including raw tensor data list) from the original preview list.
+            selected_full_record = next((r for r in records_preview if r.get('id') == selected_tensor_id), None)
+
+            if selected_full_record:
+                st.write("**Full Metadata:**")
+                st.json(selected_full_record.get('metadata', {})) # Display all metadata for the selected tensor.
+
+                shape = selected_full_record.get("shape")
+                dtype_str = selected_full_record.get("dtype")
+                data_list = selected_full_record.get("data") # Raw list representation of tensor data.
+
+                st.write(f"**Tensor Info:** Shape=`{shape}`, Dtype=`{dtype_str}`")
+                source_agent = selected_full_record.get('metadata', {}).get('created_by', 'Unknown')
+                st.write(f"**Source Agent:** `{source_agent}`") # Display the 'created_by' agent.
+
+                try:
+                    if shape and dtype_str and data_list is not None:
+                        # Reconstruct the tensor from its list representation and metadata.
+                        torch_dtype = getattr(torch, dtype_str, None) # Get torch.dtype from string.
+                        if torch_dtype is None:
+                            st.error(f"Unsupported dtype: {dtype_str}. Cannot reconstruct tensor.")
+                        else:
+                            tensor = torch.tensor(data_list, dtype=torch_dtype)
+                            st.write("**Tensor Data (first 10 elements flattened):**")
+                            st.code(f"{tensor.flatten()[:10].cpu().numpy()}...") # Display a snippet of tensor data.
+
+                            # --- Simple Visualizations based on tensor dimensions ---
+                            if tensor.ndim == 1 and tensor.numel() > 1: # 1D tensor: line chart.
+                                st.line_chart(tensor.cpu().numpy())
+                            elif tensor.ndim == 2 and tensor.shape[0] > 1 and tensor.shape[1] > 1 : # 2D tensor: heatmap.
+                                try:
+                                    fig = px.imshow(tensor.cpu().numpy(), title="Tensor Heatmap", aspect="auto")
+                                    st.plotly_chart(fig, use_container_width=True)
+                                except Exception as plot_err:
+                                    st.warning(f"Could not generate heatmap: {plot_err}")
+                            elif tensor.ndim == 3 and tensor.shape[0] in [1, 3]: # 3D tensor (potential image): try to display as image.
+                                try:
+                                    display_tensor = tensor.cpu()
+                                    if display_tensor.shape[0] == 1: # Grayscale image (C, H, W) -> (H, W)
+                                        display_tensor = display_tensor.squeeze(0)
+                                    elif display_tensor.shape[0] == 3: # RGB image (C, H, W) -> (H, W, C)
+                                        display_tensor = display_tensor.permute(1, 2, 0)
+                                    
+                                    # Normalize for display if not in typical image range [0,1] or [0,255].
+                                    if display_tensor.max() > 1.0 or display_tensor.min() < 0.0: # Basic check
+                                         display_tensor = (display_tensor - display_tensor.min()) / (display_tensor.max() - display_tensor.min() + 1e-6) # Normalize to [0,1]
+                                    
+                                    st.image(display_tensor.numpy(), caption="Tensor as Image (Attempted)", use_column_width=True)
+                                except Exception as img_err:
+                                    st.warning(f"Could not display tensor as image: {img_err}")
+                            else:
+                                st.info("No specific visualization available for this tensor's shape/dimension.")
+                    else:
+                        st.warning("Tensor data, shape, or dtype missing in the selected record.")
+                except Exception as tensor_err:
+                    st.error(f"Error processing tensor data for preview: {tensor_err}")
+            else:
+                st.warning("Selected tensor ID details not found in the fetched preview data.")
+        else:
+            st.info("Select a Tensor ID from the filtered table above to preview its details.")
+    else:
+        st.info("No records match the current filters to allow preview.")
+else:
+    st.info("Select a dataset to start exploring.")
+

pages/nql_chatbot_v2.py

@@ -0,0 +1,168 @@
+# pages/nql_chatbot_v2.py
+
+import streamlit as st
+import pandas as pd
+
+# Import from the shared utils for pages
+try:
+    from pages.pages_shared_utils import (
+        load_css as load_shared_css,
+        post_nql_query
+    )
+except ImportError:
+    st.error("Critical Error: Could not import `pages_shared_utils`. Page cannot function.")
+    def load_shared_css(): pass
+    def post_nql_query(query: str): 
+        st.error("`post_nql_query` unavailable.")
+        return {"query": query, "response_text": "Error: NQL processing unavailable.", "error": "Setup issue.", "results": None}
+    st.stop()
+
+st.set_page_config(page_title="NQL Chatbot (V2)", layout="wide")
+load_shared_css() # Load common CSS from shared utilities
+
+# Custom CSS for NQL Chatbot page
+# These styles are specific to the chat interface elements.
+st.markdown("""
+<style>
+    /* Chatbot specific styles */
+    .stChatMessage { /* Base style for chat messages */
+        border-radius: 10px; /* Rounded corners */
+        padding: 0.85rem 1.15rem; /* Comfortable padding */
+        margin-bottom: 0.75rem; /* Space between messages */
+        box-shadow: 0 2px 5px rgba(0,0,0,0.15); /* Subtle shadow */
+        border: 1px solid #2a3f5c; /* Consistent border with other elements */
+    }
+    /* User messages styling */
+    .stChatMessage[data-testid="stChatMessageContent"]:has(.user-avatar) {
+        background-color: #2a2f4c; /* Darker blue, similar to nav hover, for user messages */
+        border-left: 5px solid #3a6fbf; /* Accent blue border, consistent with active nav link */
+    }
+    /* Assistant messages styling */
+    .stChatMessage[data-testid="stChatMessageContent"]:has(.assistant-avatar) {
+        background-color: #18223f; /* Slightly lighter blue-gray, similar to common-card */
+        border-left: 5px solid #7070ff; /* Muted purple accent for assistant messages */
+    }
+    .user-avatar, .assistant-avatar { /* Styling for user/assistant avatars */
+        font-size: 1.5rem; /* Avatar size */
+        margin-right: 0.5rem; /* Space between avatar and message content */
+    }
+
+    /* Styling for the chat input area at the bottom */
+    /* The .chat-input-container class is a convention, Streamlit might not add this by default.
+       Targeting based on Streamlit's internal structure is more robust. */
+    div[data-testid="stChatInput"] { /* Main container for chat input */
+        background-color: #0a0f2c; /* Match page background */
+        border-top: 1px solid #2a3f5c; /* Separator line */
+        padding: 0.75rem;
+    }
+    div[data-testid="stChatInput"] textarea { /* The actual text input field */
+        border: 1px solid #3a3f5c !important;
+        background-color: #1a1f3c !important; /* Dark input background */
+        color: #e0e0e0 !important; /* Light text color */
+        border-radius: 5px !important;
+    }
+    div[data-testid="stChatInput"] button { /* The send button */
+        border: none !important;
+        background-color: #3a6fbf !important; /* Accent blue, consistent with primary buttons */
+        color: white !important;
+        border-radius: 5px !important;
+    }
+    div[data-testid="stChatInput"] button:hover {
+        background-color: #4a7fdc !important; /* Lighter blue on hover */
+    }
+
+    /* Styling for DataFrames displayed within chat messages */
+    .stDataFrame { /* General DataFrame styling from shared_utils will apply */
+        margin-top: 0.5rem; /* Space above DataFrame in chat */
+    }
+    /* .results-dataframe and .results-dataframe .col-header are less reliable than generic .stDataFrame */
+</style>
+""", unsafe_allow_html=True)
+
+
+st.title("💬 Natural Query Language (NQL) Chatbot")
+st.caption("Query Tensorus datasets using natural language. Powered by Tensorus NQL Agent.")
+
+# Initialize chat history in session state if it doesn't exist.
+# Use a unique key for this page's messages to avoid conflicts with other potential chat interfaces.
+if "nql_messages_v2" not in st.session_state: 
+    st.session_state.nql_messages_v2 = []
+
+# Display chat messages from history on app rerun.
+# This ensures that the conversation persists during the session.
+for message in st.session_state.nql_messages_v2:
+    avatar = "👤" if message["role"] == "user" else "🤖" # Assign avatar based on role
+    with st.chat_message(message["role"], avatar=avatar):
+        st.markdown(message["content"]) # Display the text content of the message.
+        # If the message includes a DataFrame (results from NQL query), display it.
+        if "results_df" in message and message["results_df"] is not None and not message["results_df"].empty:
+            st.dataframe(message["results_df"], use_container_width=True, hide_index=True)
+        # If the message includes an error, display it in an error box.
+        elif "error" in message and message["error"]:
+            st.error(message["error"])
+
+
+# React to user input from the chat interface.
+if prompt := st.chat_input("Enter your query (e.g., 'show 5 tensors from my_dataset')"):
+    # Add user's message to chat history and display it in the chat interface.
+    st.session_state.nql_messages_v2.append({"role": "user", "content": prompt})
+    with st.chat_message("user", avatar="👤"):
+        st.markdown(prompt)
+
+    # Get assistant's response by processing the query.
+    with st.spinner("Tensorus NQL Agent is thinking..."): # Show a loading spinner.
+        # Call the NQL processing function from shared utilities.
+        nql_api_response = post_nql_query(prompt) 
+
+        # Extract relevant information from the API response.
+        response_text = nql_api_response.get("response_text", "Sorry, I encountered an issue processing your request.")
+        results_data = nql_api_response.get("results") # Expected to be a list of dicts (records).
+        error_message = nql_api_response.get("error") # Any error message from the backend.
+        
+        # Prepare data for storing the assistant's message in session state.
+        assistant_message_data = {"role": "assistant", "content": response_text}
+
+        # Display assistant's response in the chat interface.
+        with st.chat_message("assistant", avatar="🤖"):
+            st.markdown(response_text) # Display the textual response.
+
+            if results_data: # If the API returned 'results'.
+                try:
+                    # Process results into a Pandas DataFrame for structured display.
+                    # Assumes results_data is a list of records (dictionaries).
+                    # Each record might have 'id', 'shape', 'dtype', 'metadata'.
+                    processed_for_df = []
+                    for record in results_data:
+                        row = {"tensor_id": record.get("id")} # Start with tensor_id.
+                        # Flatten metadata fields into the main row for the DataFrame.
+                        if isinstance(record.get("metadata"), dict):
+                            row.update(record["metadata"])
+                        row["shape"] = str(record.get("shape")) # Ensure shape is a string for display.
+                        row["dtype"] = record.get("dtype")
+                        processed_for_df.append(row)
+                    
+                    if processed_for_df:
+                        results_df = pd.DataFrame(processed_for_df)
+                        st.dataframe(results_df, use_container_width=True, hide_index=True)
+                        assistant_message_data["results_df"] = results_df # Store DataFrame for history.
+                    elif not error_message: # If no data and no error, it might be a query that doesn't return records.
+                        st.caption("Query processed, no specific records returned.")
+
+                except Exception as e:
+                    # Handle errors during DataFrame processing.
+                    st.error(f"Error formatting results for display: {e}")
+                    assistant_message_data["error"] = f"Error formatting results: {e}"
+            
+            elif error_message: # If there's a specific error message from the API.
+                st.error(error_message)
+                assistant_message_data["error"] = error_message
+            
+            # If no results_data and no explicit error_message, the response_text itself is the primary message.
+
+        # Add assistant's response (including any processed data or errors) to chat history.
+        st.session_state.nql_messages_v2.append(assistant_message_data)
+
+else: # This block runs when the page loads or if the chat input is empty.
+    # Show a welcome/instruction message if the chat history is empty.
+    if not st.session_state.nql_messages_v2: 
+        st.info("Ask me anything about your data! For example: 'list datasets' or 'show tensors from dataset XYZ limit 5'.")

pages/pages_shared_utils.py

@@ -0,0 +1,547 @@
+# pages/pages_shared_utils.py
+"""
+Shared utility functions for Streamlit pages.
+Copied/adapted from app.py to avoid complex import issues.
+"""
+import streamlit as st
+import requests
+import logging
+import os # Added import
+from typing import Optional, List, Dict, Any # Added for new functions
+
+logger = logging.getLogger(__name__)
+
+API_BASE_URL = os.getenv("API_BASE_URL", "http://127.0.0.1:7860") # Changed API_BASE_URL
+
+def load_css():
+    """Loads the main CSS styles. Assumes app.py's CSS content."""
+    st.markdown("""
+<style>
+    /* --- Shared Base Styles for Tensorus Platform (Nexus Theme) --- */
+
+    /* General Page Styles */
+    body {
+        font-family: 'Arial', 'Helvetica Neue', 'Helvetica', sans-serif;
+        line-height: 1.6;
+    }
+    .stApp { /* Main Streamlit app container */
+        background-color: #0a0f2c; /* Primary Background: Dark blue/purple */
+        color: #e0e0e0; /* Default Text Color: Light grey */
+    }
+
+    /* Headings & Titles */
+    h1, .stTitle { /* Main page titles */
+        color: #d0d0ff !important; /* Primary Heading Color: Light purple/blue */
+        font-weight: bold !important;
+    }
+    h2, .stSubheader { /* Section headers */
+        color: #c0c0ef !important; /* Secondary Heading Color */
+        font-weight: bold !important;
+        border-bottom: 1px solid #3a3f5c; /* Accent Border for separation */
+        padding-bottom: 0.3rem;
+        margin-top: 1.5rem;
+        margin-bottom: 1rem;
+    }
+    h3 { /* General h3, often used in st.markdown */
+        color: #b0b0df !important; /* Tertiary Heading Color */
+        font-weight: bold !important;
+    }
+    .stMarkdown p, .stText, .stListItem { /* General text elements */
+        color: #c0c0dd; /* Softer light text */
+        font-size: 1rem;
+    }
+    .stCaption, caption { /* Streamlit captions and HTML captions */
+        font-size: 0.85rem !important;
+        color: #a0a0c0 !important; /* Muted color for captions */
+    }
+
+    /* Custom Top Navigation Bar */
+    .topnav-container {
+        background-color: #1a1f3c; /* Nav Background: Slightly darker than page */
+        padding: 0.5rem 1rem;
+        border-bottom: 1px solid #3a3f5c; /* Accent Border */
+        display: flex;
+        justify-content: flex-start;
+        align-items: center;
+        position: sticky; top: 0; z-index: 1000; /* Ensure it's on top */
+        width: 100%;
+        box-sizing: border-box;
+    }
+    .topnav-container .logo {
+        font-size: 1.5em;
+        font-weight: bold;
+        color: #d0d0ff; /* Primary Heading Color for logo */
+        margin-right: 2rem;
+    }
+    .topnav-container nav a {
+        color: #c0c0ff; /* Lighter Text for Nav Links */
+        padding: 0.75rem 1rem;
+        text-decoration: none;
+        font-weight: 500;
+        margin-right: 0.5rem;
+        border-radius: 4px;
+        transition: background-color 0.2s ease, color 0.2s ease;
+    }
+    .topnav-container nav a:hover {
+        background-color: #2a2f4c; /* Nav Link Hover Background */
+        color: #ffffff; /* Nav Link Hover Text */
+    }
+    .topnav-container nav a.active {
+        background-color: #3a6fbf; /* Active Nav Link Background (Accent Blue) */
+        color: #ffffff; /* Active Nav Link Text */
+        font-weight: bold;
+    }
+
+    /* Common Card Style (base for metric cards, agent cards, etc.) */
+    .common-card {
+        background-color: #18223f; /* Card Background: Darker than nav, but lighter than page */
+        border: 1px solid #2a3f5c; /* Card Border: Accent Border color */
+        border-radius: 10px;
+        padding: 1.5rem;
+        margin-bottom: 1rem; /* Space below cards */
+        box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
+        transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
+        color: #e0e0e0; /* Default text color within cards */
+    }
+    .common-card:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 8px 16px rgba(0,0,0,0.3);
+    }
+    .common-card h3 { /* Titles within cards */
+        color: #b0b0df !important; /* Card Title Color: Slightly lighter than main headings */
+        font-size: 1.2em !important; /* Slightly larger for card titles */
+        margin-top: 0 !important; /* Remove default top margin for h3 in card */
+        margin-bottom: 0.75rem !important;
+        font-weight: bold !important;
+        border-bottom: none !important; /* Override general h2 border for card h3 */
+    }
+    .common-card p { /* Paragraphs within cards */
+        font-size: 0.95em !important; /* Slightly smaller for card content */
+        color: #c0c0dd !important;
+        margin-bottom: 0.5rem !important;
+    }
+    .common-card .icon { /* For icons within cards, like dashboard metric cards */
+        font-size: 2.5em;
+        margin-bottom: 0.75rem;
+        color: #7070ff; /* Icon Color: Muted accent */
+    }
+
+
+    /* Status Indicators (can be used with <span> or <p> or custom divs) */
+    .status-indicator {
+        padding: 0.4rem 0.8rem !important; /* Slightly more padding */
+        border-radius: 15px !important; /* Pill shape */
+        font-weight: bold !important;
+        font-size: 0.85em !important;
+        display: inline-block !important;
+        text-align: center !important;
+    }
+    .status-success, .status-running { color: #ffffff !important; background-color: #4CAF50 !important; } /* Green */
+    .status-error   { color: #ffffff !important; background-color: #F44336 !important; } /* Red */
+    .status-warning { color: #000000 !important; background-color: #FFC107 !important; } /* Amber */
+    .status-info    { color: #ffffff !important; background-color: #2196F3 !important; } /* Blue */
+    .status-stopped { color: #e0e0e0 !important; background-color: #525252 !important; } /* Darker Grey for stopped */
+    .status-unknown { color: #333333 !important; background-color: #BDBDBD !important; } /* Lighter grey for unknown */
+
+
+    /* Standardized Streamlit Input Styling */
+    .stTextInput > div > div > input, 
+    .stTextArea > div > div > textarea, 
+    .stSelectbox > div > div,
+    .stNumberInput > div > div > input {
+        border: 1px solid #3a3f5c !important; /* Accent Border */
+        background-color: #1a1f3c !important; /* Nav Background color for inputs */
+        color: #e0e0e0 !important; /* Default Text Color */
+        border-radius: 5px !important;
+    }
+    .stMultiSelect > div > div > div { /* Multiselect options container */
+         border: 1px solid #3a3f5c !important;
+         background-color: #1a1f3c !important;
+    }
+    .stMultiSelect span[data-baseweb="tag"] { /* Selected items in multiselect */
+        background-color: #3a6fbf !important; /* Active Nav Link Background */
+    }
+
+
+    /* Standardized Streamlit Button Styling */
+    .stButton > button {
+        border: 1px solid #3a6fbf !important; /* Accent Blue for border */
+        background-color: #3a6fbf !important; /* Accent Blue for background */
+        color: white !important;
+        border-radius: 5px !important;
+        padding: 0.5rem 1rem !important;
+        transition: background-color 0.2s ease, border-color 0.2s ease;
+    }
+    .stButton > button:hover {
+        background-color: #4a7fdc !important; /* Lighter Accent Blue for hover */
+        border-color: #4a7fdc !important;
+    }
+    .stButton > button:disabled {
+        background-color: #2a2f4c !important;
+        border-color: #2a2f4c !important;
+        color: #777777 !important;
+    }
+    /* For secondary buttons, Streamlit uses a 'kind' attribute in HTML we can't directly target via pure CSS.
+       Instead, use st.button(..., type="secondary") and rely on Streamlit's handling,
+       or use st.markdown for fully custom buttons if default secondary is not enough.
+       The below attempts to style based on common Streamlit secondary button appearance.
+       Note: This specific selector for secondary buttons might be unstable if Streamlit changes its internal class names.
+    */
+    .stButton button.st-emotion-cache-LPTKCI { /* Example selector for a secondary button, MAY BE UNSTABLE */
+        background-color: #2a2f4c !important; 
+        border: 1px solid #2a2f4c !important;
+        color: #c0c0ff !important;
+    }
+     .stButton button.st-emotion-cache-LPTKCI:hover {
+        background-color: #3a3f5c !important;
+        border-color: #3a3f5c !important;
+        color: #ffffff !important;
+    }
+    .stButton button.st-emotion-cache-LPTKCI:disabled { /* Disabled secondary button */
+        background-color: #1e2a47 !important;
+        border-color: #1e2a47 !important;
+        color: #555555 !important;
+    }
+
+
+    /* Dataframe styling */
+    .stDataFrame { /* Main container for dataframes */
+        border: 1px solid #2a3f5c !important; /* Accent Border */
+        border-radius: 5px !important;
+        background-color: #1a1f3c !important; /* Nav Background for dataframe background */
+    }
+    .stDataFrame th { /* Headers */
+        background-color: #2a2f4c !important; /* Nav Link Hover Background for headers */
+        color: #d0d0ff !important; /* Primary Heading Color for header text */
+        font-weight: bold;
+    }
+    .stDataFrame td { /* Cells */
+        color: #c0c0dd !important; /* Softer light text for cell data */
+        border-bottom-color: #2a3f5c !important; /* Accent border for cell lines */
+        border-top-color: #2a3f5c !important;
+    }
+    
+</style>
+""", unsafe_allow_html=True)
+
+def get_api_status() -> tuple[bool, dict]:
+    """
+    Checks if the backend API is reachable and returns its status.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+
+    Returns:
+        tuple[bool, dict]: A tuple where:
+            - The first element is a boolean: True if the API is reachable and returns a 2xx status, False otherwise.
+            - The second element is a dictionary: 
+                - If successful, contains API information (e.g., from `response.json()`).
+                - If unsuccessful, contains an 'error' key with a descriptive message.
+    """
+    try:
+        response = requests.get(f"{API_BASE_URL}/", timeout=3) # Increased timeout slightly
+        response.raise_for_status()
+        return True, response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API connection error in get_api_status (shared_utils): {e}")
+        return False, {"error": f"API connection failed: {str(e)}"}
+    except Exception as e:
+        logger.exception(f"Unexpected error in get_api_status (shared_utils): {e}")
+        return False, {"error": f"An unexpected error occurred: {str(e)}"}
+
+def get_agent_status() -> Optional[dict]:
+    """
+    Fetches the status for all registered agents from the backend.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    On successful API call, returns a dictionary where keys are agent IDs
+    and values are dictionaries containing status and configuration for each agent.
+    Returns None if the API call fails or an exception occurs.
+
+    Returns:
+        Optional[dict]: Agent statuses dictionary or None.
+    """
+    try:
+        response = requests.get(f"{API_BASE_URL}/agents/status", timeout=5)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error fetching agent status (shared_utils): {e}")
+        return None
+    except Exception as e:
+        logger.exception(f"Unexpected error in get_agent_status (shared_utils): {e}")
+        return None
+
+def start_agent(agent_id: str) -> dict:
+    """
+    Sends a request to the backend to start a specific agent.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Constructs a POST request to the `/agents/{agent_id}/start` endpoint.
+
+    Args:
+        agent_id (str): The unique identifier of the agent to start.
+
+    Returns:
+        dict: A dictionary containing the API response. Typically includes a 'success' boolean
+              and a 'message' string. In case of connection or unexpected errors,
+              it also returns a dict with 'success': False and an error 'message'.
+    """
+    try:
+        response = requests.post(f"{API_BASE_URL}/agents/{agent_id}/start", timeout=7) # Increased timeout
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error starting agent {agent_id} (shared_utils): {e}")
+        return {"success": False, "message": f"Failed to start agent {agent_id}: {str(e)}"}
+    except Exception as e:
+        logger.exception(f"Unexpected error in start_agent (shared_utils) for {agent_id}: {e}")
+        return {"success": False, "message": f"An unexpected error occurred: {str(e)}"}
+
+def stop_agent(agent_id: str) -> dict:
+    """
+    Sends a request to the backend to stop a specific agent.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Constructs a POST request to the `/agents/{agent_id}/stop` endpoint.
+
+    Args:
+        agent_id (str): The unique identifier of the agent to stop.
+
+    Returns:
+        dict: A dictionary containing the API response. Typically includes a 'success' boolean
+              and a 'message' string. In case of connection or unexpected errors,
+              it also returns a dict with 'success': False and an error 'message'.
+    """
+    try:
+        response = requests.post(f"{API_BASE_URL}/agents/{agent_id}/stop", timeout=7) # Increased timeout
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error stopping agent {agent_id} (shared_utils): {e}")
+        return {"success": False, "message": f"Failed to stop agent {agent_id}: {str(e)}"}
+    except Exception as e:
+        logger.exception(f"Unexpected error in stop_agent (shared_utils) for {agent_id}: {e}")
+        return {"success": False, "message": f"An unexpected error occurred: {str(e)}"}
+
+def get_datasets() -> list[str]:
+    """
+    Fetches the list of available dataset names from the backend.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Targets the `/explorer/datasets` endpoint.
+
+    Returns:
+        list[str]: A list of dataset names. Returns an empty list if the API call
+                   fails, if the 'datasets' key is missing in the response,
+                   or if an exception occurs.
+    """
+    try:
+        response = requests.get(f"{API_BASE_URL}/explorer/datasets", timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        return data.get("datasets", [])
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error fetching datasets (shared_utils): {e}")
+        return []
+    except Exception as e:
+        logger.exception(f"Unexpected error in get_datasets (shared_utils): {e}")
+        return []
+
+def get_dataset_preview(dataset_name: str, limit: int = 10) -> Optional[dict]:
+    """
+    Fetches preview data for a specific dataset from the backend.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Targets the `/explorer/dataset/{dataset_name}/preview` endpoint with a `limit` parameter.
+
+    Args:
+        dataset_name (str): The name of the dataset to preview.
+        limit (int): The maximum number of records to fetch for the preview. Defaults to 10.
+
+    Returns:
+        Optional[dict]: A dictionary containing dataset information (e.g., 'dataset', 
+                        'record_count', 'preview' list of records) if successful. 
+                        Each record in the 'preview' list is a dictionary typically 
+                        containing 'id', 'shape', 'dtype', 'metadata', and 'data' (raw list).
+                        Returns None if the API call fails or an exception occurs.
+    """
+    try:
+        response = requests.get(f"{API_BASE_URL}/explorer/dataset/{dataset_name}/preview?limit={limit}", timeout=10)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error fetching dataset preview for {dataset_name} (shared_utils): {e}")
+        return None
+    except Exception as e:
+        logger.exception(f"Unexpected error in get_dataset_preview (shared_utils) for {dataset_name}: {e}")
+        return None
+
+
+
+
+def get_tensor_metadata(dataset_name: str, tensor_id: str) -> Optional[dict]:
+    """Fetch metadata for a specific tensor via the Explorer API."""
+    try:
+        response = requests.get(
+            f"{API_BASE_URL}/explorer/dataset/{dataset_name}/tensor/{tensor_id}/metadata",
+            timeout=5,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data.get("metadata", data)
+    except requests.exceptions.RequestException as e:
+        logger.error(
+            f"API error fetching tensor metadata for {dataset_name}/{tensor_id} (shared_utils): {e}"
+        )
+        return None
+    except Exception as e:
+        logger.exception(
+            f"Unexpected error in get_tensor_metadata (shared_utils) for {dataset_name}/{tensor_id}: {e}"
+
+        )
+
+        return None
+
+def list_all_agents() -> list[dict[str, str]]:
+    """
+    Returns a list of agent details based on the current statuses fetched by `get_agent_status`.
+
+    Each agent detail is a dictionary with 'id', 'name', and 'status' keys.
+    If an agent's name is not explicitly provided in the status data, a default name
+    is generated by capitalizing the agent_id and replacing underscores with spaces.
+
+    This function is a convenience wrapper around `get_agent_status()` if a list
+    format of agent information is preferred.
+
+    Returns:
+        list[dict[str, str]]: A list of dictionaries, where each dictionary represents an agent
+                              and contains its 'id', 'name', and 'status'.
+                              Returns an empty list if agent statuses cannot be fetched.
+    """
+    agents_status_data = get_agent_status()
+    if agents_status_data:
+        return [
+            {
+                "id": agent_id,
+                "name": agent_data.get("name", agent_id.replace("_", " ").title()), # Default formatted name
+                "status": agent_data.get("status", "unknown")
+            }
+            for agent_id, agent_data in agents_status_data.items()
+        ]
+    return []
+
+def post_nql_query(query: str) -> dict:
+    """
+    Sends an NQL query to the backend for processing.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Constructs a POST request to the `/chat/query` endpoint with the user's query.
+
+    Args:
+        query (str): The Natural Query Language query string provided by the user.
+
+    Returns:
+        dict: A dictionary containing the API response. 
+              On success, this typically includes:
+                - 'query': The original query string.
+                - 'response_text': A textual summary of the NQL agent's action or findings.
+                - 'results': A list of records (tensors with metadata) if the query involved data retrieval.
+                             Each record is a dictionary, potentially including 'id', 'shape', 
+                             'dtype', 'metadata', and 'data'.
+                - 'count': Number of results found (if applicable).
+              On failure (e.g., connection error, API error, unexpected server error):
+                - 'query': The original query.
+                - 'response_text': An error message.
+                - 'error': A more detailed error string.
+                - 'results': None or an empty list.
+    """
+    try:
+        response = requests.post(
+            f"{API_BASE_URL}/chat/query", # Ensure API_BASE_URL is defined in this file
+            json={"query": query},
+            timeout=15
+        )
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error posting NQL query from pages_shared_utils: {e}")
+        # Let the caller handle UI error display
+        return {"query": query, "response_text": "Error connecting to backend or processing query.", "error": str(e), "results": None}
+    except Exception as e:
+        logger.exception(f"Unexpected error in post_nql_query (pages_shared_utils): {e}")
+        return {"query": query, "response_text": "An unexpected error occurred.", "error": str(e), "results": None}
+
+# --- Functions to be added from app.py ---
+
+def configure_agent(agent_id: str, config: dict) -> dict:
+    """
+    Sends a request to the backend to configure a specific agent.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Constructs a POST request to the `/agents/{agent_id}/configure` endpoint.
+
+    Args:
+        agent_id (str): The unique identifier of the agent to configure.
+        config (dict): The configuration dictionary for the agent.
+
+    Returns:
+        dict: A dictionary containing the API response. Typically includes 'success' boolean
+              and a 'message' string. In case of connection or unexpected errors,
+              it also returns a dict with 'success': False and an error 'message'.
+    """
+    try:
+        response = requests.post(
+            f"{API_BASE_URL}/agents/{agent_id}/configure",
+            json={"config": config},
+            timeout=7 # Increased timeout similar to start/stop
+        )
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error configuring agent {agent_id} (shared_utils): {e}")
+        return {"success": False, "message": f"Failed to configure agent {agent_id}: {str(e)}"}
+    except Exception as e:
+        logger.exception(f"Unexpected error in configure_agent (shared_utils) for {agent_id}: {e}")
+        return {"success": False, "message": f"An unexpected error occurred: {str(e)}"}
+
+def operate_explorer(dataset: str, operation: str, index: int, params: dict) -> dict:
+    """
+    Sends an operation request to the data explorer for a specific tensor.
+
+    Uses the `API_BASE_URL` constant defined in this module.
+    Constructs a POST request to the `/explorer/operate` endpoint.
+
+    Args:
+        dataset (str): The name of the dataset containing the tensor.
+        operation (str): The operation to perform (e.g., 'view', 'transform').
+        index (int): The index of the tensor within the dataset.
+        params (dict): Additional parameters required for the operation.
+
+    Returns:
+        dict: A dictionary containing the API response. Typically includes:
+              - 'success': A boolean indicating if the operation was accepted.
+              - 'metadata': A dictionary with details about the operation or resulting tensor.
+              - 'result_data': The data of the resulting tensor (if applicable), or None.
+              In case of connection or server-side errors, it returns a dict with
+              'success': False, 'metadata': {'error': error_message}, and 'result_data': None.
+    """
+    payload = {
+        "dataset": dataset,
+        "operation": operation,
+        "tensor_index": index,
+        "params": params
+    }
+    try:
+        response = requests.post(
+            f"{API_BASE_URL}/explorer/operate",
+            json=payload,
+            timeout=15 # Standard timeout for potentially long operations
+        )
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API error in operate_explorer for {dataset} (shared_utils): {e}")
+        return {"success": False, "metadata": {"error": str(e)}, "result_data": None}
+    except Exception as e:
+        logger.exception(f"Unexpected error in operate_explorer (shared_utils) for {dataset}: {e}")
+        return {"success": False, "metadata": {"error": str(e)}, "result_data": None}

pages/ui_utils.py

@@ -0,0 +1,267 @@
+# ui_utils.py (Modifications for Step 3)
+"""Utility functions for the Tensorus Streamlit UI, now using API calls."""
+
+import requests
+import streamlit as st
+import logging
+from typing import List, Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# --- Configuration ---
+TENSORUS_API_URL = "http://127.0.0.1:7860" # Ensure FastAPI runs here
+
+# --- API Interaction Functions ---
+
+def get_api_status() -> bool:
+    """Checks if the Tensorus API is reachable."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/", timeout=2)
+        return response.status_code == 200
+    except requests.exceptions.ConnectionError:
+        return False
+    except Exception as e:
+        logger.error(f"Error checking API status: {e}")
+        return False
+
+def list_datasets() -> Optional[List[str]]:
+    """Fetches the list of dataset names from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/datasets")
+        response.raise_for_status()
+        data = response.json()
+        if data.get("success"):
+            return data.get("data", [])
+        else:
+            st.error(f"API Error listing datasets: {data.get('message')}")
+            return None
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error listing datasets: {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error listing datasets: {e}")
+        logger.exception("Unexpected error in list_datasets")
+        return None
+
+def fetch_dataset_data(dataset_name: str, offset: int = 0, limit: int = 50) -> Optional[List[Dict[str, Any]]]:
+    """Fetches a page of records from a dataset via API."""
+    try:
+        params = {"offset": offset, "limit": limit}
+        response = requests.get(f"{TENSORUS_API_URL}/datasets/{dataset_name}/records", params=params)
+        response.raise_for_status()
+        data = response.json()
+        if data.get("success"):
+            return data.get("data", [])
+        else:
+            st.error(f"API Error fetching '{dataset_name}': {data.get('message')}")
+            return None
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error fetching '{dataset_name}': {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error fetching '{dataset_name}': {e}")
+        logger.exception(f"Unexpected error in fetch_dataset_data for {dataset_name}")
+        return None
+
+def execute_nql_query(query: str) -> Optional[Dict[str, Any]]:
+    """Sends an NQL query to the API."""
+    try:
+        payload = {"query": query}
+        response = requests.post(f"{TENSORUS_API_URL}/query", json=payload)
+        # Handle specific NQL errors (400) vs other errors
+        if response.status_code == 400:
+             error_detail = response.json().get("detail", "Unknown NQL processing error")
+             return {"success": False, "message": error_detail, "results": None, "count": None}
+        response.raise_for_status() # Raise for 5xx etc.
+        return response.json() # Return the full NQLResponse structure
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error executing NQL query: {e}")
+        return {"success": False, "message": f"Connection Error: {e}", "results": None, "count": None}
+    except Exception as e:
+        st.error(f"Unexpected error executing NQL query: {e}")
+        logger.exception("Unexpected error in execute_nql_query")
+        return {"success": False, "message": f"Unexpected Error: {e}", "results": None, "count": None}
+
+# --- NEW/UPDATED Agent and Metrics Functions ---
+
+def list_all_agents() -> Optional[List[Dict[str, Any]]]:
+    """Fetches the list of all registered agents from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/agents")
+        response.raise_for_status()
+        # The response is directly the list of AgentInfo objects
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error listing agents: {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error listing agents: {e}")
+        logger.exception("Unexpected error in list_all_agents")
+        return None
+
+def get_agent_status(agent_id: str) -> Optional[Dict[str, Any]]:
+    """Fetches status for a specific agent from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/agents/{agent_id}/status")
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API.")
+            return None
+        response.raise_for_status()
+        # Returns AgentStatus model dict
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error getting status for agent '{agent_id}': {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error getting status for agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in get_agent_status for {agent_id}")
+        return None
+
+def get_agent_logs(agent_id: str, lines: int = 20) -> Optional[List[str]]:
+    """Fetches recent logs for a specific agent from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/agents/{agent_id}/logs", params={"lines": lines})
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API for logs.")
+            return None
+        response.raise_for_status()
+        data = response.json()
+        # Returns AgentLogResponse model dict
+        return data.get("logs", [])
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error getting logs for agent '{agent_id}': {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error getting logs for agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in get_agent_logs for {agent_id}")
+        return None
+
+def start_agent(agent_id: str) -> bool:
+    """Sends a start signal to an agent via the API."""
+    try:
+        response = requests.post(f"{TENSORUS_API_URL}/agents/{agent_id}/start")
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API.")
+            return False
+        # 202 Accepted is success, other 2xx might be okay too (e.g. already running if handled gracefully)
+        # 4xx errors indicate failure
+        if 200 <= response.status_code < 300:
+            api_response = response.json()
+            if api_response.get("success"):
+                 st.success(f"API: {api_response.get('message', 'Start signal sent.')}")
+                 return True
+            else:
+                 # API indicated logical failure (e.g., already running)
+                 st.warning(f"API: {api_response.get('message', 'Agent might already be running.')}")
+                 return False
+        else:
+             # Handle other potential errors reported by API
+             error_detail = "Unknown error"
+             try: error_detail = response.json().get("detail", error_detail)
+             except: pass
+             st.error(f"API Error starting agent '{agent_id}': {error_detail} (Status: {response.status_code})")
+             return False
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error starting agent '{agent_id}': {e}")
+        return False
+    except Exception as e:
+        st.error(f"Unexpected error starting agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in start_agent for {agent_id}")
+        return False
+
+def stop_agent(agent_id: str) -> bool:
+    """Sends a stop signal to an agent via the API."""
+    try:
+        response = requests.post(f"{TENSORUS_API_URL}/agents/{agent_id}/stop")
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API.")
+            return False
+        if 200 <= response.status_code < 300:
+            api_response = response.json()
+            if api_response.get("success"):
+                 st.success(f"API: {api_response.get('message', 'Stop signal sent.')}")
+                 return True
+            else:
+                 st.warning(f"API: {api_response.get('message', 'Agent might already be stopped.')}")
+                 return False
+        else:
+             error_detail = "Unknown error"
+             try: error_detail = response.json().get("detail", error_detail)
+             except: pass
+             st.error(f"API Error stopping agent '{agent_id}': {error_detail} (Status: {response.status_code})")
+             return False
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error stopping agent '{agent_id}': {e}")
+        return False
+    except Exception as e:
+        st.error(f"Unexpected error stopping agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in stop_agent for {agent_id}")
+        return False
+
+def get_dashboard_metrics() -> Optional[Dict[str, Any]]:
+    """Fetches dashboard metrics from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/metrics/dashboard")
+        response.raise_for_status()
+        # Returns DashboardMetrics model dict
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error fetching dashboard metrics: {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error fetching dashboard metrics: {e}")
+        logger.exception("Unexpected error in get_dashboard_metrics")
+        return None
+
+def get_agent_config(agent_id: str) -> Optional[Dict[str, Any]]:
+    """Fetch an agent's configuration from the API."""
+    try:
+        response = requests.get(f"{TENSORUS_API_URL}/agents/{agent_id}/config")
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API for configuration.")
+            return None
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error fetching config for agent '{agent_id}': {e}")
+        return None
+    except Exception as e:
+        st.error(f"Unexpected error fetching config for agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in get_agent_config for {agent_id}")
+        return None
+
+def update_agent_config(agent_id: str, config: Dict[str, Any]) -> bool:
+    """Send updated configuration for an agent to the API."""
+    try:
+        response = requests.post(
+            f"{TENSORUS_API_URL}/agents/{agent_id}/configure",
+            json={"config": config},
+        )
+        if response.status_code == 404:
+            st.error(f"Agent '{agent_id}' not found via API for configuration.")
+            return False
+        if 200 <= response.status_code < 300:
+            api_response = response.json()
+            if api_response.get("success"):
+                st.success(api_response.get("message", "Configuration updated."))
+                return True
+            else:
+                st.error(api_response.get("message", "Failed to update configuration."))
+                return False
+        else:
+            error_detail = "Unknown error"
+            try:
+                error_detail = response.json().get("detail", error_detail)
+            except Exception:
+                pass
+            st.error(
+                f"API Error updating config for '{agent_id}': {error_detail} (Status: {response.status_code})"
+            )
+            return False
+    except requests.exceptions.RequestException as e:
+        st.error(f"Connection Error updating config for agent '{agent_id}': {e}")
+        return False
+    except Exception as e:
+        st.error(f"Unexpected error updating config for agent '{agent_id}': {e}")
+        logger.exception(f"Unexpected error in update_agent_config for {agent_id}")
+        return False

pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "tensorus"
+version = "0.0.4"
+authors = [
+    { name = "Tensorus Team", email = "ai@tensorus.com" }
+]
+description = "An agentic tensor database and agent framework for managing, querying, and automating tensor data workflows."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+keywords = [
+    "tensor", "database", "agent", "ai", "pytorch", "fastapi", "streamlit", "automl", "reinforcement-learning", "data-ingestion"
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries"
+]
+[project.urls]
+Homepage = "https://tensorus.com"
+Repository = "https://github.com/tensorus/tensorus"
+
+[project.optional-dependencies]
+models = ["tensorus-models>=0.0.3"]
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["tensorus*"]
+namespaces = false

requirements-test.txt

@@ -0,0 +1,18 @@
+# requirements-test.txt
+# Python dependencies needed to run the test suite
+
+pytest
+pytest-asyncio
+httpx>=0.28.1
+scipy
+semopy>=2.3
+pydantic-settings>=2.0
+# FastAPI version supporting the Pydantic v2 API
+fastapi>=0.110
+numpy>=1.21.0
+torch
+tensorly
+transformers
+psycopg2-binary
+python-jose[cryptography]
+fastmcp>=0.2.0

requirements.txt

@@ -0,0 +1,54 @@
+# requirements.txt
+# Updated: 2024-07-26 # (Date updated)
+
+# --- Core Tensor and Numerics ---
+torch>=1.13.0
+torchvision>=0.14.0
+segmentation-models-pytorch
+transformers
+numpy>=1.21.0
+tensorly
+
+# --- Agent Specific Dependencies ---
+# For Ingestion Agent image processing example
+Pillow>=9.0.0
+
+# --- API Layer Dependencies ---
+# FastAPI with Pydantic v2 support
+fastapi>=0.110.0
+pydantic>=2.0.0
+pydantic-settings>=2.0
+# ASGI Server (standard includes extras like watchfiles for reload)
+uvicorn[standard]>=0.20.0
+fastmcp>=0.2.0
+# For PostgreSQL connectivity (used by PostgresMetadataStorage)
+psycopg2-binary>=2.9.0
+# Optional: Needed if using FastAPI file uploads via forms
+# python-multipart>=0.0.5
+
+# --- Streamlit UI Dependencies ---
+streamlit>=1.25.0
+# For calling the FastAPI backend from the Streamlit UI
+requests>=2.28.0
+# For JWT validation
+python-jose[cryptography]
+# For plotting in the Streamlit UI (Dashboard, Data Explorer)
+plotly>=5.10.0
+
+# --- Testing Dependencies ---
+pytest>=7.0.0
+httpx>=0.28.1 # For FastAPI TestClient
+
+# --- Data Analysis & Modeling ---
+# Optional: For plotting example in rl_agent.py
+matplotlib>=3.5.0
+# For Time Series Analysis (ARIMA model)
+scikit-learn>=1.3.0
+umap-learn
+pandas>=1.5.0
+arch>=5.7
+lifelines>=0.28
+semopy>=2.3
+gensim
+joblib
+opencv-python

setup.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Setup script for Tensorus development environment
+set -e
+
+# Install Python dependencies
+pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision
+pip install -r requirements.txt
+pip install -r requirements-test.txt
+
+if [[ "$INSTALL_MODELS" == "1" ]]; then
+  pip install -e .[models]
+fi

tensor_data/count_ds.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53a9272a45a56453ca8c3fbb9a463b86856367419eecc744bdada7a17a63205c
+size 2157

tensor_data/test_dataset_ingest_5dde494e-54cd-4fbc-bc87-e5949cc3809e.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a404c8b32dbc7c670076b5c02d470a5d0a60dc7da3e41450868ad97ccc3a6c62
+size 2432

tensor_data/test_dataset_ingest_743ec0f5-c46c-42b3-92fd-5b8e504b147c.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089e55853478bbbf88e4952df508300589746abb0b2834f78e05d74a906d8079
+size 2432

tensor_data/test_dataset_ingest_d17cec44-8f9f-4856-a343-3ef700784edb.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bca92e9ab4dfc7aec08d872fc264afca1acdf7683e46586ec269331f647acf1
+size 1747

tensor_data/test_dataset_ingest_f3e79d98-6fb4-496d-9763-f0ce3aa1c801.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf925fa73ab1fb4ecf6c4bf8436432d202161f15c18dfcbd6571340e1c525ea
+size 2432

tensor_data/test_dataset_integration_69bf9dbf-0ef4-4dd7-ba2d-6ba3bf31b799.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a88b22db97bf4b62a2025175c422772f95165030479757e6ff0ed5442e0da569
+size 1841

tensorus/__init__.py

@@ -0,0 +1,18 @@
+"""Tensorus core package."""
+
+import os
+
+# The repository previously exposed a large collection of models under
+# ``tensorus.models``. These models have been moved to a separate package.
+# Tensorus now only attempts to import them if available.
+
+if not os.environ.get("TENSORUS_MINIMAL_IMPORT"):
+    try:
+        import importlib
+
+        models = importlib.import_module("tensorus.models")
+        __all__ = ["models"]
+    except ModuleNotFoundError:
+        __all__ = []
+else:
+    __all__ = []

tensorus/api/__init__.py

@@ -0,0 +1,25 @@
+"""
+Tensorus API Package.
+
+This package contains the FastAPI application and endpoints for interacting
+with Tensor Descriptors and Semantic Metadata.
+
+To run the API (assuming Uvicorn is installed and you are in the project
+root):
+`uvicorn tensorus.api:app --reload --port 7860`
+"""
+
+# Re-export the FastAPI application defined in ``tensorus/api.py`` so users
+# can simply ``import tensorus.api`` to access ``app``. We load the module by
+# path to avoid import order issues with the ``tensorus.api`` package itself.
+import importlib.util
+from pathlib import Path
+
+_spec = importlib.util.spec_from_file_location(
+    "tensorus.api_app", Path(__file__).resolve().parent.parent / "api.py"
+)
+_api = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_api)  # type: ignore[call-arg]
+app = _api.app
+
+__all__ = ["app"]

tensorus/api/dependencies.py

@@ -0,0 +1,36 @@
+from tensorus.metadata import storage_instance as globally_configured_storage_instance
+from tensorus.metadata.storage_abc import MetadataStorage
+
+# Note: The `storage_instance` imported here is already configured (InMemory or Postgres)
+# based on the logic in `tensorus/metadata/__init__.py` which reads `tensorus.config.settings`.
+
+def get_storage_instance() -> MetadataStorage:
+    """
+    FastAPI dependency to get the currently configured metadata storage instance.
+    """
+    return globally_configured_storage_instance
+
+# If we needed to re-initialize or pass settings directly to the storage for each request
+# (e.g. for request-scoped sessions or dynamic configuration per request, which is not the case here),
+# this function would be more complex. For now, it just returns the global instance.
+#
+# Example of re-initializing if storage_instance could change or needs request context:
+# from tensorus.metadata import get_configured_storage_instance, ConfigurationError
+# from tensorus.config import settings
+#
+# def get_storage_instance_dynamic() -> MetadataStorage:
+#     try:
+#         # This would re-evaluate settings and re-create the instance per request if needed,
+#         # or could access request-specific config.
+#         # For our current setup, the global instance is fine.
+#         return get_configured_storage_instance()
+#     except ConfigurationError as e:
+#         # This would ideally be caught by a global exception handler in FastAPI
+#         # to return a 500 error if configuration is bad during a request.
+#         # However, configuration should typically be validated at startup.
+#         raise RuntimeError(f"Storage configuration error: {e}")
+
+# The current `storage_instance` is initialized once when `tensorus.metadata` is first imported.
+# This is generally fine for many applications unless the configuration needs to change without restarting.
+# The FastAPI `Depends` system will call `get_storage_instance` for each request that uses it,
+# but this function will always return the same globally initialized `storage_instance`.

tensorus/api/endpoints.py

@@ -0,0 +1,601 @@
+from typing import List, Dict, Optional, Any, Annotated, Literal
+from uuid import UUID
+from datetime import datetime
+
+from fastapi import APIRouter, HTTPException, Body, Query, Path
+from pydantic import BaseModel, ValidationError
+
+from tensorus.metadata.schemas import (
+    TensorDescriptor, SemanticMetadata, DataType,
+    LineageSourceType, LineageMetadata, ParentTensorLink,
+    ComputationalMetadata, QualityMetadata, RelationalMetadata, UsageMetadata,
+)
+from tensorus.metadata.storage_abc import MetadataStorage
+from tensorus.api.dependencies import get_storage_instance
+from tensorus.storage.connectors import mock_tensor_connector_instance
+from tensorus.metadata.schemas_iodata import TensorusExportData
+
+from pydantic import BaseModel as PydanticBaseModel
+from typing import TypeVar, Generic
+
+from fastapi import Depends, Security, status # Added status for HTTPException
+from fastapi.responses import JSONResponse
+from .security import verify_api_key, api_key_header_auth
+from tensorus.audit import log_audit_event
+
+import copy
+import uuid
+
+# Router for TensorDescriptor
+router_tensor_descriptor = APIRouter(
+    prefix="/tensor_descriptors",
+    tags=["Tensor Descriptors"],
+    responses={404: {"description": "Not found"}},
+)
+
+# Router for SemanticMetadata
+router_semantic_metadata = APIRouter(
+    prefix="/tensor_descriptors/{tensor_id}/semantic", # Corrected prefix for consistency
+    tags=["Semantic Metadata (Per Tensor)"],
+    responses={404: {"description": "Not found"}},
+)
+
+# --- TensorDescriptor Endpoints ---
+@router_tensor_descriptor.post("/", response_model=TensorDescriptor, status_code=status.HTTP_201_CREATED)
+async def create_tensor_descriptor(
+    descriptor_data: Dict[str, Any],
+    storage: MetadataStorage = Depends(get_storage_instance),
+    api_key: str = Depends(verify_api_key)
+):
+    tensor_id_str = descriptor_data.get("tensor_id")
+    temp_id_for_lookup = UUID(tensor_id_str) if tensor_id_str else uuid.uuid4()
+
+    fields_to_fetch = ["shape", "data_type", "byte_size", "dimensionality"]
+    missing_fields = [field for field in fields_to_fetch if field not in descriptor_data or descriptor_data[field] is None]
+
+    if missing_fields:
+        storage_details = mock_tensor_connector_instance.get_tensor_details(temp_id_for_lookup)
+        if storage_details:
+            for field in missing_fields:
+                if field in storage_details and (field not in descriptor_data or descriptor_data[field] is None) :
+                    descriptor_data[field] = storage_details[field]
+            if "shape" in descriptor_data and descriptor_data["shape"] is not None \
+               and ("dimensionality" not in descriptor_data or descriptor_data["dimensionality"] is None):
+                descriptor_data["dimensionality"] = len(descriptor_data["shape"])
+    try:
+        final_descriptor = TensorDescriptor(**descriptor_data)
+        storage.add_tensor_descriptor(final_descriptor)
+        if mock_tensor_connector_instance.retrieve_tensor(final_descriptor.tensor_id) is None:
+            mock_tensor_data_payload = {
+                "shape": final_descriptor.shape, "data_type": final_descriptor.data_type.value,
+                "byte_size": final_descriptor.byte_size, "info": "Placeholder data by create_tensor_descriptor"
+            }
+            mock_tensor_connector_instance.store_tensor(final_descriptor.tensor_id, mock_tensor_data_payload)
+        log_audit_event(action="CREATE_TENSOR_DESCRIPTOR", user=api_key, tensor_id=str(final_descriptor.tensor_id),
+                        details={"owner": final_descriptor.owner, "data_type": final_descriptor.data_type.value})
+        return final_descriptor
+    except ValidationError as e:
+        log_audit_event(action="CREATE_TENSOR_DESCRIPTOR_FAILED_VALIDATION", user=api_key, details={"error": str(e), "input_tensor_id": tensor_id_str})
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors())
+    except ValueError as e:
+        log_audit_event(action="CREATE_TENSOR_DESCRIPTOR_FAILED_STORAGE", user=api_key, details={"error": str(e), "input_tensor_id": tensor_id_str})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+@router_tensor_descriptor.get("/", response_model=List[TensorDescriptor], summary="List Tensor Descriptors with Advanced Filtering")
+async def list_tensor_descriptors(
+    owner: Optional[str] = Query(None), data_type: Optional[DataType] = Query(None),
+    tags_contain: Optional[List[str]] = Query(None), lineage_version: Optional[str] = Query(None, alias="lineage.version"),
+    lineage_source_type: Optional[LineageSourceType] = Query(None, alias="lineage.source.type"),
+    comp_algorithm: Optional[str] = Query(None, alias="computational.algorithm"),
+    comp_gpu_model: Optional[str] = Query(None, alias="computational.hardware_info.gpu_model"),
+    quality_confidence_gt: Optional[float] = Query(None, alias="quality.confidence_score_gt"),
+    quality_noise_lt: Optional[float] = Query(None, alias="quality.noise_level_lt"),
+    rel_collection: Optional[str] = Query(None, alias="relational.collection"),
+    rel_has_related_tensor_id: Optional[UUID] = Query(None, alias="relational.has_related_tensor_id"),
+    usage_last_accessed_before: Optional[datetime] = Query(None, alias="usage.last_accessed_before"),
+    usage_used_by_app: Optional[str] = Query(None, alias="usage.used_by_app"),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    all_descriptors = storage.list_tensor_descriptors(
+        owner=owner, data_type=data_type, tags_contain=tags_contain, lineage_version=lineage_version
+    ) # Pass some common filters
+    filtered_descriptors = []
+    for desc in all_descriptors: # Apply remaining filters in memory
+        if lineage_source_type and (not (lm := storage.get_lineage_metadata(desc.tensor_id)) or not lm.source or lm.source.type != lineage_source_type): continue
+        if comp_algorithm or comp_gpu_model:
+            cm = storage.get_computational_metadata(desc.tensor_id)
+            if not cm: continue
+            if comp_algorithm and cm.algorithm != comp_algorithm: continue
+            if comp_gpu_model and (not cm.hardware_info or cm.hardware_info.get("gpu_model") != comp_gpu_model): continue
+        if quality_confidence_gt or quality_noise_lt:
+            qm = storage.get_quality_metadata(desc.tensor_id)
+            if not qm: continue
+            if quality_confidence_gt and (qm.confidence_score is None or qm.confidence_score <= quality_confidence_gt): continue
+            if quality_noise_lt and (qm.noise_level is None or qm.noise_level >= quality_noise_lt): continue
+        if rel_collection or rel_has_related_tensor_id:
+            rm = storage.get_relational_metadata(desc.tensor_id)
+            if not rm: continue
+            if rel_collection and rel_collection not in rm.collections: continue
+            if rel_has_related_tensor_id and not any(rtl.related_tensor_id == rel_has_related_tensor_id for rtl in rm.related_tensors): continue
+        if usage_last_accessed_before or usage_used_by_app:
+            um = storage.get_usage_metadata(desc.tensor_id)
+            if not um: continue
+            if usage_last_accessed_before and (not um.last_accessed_at or um.last_accessed_at >= usage_last_accessed_before): continue
+            if usage_used_by_app and usage_used_by_app not in um.application_references: continue
+        filtered_descriptors.append(desc)
+    return filtered_descriptors
+
+@router_tensor_descriptor.get("/{tensor_id}", response_model=TensorDescriptor)
+async def get_tensor_descriptor(tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance)):
+    descriptor = storage.get_tensor_descriptor(tensor_id)
+    if not descriptor: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TensorDescriptor ID {tensor_id} not found.")
+    return descriptor
+
+class TensorDescriptorUpdate(BaseModel):
+    dimensionality: Optional[int]=None; shape: Optional[List[int]]=None; data_type: Optional[DataType]=None
+    storage_format: Optional[str]=None; owner: Optional[str]=None; access_control: Optional[Dict[str, List[str]]]=None
+    byte_size: Optional[int]=None; compression_info: Optional[Dict[str, Any]]=None
+    tags: Optional[List[str]]=None; metadata: Optional[Dict[str, Any]]=None
+
+@router_tensor_descriptor.put("/{tensor_id}", response_model=TensorDescriptor)
+async def update_tensor_descriptor(
+    tensor_id: UUID = Path(...), updates: TensorDescriptorUpdate = Body(...),
+    storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    update_data = updates.model_dump(exclude_unset=True)
+    if not update_data: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No update data provided.")
+    current = storage.get_tensor_descriptor(tensor_id)
+    if not current:
+        log_audit_event("UPDATE_TENSOR_DESCRIPTOR_FAILED_NOT_FOUND", api_key, str(tensor_id))
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TensorDescriptor ID {tensor_id} not found.")
+    try:
+        updated = storage.update_tensor_descriptor(tensor_id, **update_data)
+        log_audit_event("UPDATE_TENSOR_DESCRIPTOR", api_key, str(tensor_id), {"updated_fields": list(update_data.keys())})
+        return updated
+    except (ValidationError, ValueError) as e:
+        log_audit_event("UPDATE_TENSOR_DESCRIPTOR_FAILED_VALIDATION", api_key, str(tensor_id), {"error": str(e)})
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY if isinstance(e, ValidationError) else status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+@router_tensor_descriptor.delete("/{tensor_id}", status_code=status.HTTP_200_OK)
+async def delete_tensor_descriptor(
+    tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    if not storage.get_tensor_descriptor(tensor_id):
+        log_audit_event("DELETE_TENSOR_DESCRIPTOR_FAILED_NOT_FOUND", api_key, str(tensor_id))
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TensorDescriptor ID {tensor_id} not found.")
+    storage.delete_tensor_descriptor(tensor_id)
+    mock_tensor_connector_instance.delete_tensor(tensor_id)
+    log_audit_event("DELETE_TENSOR_DESCRIPTOR", api_key, str(tensor_id))
+    return {"message": f"TensorDescriptor {tensor_id} and associated data deleted."}
+
+# --- SemanticMetadata Endpoints ---
+def _check_td_exists_for_semantic(tensor_id: UUID, storage: MetadataStorage, api_key: Optional[str] = None, action_prefix: str = ""):
+    if not storage.get_tensor_descriptor(tensor_id):
+        if api_key and action_prefix:
+            log_audit_event(f"{action_prefix}_SEMANTIC_METADATA_FAILED_TD_NOT_FOUND", user=api_key, tensor_id=str(tensor_id))
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Parent TensorDescriptor ID {tensor_id} not found.")
+
+@router_semantic_metadata.post("/", response_model=SemanticMetadata, status_code=status.HTTP_201_CREATED)
+async def create_semantic_metadata_for_tensor(
+    tensor_id: UUID = Path(...), metadata_in: SemanticMetadata = Body(...),
+    storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    _check_td_exists_for_semantic(tensor_id, storage, api_key, "CREATE")
+    if metadata_in.tensor_id != tensor_id:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="tensor_id in path and body must match.")
+    try:
+        storage.add_semantic_metadata(metadata_in)
+        log_audit_event("CREATE_SEMANTIC_METADATA", api_key, str(tensor_id), {"name": metadata_in.name})
+        return metadata_in
+    except (ValidationError, ValueError) as e:
+        log_audit_event("CREATE_SEMANTIC_METADATA_FAILED", api_key, str(tensor_id), {"name": metadata_in.name, "error": str(e)})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST if isinstance(e, ValueError) else status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
+
+@router_semantic_metadata.get("/", response_model=List[SemanticMetadata])
+async def get_all_semantic_metadata_for_tensor(tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance)):
+    _check_td_exists_for_semantic(tensor_id, storage)
+    return storage.get_semantic_metadata(tensor_id)
+
+class SemanticMetadataUpdate(BaseModel): name: Optional[str] = None; description: Optional[str] = None
+
+@router_semantic_metadata.put("/{current_name}", response_model=SemanticMetadata)
+async def update_named_semantic_metadata_for_tensor(
+    tensor_id: UUID = Path(...), current_name: str = Path(...), updates: SemanticMetadataUpdate = Body(...),
+    storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    _check_td_exists_for_semantic(tensor_id, storage, api_key, "UPDATE")
+    if not updates.model_dump(exclude_unset=True):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No update data provided.")
+    if not storage.get_semantic_metadata_by_name(tensor_id, current_name):
+        log_audit_event("UPDATE_SEMANTIC_METADATA_FAILED_NOT_FOUND", api_key, str(tensor_id), {"name": current_name})
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"SemanticMetadata '{current_name}' not found for tensor {tensor_id}.")
+    try:
+        updated = storage.update_semantic_metadata(tensor_id, current_name, new_description=updates.description, new_name=updates.name)
+        log_audit_event(
+            "UPDATE_SEMANTIC_METADATA",
+            api_key,
+            str(tensor_id),
+            {"original_name": current_name, "updated_fields": updates.model_dump(exclude_unset=True)},
+        )
+        return updated
+    except (ValidationError, ValueError) as e:
+        log_audit_event("UPDATE_SEMANTIC_METADATA_FAILED", api_key, str(tensor_id), {"name": current_name, "error": str(e)})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST if isinstance(e, ValueError) else status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
+
+@router_semantic_metadata.delete("/{name}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_named_semantic_metadata_for_tensor(
+    tensor_id: UUID = Path(...), name: str = Path(...),
+    storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    _check_td_exists_for_semantic(tensor_id, storage, api_key, "DELETE")
+    if not storage.get_semantic_metadata_by_name(tensor_id, name):
+        log_audit_event("DELETE_SEMANTIC_METADATA_FAILED_NOT_FOUND", api_key, str(tensor_id), {"name": name})
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"SemanticMetadata '{name}' not found for tensor {tensor_id}.")
+    storage.delete_semantic_metadata(tensor_id, name)
+    log_audit_event("DELETE_SEMANTIC_METADATA", api_key, str(tensor_id), {"name": name})
+    return None
+
+# --- Search and Aggregation Routers (GET - No Auth/Audit) ---
+router_search_aggregate = APIRouter(tags=["Search & Aggregate"])
+@router_search_aggregate.get("/search/tensors/", response_model=List[TensorDescriptor])
+async def search_tensors(
+    text_query: str = Query(..., min_length=1), fields_to_search: Optional[List[str]] = Query(None),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    default_fields = ["tensor_id", "owner", "tags", "metadata", "semantic.name", "semantic.description", "lineage.source.identifier", "lineage.version", "computational.algorithm"]
+    try: return storage.search_tensor_descriptors(text_query, fields_to_search or default_fields)
+    except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+@router_search_aggregate.get("/aggregate/tensors/", response_model=Dict[str, Any])
+async def aggregate_tensors(
+    group_by_field: str = Query(...), agg_function: str = Query(...), agg_field: Optional[str] = Query(None),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    if agg_function in ["avg", "sum", "min", "max"] and not agg_field:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"'{agg_function}' requires 'agg_field'.")
+    try: return storage.aggregate_tensor_descriptors(group_by_field, agg_function, agg_field)
+    except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+    except NotImplementedError as e: raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(e))
+
+# --- Versioning and Lineage Router ---
+router_version_lineage = APIRouter(tags=["Versioning & Lineage"])
+class NewTensorVersionRequest(BaseModel):
+    new_version_string: str; dimensionality: Optional[int]=None; shape: Optional[List[int]]=None; data_type: Optional[DataType]=None
+    storage_format: Optional[str]=None; owner: Optional[str]=None; access_control: Optional[Dict[str, List[str]]]=None
+    byte_size: Optional[int]=None; checksum: Optional[str]=None; compression_info: Optional[Dict[str, Any]]=None
+    tags: Optional[List[str]]=None; metadata: Optional[Dict[str, Any]]=None
+    lineage_source_identifier: Optional[str]=None; lineage_source_type: Optional[LineageSourceType]=None
+
+@router_version_lineage.post("/tensors/{tensor_id}/versions", response_model=TensorDescriptor, status_code=status.HTTP_201_CREATED)
+async def create_tensor_version(
+    tensor_id: UUID = Path(...), version_request: NewTensorVersionRequest = Body(...),
+    storage: MetadataStorage = Depends(get_storage_instance), api_key: str = Depends(verify_api_key)
+):
+    parent_td = storage.get_tensor_descriptor(tensor_id)
+    if not parent_td:
+        log_audit_event("CREATE_TENSOR_VERSION_FAILED_PARENT_NOT_FOUND", api_key, str(tensor_id), {"new_version": version_request.new_version_string})
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Parent TensorDescriptor ID {tensor_id} not found.")
+    new_version_id = uuid.uuid4()
+    new_td_data = parent_td.model_dump(
+        exclude={"tensor_id", "creation_timestamp", "last_modified_timestamp"}
+    )
+    for field, value in version_request.model_dump(exclude_unset=True).items():
+        if field in TensorDescriptor.model_fields and value is not None: new_td_data[field] = value
+        elif field not in ['new_version_string', 'lineage_source_identifier', 'lineage_source_type']:
+            if new_td_data.get("metadata") is None: new_td_data["metadata"] = {}
+            new_td_data["metadata"][field] = value
+    new_td_data.update({
+        "tensor_id": new_version_id, "creation_timestamp": datetime.utcnow(),
+        "last_modified_timestamp": datetime.utcnow(),
+        "owner": new_td_data.get('owner', parent_td.owner),
+        "byte_size": new_td_data.get('byte_size', parent_td.byte_size)
+    })
+    try:
+        new_td = TensorDescriptor(**new_td_data)
+        storage.add_tensor_descriptor(new_td)
+    except (ValidationError, ValueError) as e:
+        log_audit_event("CREATE_TENSOR_VERSION_FAILED_VALIDATION", api_key, str(new_version_id), {"parent_id": str(tensor_id), "error": str(e)})
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY if isinstance(e, ValidationError) else status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+    lineage_details = {"tensor_id": new_version_id, "parent_tensors": [ParentTensorLink(tensor_id=tensor_id, relationship="new_version_of")], "version": version_request.new_version_string}
+    if version_request.lineage_source_identifier and version_request.lineage_source_type:
+        lineage_details["source"] = LineageSource(type=version_request.lineage_source_type, identifier=version_request.lineage_source_identifier) # type: ignore
+    storage.add_lineage_metadata(LineageMetadata(**lineage_details))
+    log_audit_event("CREATE_TENSOR_VERSION", api_key, str(new_version_id), {"parent_id": str(tensor_id), "version": version_request.new_version_string})
+    return new_td
+
+@router_version_lineage.get("/tensors/{tensor_id}/versions", response_model=List[TensorDescriptor])
+async def list_tensor_versions(tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance)):
+    results = []
+    current_td = storage.get_tensor_descriptor(tensor_id)
+    if not current_td: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TensorDescriptor ID {tensor_id} not found.")
+    results.append(current_td)
+    child_ids = storage.get_child_tensor_ids(tensor_id)
+    for child_id in child_ids:
+        child_lineage = storage.get_lineage_metadata(child_id)
+        if child_lineage and any(p.tensor_id == tensor_id and p.relationship == "new_version_of" for p in child_lineage.parent_tensors):
+            child_td_obj = storage.get_tensor_descriptor(child_id)
+            if child_td_obj: results.append(child_td_obj)
+    return results
+
+class LineageRelationshipRequest(BaseModel): source_tensor_id: UUID; target_tensor_id: UUID; relationship_type: str; details: Optional[Dict[str, Any]] = None
+
+@router_version_lineage.post("/lineage/relationships/", status_code=status.HTTP_201_CREATED)
+async def create_lineage_relationship(
+    req: LineageRelationshipRequest, storage: MetadataStorage = Depends(get_storage_instance),
+    api_key: str = Depends(verify_api_key)
+):
+    audit_details = req.model_dump()
+    if not storage.get_tensor_descriptor(req.source_tensor_id):
+        log_audit_event("CREATE_LINEAGE_REL_FAILED_SRC_NOT_FOUND", api_key, details=audit_details)
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Source TD {req.source_tensor_id} not found.")
+    if not storage.get_tensor_descriptor(req.target_tensor_id):
+        log_audit_event("CREATE_LINEAGE_REL_FAILED_TGT_NOT_FOUND", api_key, details=audit_details)
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Target TD {req.target_tensor_id} not found.")
+
+    target_lineage = storage.get_lineage_metadata(req.target_tensor_id) or LineageMetadata(tensor_id=req.target_tensor_id)
+    if any(p.tensor_id == req.source_tensor_id and p.relationship == req.relationship_type for p in target_lineage.parent_tensors):
+        return {"message": "Relationship already exists.", "lineage": target_lineage}
+
+    target_lineage.parent_tensors.append(ParentTensorLink(tensor_id=req.source_tensor_id, relationship=req.relationship_type))
+    try:
+        storage.add_lineage_metadata(target_lineage)
+        log_audit_event("CREATE_LINEAGE_RELATIONSHIP", api_key, str(req.target_tensor_id), details=audit_details)
+        return {"message": "Lineage relationship created/updated.", "lineage": storage.get_lineage_metadata(req.target_tensor_id)}
+    except (ValidationError, ValueError) as e:
+        log_audit_event("CREATE_LINEAGE_REL_FAILED_VALIDATION", api_key, str(req.target_tensor_id), {**audit_details, "error": str(e)})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+@router_version_lineage.get("/tensors/{tensor_id}/lineage/parents", response_model=List[TensorDescriptor])
+async def get_parent_tensors(tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance)):
+    if not storage.get_tensor_descriptor(tensor_id): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TD {tensor_id} not found.")
+    parent_ids = storage.get_parent_tensor_ids(tensor_id)
+    return [td for pid in parent_ids if (td := storage.get_tensor_descriptor(pid)) is not None]
+
+@router_version_lineage.get("/tensors/{tensor_id}/lineage/children", response_model=List[TensorDescriptor])
+async def get_child_tensors(tensor_id: UUID = Path(...), storage: MetadataStorage = Depends(get_storage_instance)):
+    if not storage.get_tensor_descriptor(tensor_id): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"TD {tensor_id} not found.")
+    child_ids = storage.get_child_tensor_ids(tensor_id)
+    return [td for cid in child_ids if (td := storage.get_tensor_descriptor(cid)) is not None]
+
+# --- Router for Extended Metadata (CRUD per type) ---
+router_extended_metadata = APIRouter(prefix="/tensor_descriptors/{tensor_id}", tags=["Extended Metadata (Per Tensor)"])
+
+def _get_td_or_404_for_extended_meta(tensor_id: UUID, storage: MetadataStorage, api_key: Optional[str]=None, action_prefix: str = ""): # Renamed from previous version
+    td = storage.get_tensor_descriptor(tensor_id)
+    if not td:
+        if api_key and action_prefix: log_audit_event(f"{action_prefix}_METADATA_FAILED_TD_NOT_FOUND", user=api_key, tensor_id=str(tensor_id)) # Generic prefix
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Parent TensorDescriptor ID {tensor_id} not found.")
+    return td
+
+async def _upsert_extended_metadata(tensor_id: UUID, metadata_name_cap: str, metadata_in: Any, storage: MetadataStorage, api_key: str):
+    _get_td_or_404_for_extended_meta(tensor_id, storage, api_key, f"UPSERT_{metadata_name_cap}")
+    if metadata_in.tensor_id != tensor_id:
+        log_audit_event(f"UPSERT_{metadata_name_cap}_FAILED_ID_MISMATCH", api_key, str(tensor_id), {"body_id": str(metadata_in.tensor_id)})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Path tensor_id and body tensor_id must match.")
+    try:
+        add_method = getattr(storage, f"add_{metadata_name_cap.lower()}_metadata")
+        add_method(metadata_in)
+        log_audit_event(f"UPSERT_{metadata_name_cap}_METADATA", api_key, str(tensor_id))
+        return metadata_in
+    except (ValidationError, ValueError) as e:
+        log_audit_event(f"UPSERT_{metadata_name_cap}_FAILED", api_key, str(tensor_id), {"error": str(e)})
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST if isinstance(e, ValueError) else status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
+
+async def _get_extended_metadata_ep(tensor_id: UUID, metadata_name_cap: str, storage: MetadataStorage): # Renamed to avoid clash
+    _get_td_or_404_for_extended_meta(tensor_id, storage)
+    get_method = getattr(storage, f"get_{metadata_name_cap.lower()}_metadata")
+    meta = get_method(tensor_id)
+    if not meta: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"{metadata_name_cap}Metadata not found for tensor {tensor_id}.")
+    return meta
+
+async def _patch_extended_metadata(tensor_id: UUID, metadata_name_cap: str, updates: Dict[str, Any], storage: MetadataStorage, api_key: str):
+    _get_td_or_404_for_extended_meta(tensor_id, storage, api_key, f"PATCH_{metadata_name_cap}")
+    get_method = getattr(storage, f"get_{metadata_name_cap.lower()}_metadata")
+    if not get_method(tensor_id):
+        log_audit_event(f"PATCH_{metadata_name_cap}_FAILED_NOT_FOUND", api_key, str(tensor_id))
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"{metadata_name_cap}Metadata not found for update.")
+    try:
+        update_method = getattr(storage, f"update_{metadata_name_cap.lower()}_metadata")
+        updated_meta = update_method(tensor_id, **updates)
+        log_audit_event(f"PATCH_{metadata_name_cap}_METADATA", api_key, str(tensor_id), {"updated_fields": list(updates.keys())})
+        return updated_meta
+    except (ValidationError, ValueError) as e:
+        log_audit_event(f"PATCH_{metadata_name_cap}_FAILED_VALIDATION", api_key, str(tensor_id), {"error": str(e)})
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY if isinstance(e, ValidationError) else status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+async def _delete_extended_metadata(tensor_id: UUID, metadata_name_cap: str, storage: MetadataStorage, api_key: str):
+    _get_td_or_404_for_extended_meta(tensor_id, storage, api_key, f"DELETE_{metadata_name_cap}")
+    delete_method = getattr(storage, f"delete_{metadata_name_cap.lower()}_metadata")
+    if not delete_method(tensor_id):
+        log_audit_event(f"DELETE_{metadata_name_cap}_FAILED_NOT_FOUND", api_key, str(tensor_id))
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"{metadata_name_cap}Metadata not found for delete.")
+    log_audit_event(f"DELETE_{metadata_name_cap}_METADATA", api_key, str(tensor_id))
+    return None
+
+# Explicit CRUD endpoints for each extended metadata type
+@router_extended_metadata.post("/lineage", response_model=LineageMetadata, status_code=status.HTTP_201_CREATED)
+async def upsert_lineage_metadata_ep(tensor_id: UUID=Path(...), lineage_in: LineageMetadata=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _upsert_extended_metadata(tensor_id, "Lineage", lineage_in, storage, api_key)
+@router_extended_metadata.get("/lineage", response_model=LineageMetadata)
+async def get_lineage_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance)):
+    return await _get_extended_metadata_ep(tensor_id, "Lineage", storage)
+@router_extended_metadata.patch("/lineage", response_model=LineageMetadata)
+async def patch_lineage_metadata_ep(tensor_id: UUID=Path(...), updates: Dict[str,Any]=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _patch_extended_metadata(tensor_id, "Lineage", updates, storage, api_key)
+@router_extended_metadata.delete("/lineage", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_lineage_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _delete_extended_metadata(tensor_id, "Lineage", storage, api_key)
+
+@router_extended_metadata.post("/computational", response_model=ComputationalMetadata, status_code=status.HTTP_201_CREATED)
+async def upsert_computational_metadata_ep(tensor_id: UUID=Path(...), computational_in: ComputationalMetadata=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _upsert_extended_metadata(tensor_id, "Computational", computational_in, storage, api_key)
+@router_extended_metadata.get("/computational", response_model=ComputationalMetadata)
+async def get_computational_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance)):
+    return await _get_extended_metadata_ep(tensor_id, "Computational", storage)
+@router_extended_metadata.patch("/computational", response_model=ComputationalMetadata)
+async def patch_computational_metadata_ep(tensor_id: UUID=Path(...), updates: Dict[str,Any]=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _patch_extended_metadata(tensor_id, "Computational", updates, storage, api_key)
+@router_extended_metadata.delete("/computational", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_computational_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _delete_extended_metadata(tensor_id, "Computational", storage, api_key)
+
+@router_extended_metadata.post("/quality", response_model=QualityMetadata, status_code=status.HTTP_201_CREATED)
+async def upsert_quality_metadata_ep(tensor_id: UUID=Path(...), quality_in: QualityMetadata=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _upsert_extended_metadata(tensor_id, "Quality", quality_in, storage, api_key)
+@router_extended_metadata.get("/quality", response_model=QualityMetadata)
+async def get_quality_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance)):
+    return await _get_extended_metadata_ep(tensor_id, "Quality", storage)
+@router_extended_metadata.patch("/quality", response_model=QualityMetadata)
+async def patch_quality_metadata_ep(tensor_id: UUID=Path(...), updates: Dict[str,Any]=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _patch_extended_metadata(tensor_id, "Quality", updates, storage, api_key)
+@router_extended_metadata.delete("/quality", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_quality_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _delete_extended_metadata(tensor_id, "Quality", storage, api_key)
+
+@router_extended_metadata.post("/relational", response_model=RelationalMetadata, status_code=status.HTTP_201_CREATED)
+async def upsert_relational_metadata_ep(tensor_id: UUID=Path(...), relational_in: RelationalMetadata=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _upsert_extended_metadata(tensor_id, "Relational", relational_in, storage, api_key)
+@router_extended_metadata.get("/relational", response_model=RelationalMetadata)
+async def get_relational_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance)):
+    return await _get_extended_metadata_ep(tensor_id, "Relational", storage)
+@router_extended_metadata.patch("/relational", response_model=RelationalMetadata)
+async def patch_relational_metadata_ep(tensor_id: UUID=Path(...), updates: Dict[str,Any]=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _patch_extended_metadata(tensor_id, "Relational", updates, storage, api_key)
+@router_extended_metadata.delete("/relational", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_relational_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _delete_extended_metadata(tensor_id, "Relational", storage, api_key)
+
+@router_extended_metadata.post("/usage", response_model=UsageMetadata, status_code=status.HTTP_201_CREATED)
+async def upsert_usage_metadata_ep(tensor_id: UUID=Path(...), usage_in: UsageMetadata=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _upsert_extended_metadata(tensor_id, "Usage", usage_in, storage, api_key)
+@router_extended_metadata.get("/usage", response_model=UsageMetadata)
+async def get_usage_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance)):
+    return await _get_extended_metadata_ep(tensor_id, "Usage", storage)
+@router_extended_metadata.patch("/usage", response_model=UsageMetadata)
+async def patch_usage_metadata_ep(tensor_id: UUID=Path(...), updates: Dict[str,Any]=Body(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _patch_extended_metadata(tensor_id, "Usage", updates, storage, api_key)
+@router_extended_metadata.delete("/usage", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_usage_metadata_ep(tensor_id: UUID=Path(...), storage: MetadataStorage=Depends(get_storage_instance), api_key: str=Depends(verify_api_key)):
+    return await _delete_extended_metadata(tensor_id, "Usage", storage, api_key)
+
+# --- I/O Router for Export/Import ---
+router_io = APIRouter(prefix="/tensors", tags=["Import/Export"])
+
+@router_io.get("/export", response_model=TensorusExportData)
+async def export_tensor_metadata(
+    tensor_ids_str: Optional[str] = Query(None, alias="tensor_ids"), # Changed FastAPIQuery to Query
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    parsed_tensor_ids: Optional[List[UUID]] = None
+    if tensor_ids_str:
+        try: parsed_tensor_ids = [UUID(tid.strip()) for tid in tensor_ids_str.split(',')]
+        except ValueError: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid UUID format in tensor_ids.")
+    export_data = storage.get_export_data(tensor_ids=parsed_tensor_ids)
+    filename = f"tensorus_export_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.json"
+    headers = {"Content-Disposition": f'attachment; filename="{filename}"'}
+    return JSONResponse(content=export_data.model_dump(mode="json"), headers=headers)
+
+@router_io.post("/import", summary="Import Tensor Metadata")
+async def import_tensor_metadata(
+    import_data_payload: TensorusExportData,
+    conflict_strategy: Annotated[Literal["skip", "overwrite"], Query()] = "skip",
+    storage: MetadataStorage = Depends(get_storage_instance),
+    api_key: str = Depends(verify_api_key)
+):
+    try:
+        result_summary = storage.import_data(import_data_payload, conflict_strategy=conflict_strategy)
+        log_audit_event("IMPORT_DATA", api_key, details={"strategy": conflict_strategy, "summary": result_summary})
+        return result_summary
+    except NotImplementedError:
+        log_audit_event("IMPORT_DATA_FAILED_NOT_IMPLEMENTED", api_key, details={"strategy": conflict_strategy})
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Import functionality is not implemented")
+    except Exception as e:
+        log_audit_event("IMPORT_DATA_FAILED_UNEXPECTED", api_key, details={"strategy": conflict_strategy, "error": str(e)})
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Unexpected error during import: {e}")
+
+# --- Management Router for Health and Metrics ---
+router_management = APIRouter(tags=["Management"])
+class HealthResponse(PydanticBaseModel): status: str; backend: str; detail: Optional[str] = None
+@router_management.get("/health", response_model=HealthResponse)
+async def health_check(storage: MetadataStorage = Depends(get_storage_instance)):
+    is_healthy, backend_type = storage.check_health()
+    if is_healthy: return HealthResponse(status="ok", backend=backend_type)
+    else: return JSONResponse(status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                              content=HealthResponse(status="error", backend=backend_type, detail="Storage backend connection failed.").model_dump())
+class MetricsResponse(PydanticBaseModel):
+    total_tensor_descriptors: int; semantic_metadata_count: int; lineage_metadata_count: int
+    computational_metadata_count: int; quality_metadata_count: int; relational_metadata_count: int; usage_metadata_count: int
+@router_management.get("/metrics", response_model=MetricsResponse)
+async def get_metrics(storage: MetadataStorage = Depends(get_storage_instance)):
+    return MetricsResponse(
+        total_tensor_descriptors=storage.get_tensor_descriptors_count(),
+        semantic_metadata_count=storage.get_extended_metadata_count("SemanticMetadata"),
+        lineage_metadata_count=storage.get_extended_metadata_count("LineageMetadata"),
+        computational_metadata_count=storage.get_extended_metadata_count("ComputationalMetadata"),
+        quality_metadata_count=storage.get_extended_metadata_count("QualityMetadata"),
+        relational_metadata_count=storage.get_extended_metadata_count("RelationalMetadata"),
+        usage_metadata_count=storage.get_extended_metadata_count("UsageMetadata")
+    )
+
+# --- Analytics Router ---
+router_analytics = APIRouter(
+    prefix="/analytics",
+    tags=["Analytics"]
+)
+
+@router_analytics.get("/co_occurring_tags", response_model=Dict[str, List[Dict[str, Any]]],
+                       summary="Get Co-occurring Tags",
+                       description="Finds tags that frequently co-occur with other tags on tensor descriptors.")
+async def api_get_co_occurring_tags(
+    min_co_occurrence: int = Query(2, ge=1, description="Minimum number of times tags must appear together."),
+    limit: int = Query(10, ge=1, le=100, description="Maximum number of co-occurring tags to return for each primary tag."),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    try:
+        return storage.get_co_occurring_tags(min_co_occurrence=min_co_occurrence, limit=limit)
+    except NotImplementedError:
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Co-occurring tags analytics not implemented for the current storage backend.")
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error calculating co-occurring tags: {e}")
+
+
+@router_analytics.get("/stale_tensors", response_model=List[TensorDescriptor],
+                       summary="Get Stale Tensors",
+                       description="Finds tensors that have not been accessed or modified for a given number of days.")
+async def api_get_stale_tensors(
+    threshold_days: int = Query(90, ge=1, description="Number of days to consider a tensor stale."),
+    limit: int = Query(100, ge=1, le=1000, description="Maximum number of stale tensors to return."),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    try:
+        return storage.get_stale_tensors(threshold_days=threshold_days, limit=limit)
+    except NotImplementedError:
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Stale tensor analytics not implemented for the current storage backend.")
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error fetching stale tensors: {e}")
+
+
+@router_analytics.get("/complex_tensors", response_model=List[TensorDescriptor],
+                       summary="Get Complex Tensors",
+                       description="Finds tensors considered complex based on lineage (number of parents or transformation steps).")
+async def api_get_complex_tensors(
+    min_parent_count: Optional[int] = Query(None, ge=0, description="Minimum number of parent tensors."),
+    min_transformation_steps: Optional[int] = Query(None, ge=0, description="Minimum number of transformation steps."),
+    limit: int = Query(100, ge=1, le=1000, description="Maximum number of complex tensors to return."),
+    storage: MetadataStorage = Depends(get_storage_instance)
+):
+    if min_parent_count is None and min_transformation_steps is None:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="At least one criterion (min_parent_count or min_transformation_steps) must be provided.")
+    try:
+        return storage.get_complex_tensors(
+            min_parent_count=min_parent_count,
+            min_transformation_steps=min_transformation_steps,
+            limit=limit
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+    except NotImplementedError:
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Complex tensor analytics not implemented for the current storage backend.")
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error fetching complex tensors: {e}")

tensorus/api/routers/query.py

@@ -0,0 +1,35 @@
+from fastapi import APIRouter, Depends, HTTPException
+from typing import Any, Dict, List
+
+from ..utils import tensor_to_list
+from ...nql_agent import NQLAgent
+from ...tensor_storage import TensorStorage
+from ...api import NQLQueryRequest, NQLResponse, get_nql_agent
+
+router = APIRouter()
+
+@router.post("/query", response_model=NQLResponse)
+async def query_route(
+    request: NQLQueryRequest,
+    nql_agent: NQLAgent = Depends(get_nql_agent),
+):
+    """Execute an NQL query and convert tensors to list for JSON responses."""
+    try:
+        result = nql_agent.process_query(request.query)
+        if result.get("success") and isinstance(result.get("results"), list):
+            converted: List[Dict[str, Any]] = []
+            for rec in result["results"]:
+                shape, dtype, data = tensor_to_list(rec["tensor"])
+                converted.append(
+                    {
+                        "record_id": rec["metadata"].get("record_id"),
+                        "shape": shape,
+                        "dtype": dtype,
+                        "data": data,
+                        "metadata": rec["metadata"],
+                    }
+                )
+            result["results"] = converted
+        return NQLResponse(**result)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=str(exc))

tensorus/api/security.py

@@ -0,0 +1,133 @@
+from fastapi import Security, HTTPException, status, Depends
+from fastapi.security.api_key import APIKeyHeader
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from typing import Optional, Dict, Any # Added Dict, Any for JWT payload
+
+from tensorus.config import settings
+from tensorus.audit import log_audit_event
+from jose import jwt, JWTError
+import requests
+
+
+class MutableAPIKeyHeader(APIKeyHeader):
+    """APIKeyHeader that allows updating the header name for testing."""
+
+    @property
+    def name(self) -> str:  # type: ignore[override]
+        return self.model.name
+
+    @name.setter
+    def name(self, value: str) -> None:  # type: ignore[override]
+        self.model.name = value
+
+# --- API Key Authentication ---
+api_key_header_auth = MutableAPIKeyHeader(name=settings.API_KEY_HEADER_NAME, auto_error=False)
+
+async def verify_api_key(api_key: Optional[str] = Security(api_key_header_auth)):
+    """
+    Verifies the API key provided in the request header.
+    Raises HTTPException if the API key is missing or invalid.
+    Returns the API key string if valid.
+    """
+    if not settings.VALID_API_KEYS:
+        # If no API keys are configured, treat as no valid keys configured.
+        # Endpoints depending on this will be inaccessible unless keys are provided.
+        pass
+    if not api_key:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing API Key"
+        )
+    if not settings.VALID_API_KEYS or api_key not in settings.VALID_API_KEYS:
+        # If list is empty OR key is not in the list
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API Key"
+        )
+    return api_key
+
+
+# --- JWT Token Authentication (Conceptual) ---
+oauth2_scheme = HTTPBearer(auto_error=False) # auto_error=False means it won't raise error if token is missing
+
+async def verify_jwt_token(token: Optional[HTTPAuthorizationCredentials] = Security(oauth2_scheme)) -> Dict[str, Any]:
+    """
+    Conceptual JWT token verification dependency.
+    - If JWT auth is disabled, denies access if an endpoint specifically requires it (unless in dev dummy mode).
+    - If enabled and in dev dummy mode, allows any bearer token string.
+    - If enabled and not in dev dummy mode, raises 501 Not Implemented (actual validation needed here).
+    """
+    if not settings.AUTH_JWT_ENABLED:
+        # If JWT auth is globally disabled:
+        # If an endpoint *still* tries to use this JWT verifier, it means it expects JWT.
+        # So, deny access because the system isn't configured for it.
+        # However, if AUTH_DEV_MODE_ALLOW_DUMMY_JWT is true, we might let it pass for local dev convenience
+        # even if AUTH_JWT_ENABLED is false (treating dummy mode as an override).
+        if settings.AUTH_DEV_MODE_ALLOW_DUMMY_JWT and token: # Token provided, dummy mode on
+             return {"sub": "dummy_jwt_user_jwt_disabled_but_dev_mode", "username": "dummy_dev_jwt", "token_type": "dummy_bearer_dev"}
+
+        # Standard behavior: if JWT is not enabled, this verifier should fail if called.
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE, # Or 403 Forbidden
+            detail="JWT authentication is not enabled or configured for this service."
+        )
+
+    # JWT Auth is enabled, proceed.
+    if not token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated via JWT (No token provided)"
+        )
+
+    if settings.AUTH_DEV_MODE_ALLOW_DUMMY_JWT:
+        # In dev dummy mode, allow any token value.
+        return {"sub": "dummy_jwt_user", "username": "dummy_jwt_user", "token_type": "dummy_bearer", "token_value": token.credentials}
+
+    if not settings.AUTH_JWT_JWKS_URI:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="JWKS URI not configured")
+
+    try:
+        jwks_data = requests.get(settings.AUTH_JWT_JWKS_URI).json()
+    except Exception as e:  # pragma: no cover - network issues
+        log_audit_event("JWT_VALIDATION_FAILED", details={"error": f"Failed fetching JWKS: {e}"})
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Unable to fetch JWKS")
+
+    unverified_header = jwt.get_unverified_header(token.credentials)
+    rsa_key = None
+    for key in jwks_data.get("keys", []):
+        if key.get("kid") == unverified_header.get("kid"):
+            rsa_key = key
+            break
+
+    if rsa_key is None:
+        log_audit_event("JWT_VALIDATION_FAILED", details={"error": "kid not found"})
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token header")
+
+    try:
+        claims = jwt.decode(
+            token.credentials,
+            rsa_key,
+            algorithms=[settings.AUTH_JWT_ALGORITHM],
+            issuer=settings.AUTH_JWT_ISSUER,
+            audience=settings.AUTH_JWT_AUDIENCE,
+        )
+    except JWTError as e:
+        log_audit_event("JWT_VALIDATION_FAILED", details={"error": str(e)})
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid JWT token")
+
+    log_audit_event("JWT_VALIDATION_SUCCESS", user=claims.get("sub"), details={"issuer": claims.get("iss"), "aud": claims.get("aud")})
+    return claims
+
+# Example of how to use it in an endpoint:
+# from fastapi import Depends
+# from .security import verify_api_key
+#
+# @router.post("/some_protected_route", dependencies=[Depends(verify_api_key)])
+# async def protected_route_function():
+#     # ...
+#
+# Or if you need the key value:
+# @router.post("/another_route")
+# async def another_route_function(api_key: str = Depends(verify_api_key)):
+#     # api_key variable now holds the validated key
+#     # ...

tensorus/audit.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Optional, Dict, Any
+import sys
+from tensorus.config import settings
+
+# Configure basic logger
+# In a real app, this might be more complex (e.g., JSON logging, log rotation, external service)
+LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+LOG_DEFAULT_HANDLERS = [logging.StreamHandler(sys.stdout)] # Log to stdout by default
+
+# Attempt to create a log file handler.
+# This is a simple file logger; in production, consider more robust solutions.
+try:
+    file_handler = logging.FileHandler(settings.AUDIT_LOG_PATH)
+    file_handler.setFormatter(logging.Formatter(LOG_FORMAT))
+    LOG_DEFAULT_HANDLERS.append(file_handler)
+except IOError:
+    # Handle cases where file cannot be opened (e.g. permissions)
+    print(
+        f"Warning: Could not open {settings.AUDIT_LOG_PATH} for writing. Audit logs will go to stdout only.",
+        file=sys.stderr,
+    )
+
+
+logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, handlers=LOG_DEFAULT_HANDLERS)
+audit_logger = logging.getLogger("tensorus.audit")
+
+
+def log_audit_event(
+    action: str,
+    user: Optional[str] = "anonymous",
+    tensor_id: Optional[str] = None, # Make tensor_id a common, optional parameter
+    details: Optional[Dict[str, Any]] = None
+):
+    """
+    Logs an audit event.
+
+    Args:
+        action: A string describing the action performed (e.g., "CREATE_TENSOR_DESCRIPTOR").
+        user: The user or API key performing the action. Defaults to "anonymous".
+        tensor_id: The primary tensor_id involved in the action, if applicable.
+        details: A dictionary of additional relevant information about the event.
+    """
+    log_message_parts = [f"Action: {action}"]
+
+    log_message_parts.append(f"User: {user if user else 'unknown'}")
+
+    if tensor_id:
+        log_message_parts.append(f"TensorID: {tensor_id}")
+
+    if details:
+        # Convert details dict to a string format, e.g., key1=value1, key2=value2
+        details_str = ", ".join([f"{k}={v}" for k, v in details.items()])
+        log_message_parts.append(f"Details: [{details_str}]")
+
+    audit_logger.info(" | ".join(log_message_parts))
+
+
+# Example Usage (not part of the library code, just for demonstration):
+if __name__ == "__main__":
+    # This will only run if the script is executed directly
+    log_audit_event(action="TEST_EVENT", user="test_user", tensor_id="dummy-uuid-123", details={"param1": "value1", "status": "success"})
+    audit_logger.info("This is a direct info log from audit_logger for testing handlers.")
+    audit_logger.warning("This is a warning log for testing handlers.")
+    print(f"Audit logger '{audit_logger.name}' has handlers: {audit_logger.handlers}")
+    if not audit_logger.handlers:
+         print("Warning: Audit logger has no handlers configured if run outside main app context without direct basicConfig call here.")
+    # In the application, basicConfig in this file should set up the handlers globally once.

tensorus/automl_agent.py

@@ -0,0 +1,356 @@
+# automl_agent.py
+"""
+Implements the AutoML Agent for Tensorus.
+
+This agent performs basic hyperparameter optimization using random search.
+It trains a simple dummy model on synthetic data, evaluates its performance,
+and logs the results, including storing trial results in TensorStorage.
+
+Future Enhancements:
+- Implement more advanced search strategies (Bayesian Optimization, Hyperband).
+- Allow configuration of different model architectures.
+- Integrate with real datasets from TensorStorage.
+- Implement early stopping and other training optimizations.
+- Store best model state_dict (requires serialization strategy).
+- Parallelize trials for faster search.
+- Use dedicated hyperparameter optimization libraries (Optuna, Ray Tune).
+"""
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+import random
+import logging
+import time
+from typing import Dict, Any, Callable, Tuple, Optional
+
+from .tensor_storage import TensorStorage # Import our storage module
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+# --- Dummy Model Definition ---
+class DummyMLP(nn.Module):
+    """A simple Multi-Layer Perceptron for regression/classification."""
+    def __init__(self, input_dim: int, output_dim: int, hidden_size: int = 64, activation_fn: Callable = nn.ReLU):
+        super().__init__()
+        self.layer_1 = nn.Linear(input_dim, hidden_size)
+        self.activation = activation_fn()
+        self.layer_2 = nn.Linear(hidden_size, output_dim)
+
+    def forward(self, x):
+        x = self.activation(self.layer_1(x))
+        x = self.layer_2(x)
+        return x
+
+
+# --- AutoML Agent Class ---
+class AutoMLAgent:
+    """Performs random search hyperparameter optimization."""
+
+    def __init__(self,
+                 tensor_storage: TensorStorage,
+                 search_space: Dict[str, Callable[[], Any]],
+                 input_dim: int,
+                 output_dim: int,
+                 task_type: str = 'regression', # 'regression' or 'classification'
+                 results_dataset: str = "automl_results"):
+        """
+        Initializes the AutoML Agent.
+
+        Args:
+            tensor_storage: An instance of TensorStorage.
+            search_space: Dictionary defining the hyperparameter search space.
+                          Keys are param names (e.g., 'lr', 'hidden_size').
+                          Values are functions that sample a value for that param (e.g., lambda: 10**random.uniform(-4,-2)).
+            input_dim: Input dimension for the dummy model.
+            output_dim: Output dimension for the dummy model.
+            task_type: Type of task ('regression' or 'classification'), influences loss and data generation.
+            results_dataset: Name of the dataset in TensorStorage to store trial results.
+        """
+        if not isinstance(tensor_storage, TensorStorage):
+            raise TypeError("tensor_storage must be an instance of TensorStorage")
+        if task_type not in ['regression', 'classification']:
+             raise ValueError("task_type must be 'regression' or 'classification'")
+
+        self.tensor_storage = tensor_storage
+        self.search_space = search_space
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.task_type = task_type
+        self.results_dataset = results_dataset
+
+        # Ensure results dataset exists
+        try:
+            self.tensor_storage.get_dataset(self.results_dataset)
+        except ValueError:
+            logger.info(f"Dataset '{self.results_dataset}' not found. Creating it.")
+            self.tensor_storage.create_dataset(self.results_dataset)
+
+        # Track best results found during the search
+        self.best_score: Optional[float] = None # Use negative infinity for maximization tasks if needed
+        self.best_params: Optional[Dict[str, Any]] = None
+        # Assuming lower score is better (e.g., loss)
+        self.higher_score_is_better = False if task_type == 'regression' else True # Accuracy for classification
+
+
+        # Device configuration
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        logger.info(f"AutoML Agent using device: {self.device}")
+        logger.info(f"AutoML Agent initialized for {task_type} task. Results stored in '{results_dataset}'.")
+
+
+    def _generate_synthetic_data(self, n_samples=500, batch_size=32) -> Tuple[Any, Any]:
+        """Generates synthetic data loaders for training and validation."""
+        X = torch.randn(n_samples, self.input_dim, device=self.device)
+
+        if self.task_type == 'regression':
+            # Simple linear relationship with noise
+            true_weight = torch.randn(self.input_dim, self.output_dim, device=self.device) * 2
+            true_bias = torch.randn(self.output_dim, device=self.device)
+            y = X @ true_weight + true_bias + torch.randn(n_samples, self.output_dim, device=self.device) * 0.5
+            loss_fn = nn.MSELoss()
+        else: # classification
+            # Simple linear separation + softmax for multi-class
+            if self.output_dim <= 1:
+                 raise ValueError("Output dimension must be > 1 for classification task example.")
+            true_weight = torch.randn(self.input_dim, self.output_dim, device=self.device)
+            logits = X @ true_weight
+            y = torch.softmax(logits, dim=1).argmax(dim=1) # Get class labels
+            loss_fn = nn.CrossEntropyLoss()
+
+        # Simple split
+        split_idx = int(n_samples * 0.8)
+        X_train, X_val = X[:split_idx], X[split_idx:]
+        y_train, y_val = y[:split_idx], y[split_idx:]
+
+        train_dataset = torch.utils.data.TensorDataset(X_train, y_train)
+        val_dataset = torch.utils.data.TensorDataset(X_val, y_val)
+
+        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+        val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size)
+
+        return train_loader, val_loader, loss_fn
+
+
+    def _build_dummy_model(self, params: Dict[str, Any]) -> nn.Module:
+        """Builds the dummy MLP model based on hyperparameters."""
+        hidden_size = params.get('hidden_size', 64) # Default if not in params
+        activation_name = params.get('activation', 'relu') # Default activation
+
+        act_fn_map = {'relu': nn.ReLU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid}
+        activation_fn = act_fn_map.get(activation_name.lower(), nn.ReLU) # Default to ReLU if unknown
+
+        model = DummyMLP(
+            input_dim=self.input_dim,
+            output_dim=self.output_dim,
+            hidden_size=hidden_size,
+            activation_fn=activation_fn
+        ).to(self.device)
+        return model
+
+
+    def _train_and_evaluate(self, params: Dict[str, Any], num_epochs: int = 5) -> Optional[float]:
+        """Trains and evaluates a model with given hyperparameters."""
+        logger.debug(f"Training trial with params: {params}")
+        start_time = time.time()
+
+        try:
+            # 1. Build Model
+            model = self._build_dummy_model(params)
+
+            # 2. Get Data and Loss Function
+            train_loader, val_loader, loss_fn = self._generate_synthetic_data()
+
+            # 3. Setup Optimizer
+            lr = params.get('lr', 1e-3) # Default LR
+            optimizer = optim.Adam(model.parameters(), lr=lr)
+
+            # 4. Training Loop
+            model.train()
+            for epoch in range(num_epochs):
+                epoch_loss = 0
+                for batch_X, batch_y in train_loader:
+                    optimizer.zero_grad()
+                    outputs = model(batch_X)
+                    loss = loss_fn(outputs, batch_y)
+
+                    # Check for NaN/inf loss
+                    if not torch.isfinite(loss):
+                         logger.warning(f"Trial failed: Non-finite loss detected during training epoch {epoch}. Params: {params}")
+                         return None # Indicate failure
+
+                    loss.backward()
+                    optimizer.step()
+                    epoch_loss += loss.item()
+                # logger.debug(f" Epoch {epoch+1}/{num_epochs}, Train Loss: {epoch_loss/len(train_loader):.4f}")
+
+
+            # 5. Evaluation Loop
+            model.eval()
+            total_val_loss = 0
+            total_correct = 0
+            total_samples = 0
+            with torch.no_grad():
+                for batch_X, batch_y in val_loader:
+                    outputs = model(batch_X)
+                    loss = loss_fn(outputs, batch_y)
+                    total_val_loss += loss.item()
+
+                    if self.task_type == 'classification':
+                         predicted = outputs.argmax(dim=1)
+                         total_correct += (predicted == batch_y).sum().item()
+                         total_samples += batch_y.size(0)
+
+            avg_val_loss = total_val_loss / len(val_loader)
+            duration = time.time() - start_time
+            logger.debug(f"Trial completed in {duration:.2f}s. Val Loss: {avg_val_loss:.4f}")
+
+
+            # 6. Return Score
+            if self.task_type == 'regression':
+                score = avg_val_loss # Lower is better
+            else: # classification
+                 accuracy = total_correct / total_samples if total_samples > 0 else 0
+                 score = accuracy # Higher is better
+                 logger.debug(f" Trial Val Accuracy: {accuracy:.4f}")
+
+            return score
+
+        except Exception as e:
+            logger.error(f"Trial failed with exception for params {params}: {e}", exc_info=True)
+            return None # Indicate failure
+
+
+    def hyperparameter_search(self, trials: int, num_epochs_per_trial: int = 5) -> Optional[Dict[str, Any]]:
+        """
+        Performs random search for the specified number of trials.
+
+        Args:
+            trials: The number of hyperparameter configurations to try.
+            num_epochs_per_trial: Number of epochs to train each model configuration.
+
+        Returns:
+            The dictionary of hyperparameters that achieved the best score, or None if no trial succeeded.
+        """
+        logger.info(f"--- Starting Hyperparameter Search ({trials} trials) ---")
+        self.best_score = None
+        self.best_params = None
+
+        for i in range(trials):
+            # 1. Sample hyperparameters
+            current_params = {name: sampler() for name, sampler in self.search_space.items()}
+            logger.info(f"Trial {i+1}/{trials}: Testing params: {current_params}")
+
+            # 2. Train and evaluate
+            score = self._train_and_evaluate(current_params, num_epochs=num_epochs_per_trial)
+
+            # 3. Store results in TensorStorage (even if trial failed, record params and score=None)
+            score_tensor = torch.tensor(float('nan') if score is None else score) # Store NaN for failed trials
+            trial_metadata = {
+                "trial_id": i + 1,
+                "params": current_params, # Store params dict directly in metadata
+                "score": score, # Store score also in metadata for easier querying
+                "task_type": self.task_type,
+                "search_timestamp_utc": time.time(),
+                "created_by": "AutoMLAgent" # Add agent source
+            }
+            try:
+                 record_id = self.tensor_storage.insert(
+                     self.results_dataset,
+                     score_tensor,
+                     trial_metadata
+                 )
+                 logger.debug(f"Stored trial {i+1} results (Score: {score}) with record ID: {record_id}")
+            except Exception as e:
+                 logger.error(f"Failed to store trial {i+1} results in TensorStorage: {e}")
+
+
+            # 4. Update best score if trial succeeded and is better
+            if score is not None:
+                is_better = False
+                if self.best_score is None:
+                    is_better = True
+                elif self.higher_score_is_better and score > self.best_score:
+                     is_better = True
+                elif not self.higher_score_is_better and score < self.best_score:
+                     is_better = True
+
+                if is_better:
+                     self.best_score = score
+                     self.best_params = current_params
+                     logger.info(f"*** New best score found! Trial {i+1}: Score={score:.4f}, Params={current_params} ***")
+
+
+        logger.info(f"--- Hyperparameter Search Finished ---")
+        if self.best_params:
+            logger.info(f"Best score overall: {self.best_score:.4f}")
+            logger.info(f"Best hyperparameters found: {self.best_params}")
+            # Optional: Here you could trigger saving the best model's state_dict
+            # e.g., self._save_best_model(self.best_params)
+        else:
+            logger.warning("No successful trials completed. Could not determine best parameters.")
+
+        return self.best_params
+
+
+# --- Example Usage ---
+if __name__ == "__main__":
+    print("--- Starting AutoML Agent Example ---")
+
+    # 1. Setup TensorStorage
+    storage = TensorStorage()
+
+    # 2. Define Search Space
+    # Simple example for MLP regression
+    search_space_reg = {
+        'lr': lambda: 10**random.uniform(-5, -2), # Log uniform for learning rate
+        'hidden_size': lambda: random.choice([32, 64, 128, 256]),
+        'activation': lambda: random.choice(['relu', 'tanh'])
+    }
+
+    # 3. Create the AutoML Agent
+    input_dim = 10
+    output_dim = 1 # Regression task
+    automl_agent = AutoMLAgent(
+        tensor_storage=storage,
+        search_space=search_space_reg,
+        input_dim=input_dim,
+        output_dim=output_dim,
+        task_type='regression',
+        results_dataset="automl_regression_results"
+    )
+
+    # 4. Run the hyperparameter search
+    num_trials = 20 # Number of random configurations to test
+    num_epochs = 10 # Epochs per trial (keep low for speed)
+    best_hyperparams = automl_agent.hyperparameter_search(trials=num_trials, num_epochs_per_trial=num_epochs)
+
+
+    # 5. Optional: Check TensorStorage for results
+    print("\n--- Checking TensorStorage contents (Sample) ---")
+    try:
+        results_count = len(storage.get_dataset(automl_agent.results_dataset))
+        print(f"Found {results_count} trial records in '{automl_agent.results_dataset}'.")
+
+        if results_count > 0:
+            print("\nExample trial record (metadata):")
+            # Sample one record to show structure
+            sample_trial = storage.sample_dataset(automl_agent.results_dataset, 1)
+            if sample_trial:
+                 print(f"  Metadata: {sample_trial[0]['metadata']}")
+                 print(f"  Score Tensor: {sample_trial[0]['tensor']}") # Should contain the score or NaN
+
+            # You could also query for the best score using NQLAgent if needed
+            # (e.g., find record where score = best_score) - requires parsing results first.
+
+    except ValueError as e:
+        print(f"Could not retrieve dataset '{automl_agent.results_dataset}': {e}")
+    except Exception as e:
+         print(f"An error occurred checking storage: {e}")
+
+
+    print("\n--- AutoML Agent Example Finished ---")

tensorus/config.py

@@ -0,0 +1,95 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing import Optional
+from pydantic import field_validator
+
+class Settings(BaseSettings):
+    # Configuration for environment variable prefix, .env file, etc.
+    # For Pydantic-Settings v2, environment variables are loaded by default.
+    # To specify a prefix for env vars (e.g. TENSORUS_STORAGE_BACKEND):
+    # model_config = SettingsConfigDict(env_prefix='TENSORUS_') # Pydantic v2
+    # For Pydantic v1, it was `Config.env_prefix`.
+
+    STORAGE_BACKEND: str = "in_memory"
+
+    POSTGRES_HOST: Optional[str] = None
+    POSTGRES_PORT: Optional[int] = 5432 # Default PostgreSQL port
+    POSTGRES_USER: Optional[str] = None
+    POSTGRES_PASSWORD: Optional[str] = None
+    POSTGRES_DB: Optional[str] = None
+    POSTGRES_DSN: Optional[str] = None # Alternative to individual params
+
+    # Example of how to load from a .env file if needed (not strictly required by subtask)
+    # model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore')
+
+
+# Global instance of the settings
+# The environment variables will be loaded when this instance is created.
+# e.g. TENSORUS_STORAGE_BACKEND=postgres will override the default.
+# Note: For pydantic-settings, env var names are case-insensitive by default for matching.
+# If env_prefix is set, it would be TENSORUS_STORAGE_BACKEND. Without it, it's just STORAGE_BACKEND.
+# Let's assume no prefix for now, so environment variables should be STORAGE_BACKEND, POSTGRES_HOST etc.
+# OR, more commonly, one would use the env_prefix.
+# For this exercise, I will assume the user will set environment variables like:
+# export STORAGE_BACKEND="postgres"
+# export POSTGRES_USER="myuser"
+# ...etc.
+# Or, if using an .env file:
+# STORAGE_BACKEND="postgres"
+# POSTGRES_USER="myuser"
+# ...
+#
+# For Pydantic V1 BaseSettings, it would be:
+# class Settings(BaseSettings):
+#     STORAGE_BACKEND: str = "in_memory"
+#     # ... other fields
+#     class Config:
+#         env_prefix = "TENSORUS_" # e.g. TENSORUS_STORAGE_BACKEND
+#         # case_sensitive = False # for Pydantic V1
+#
+# Given the project uses pydantic 1.10, I will use the V1 style for env_prefix.
+
+class SettingsV1(BaseSettings):
+    STORAGE_BACKEND: str = "in_memory"
+    POSTGRES_HOST: Optional[str] = None
+    POSTGRES_PORT: Optional[int] = 5432
+    POSTGRES_USER: Optional[str] = None
+    POSTGRES_PASSWORD: Optional[str] = None
+    POSTGRES_DB: Optional[str] = None
+    POSTGRES_DSN: Optional[str] = None
+
+    # API Security
+    # Accepts a comma-separated string or JSON list via env var
+    VALID_API_KEYS: list[str] | str = []
+    API_KEY_HEADER_NAME: str = "X-API-KEY"
+    AUDIT_LOG_PATH: str = "tensorus_audit.log"
+
+
+
+
+    model_config = SettingsConfigDict(env_prefix="TENSORUS_", case_sensitive=False)
+
+    @field_validator("VALID_API_KEYS", mode="before")
+    def split_valid_api_keys(cls, v):
+        """Allow comma-separated or JSON list values from the environment."""
+        if isinstance(v, str):
+            try:
+                import json
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return [str(item) for item in parsed]
+            except Exception:
+                pass
+            return [key.strip() for key in v.split(',') if key.strip()]
+        return v
+
+    # JWT Authentication (Conceptual Settings)
+    AUTH_JWT_ENABLED: bool = False
+    AUTH_JWT_ISSUER: Optional[str] = None
+    AUTH_JWT_AUDIENCE: Optional[str] = None
+    AUTH_JWT_ALGORITHM: str = "RS256"
+    AUTH_JWT_JWKS_URI: Optional[str] = None
+    AUTH_DEV_MODE_ALLOW_DUMMY_JWT: bool = False
+
+
+# Use SettingsV1 for Pydantic v1.x compatibility
+settings = SettingsV1()

tensorus/dummy_env.py

@@ -0,0 +1,91 @@
+# dummy_env.py
+"""
+A simple dummy environment for testing the RL agent.
+State: Position (1D)
+Action: Move left (-1), Stay (0), Move right (+1) (Discrete actions)
+Goal: Reach position 0
+Reward: -abs(position), +10 if at goal
+"""
+from typing import Tuple, Dict
+import torch
+import random
+import numpy as np # Use numpy for state representation convenience
+
+class DummyEnv:
+    def __init__(self, max_steps=50):
+        self.state_dim = 1  # Position is the only state variable
+        self.action_dim = 3 # Actions: 0 (left), 1 (stay), 2 (right)
+        self.max_steps = max_steps
+        self.current_pos = 0.0
+        self.steps_taken = 0
+        self.goal_pos = 0.0
+        self.max_pos = 5.0 # Boundaries
+
+    def reset(self) -> torch.Tensor:
+        """Resets the environment to a random starting position."""
+        self.current_pos = random.uniform(-self.max_pos, self.max_pos)
+        self.steps_taken = 0
+        # Return state as a PyTorch tensor
+        return torch.tensor([self.current_pos], dtype=torch.float32)
+
+    def step(self, action: int) -> Tuple[torch.Tensor, float, bool, Dict]:
+        """Takes an action, updates the state, and returns results."""
+        if not isinstance(action, int) or action not in [0, 1, 2]:
+             raise ValueError(f"Invalid action: {action}. Must be 0, 1, or 2.")
+
+        # Update position based on action
+        if action == 0: # Move left
+            self.current_pos -= 0.5
+        elif action == 2: # Move right
+             self.current_pos += 0.5
+        # Action 1 (stay) does nothing to position
+
+        # Clip position to boundaries
+        self.current_pos = np.clip(self.current_pos, -self.max_pos, self.max_pos)
+
+        self.steps_taken += 1
+
+        # Calculate reward
+        # Higher reward closer to the goal, large penalty for being far
+        reward = -abs(self.current_pos - self.goal_pos) * 0.1 # Small penalty for distance
+        done = False
+
+        # Check if goal is reached (within a small tolerance)
+        if abs(self.current_pos - self.goal_pos) < 0.1:
+             reward += 10.0 # Bonus for reaching goal
+             done = True
+
+        # Check if max steps exceeded
+        if self.steps_taken >= self.max_steps:
+            done = True
+            # Optional: small penalty for running out of time
+            # reward -= 1.0
+
+        # Return next state, reward, done flag, and info dict
+        next_state = torch.tensor([self.current_pos], dtype=torch.float32)
+        info = {} # Empty info dict for now
+
+        return next_state, float(reward), done, info
+
+# Example Usage
+if __name__ == "__main__":
+    env = DummyEnv()
+    state = env.reset()
+    print(f"Initial state: {state.item()}")
+    done = False
+    total_reward = 0
+    steps = 0
+
+    while not done:
+        action = random.choice([0, 1, 2]) # Take random action
+        next_state, reward, done, _ = env.step(action)
+        print(f"Step {steps+1}: Action={action}, Next State={next_state.item():.2f}, Reward={reward:.2f}, Done={done}")
+        state = next_state
+        total_reward += reward
+        steps += 1
+        if steps > env.max_steps + 5: # Safety break
+             print("Exceeded max steps significantly, breaking.")
+             break
+
+
+    print(f"\nEpisode finished after {steps} steps. Total reward: {total_reward:.2f}")

tensorus/financial_data_generator.py

@@ -0,0 +1,122 @@
+import pandas as pd
+import numpy as np
+import datetime
+
+def generate_financial_data(
+    days: int = 500,
+    initial_price: float = 100.0,
+    trend_slope: float = 0.05,
+    seasonality_amplitude: float = 10.0,
+    seasonality_period: int = 90,
+    noise_level: float = 2.0,
+    base_volume: int = 100000,
+    volume_volatility: float = 0.3,
+    start_date_str: str = "2022-01-01"
+) -> pd.DataFrame:
+    """
+    Generates synthetic time series data resembling daily stock prices.
+
+    Args:
+        days (int): Number of trading days to generate data for.
+        initial_price (float): Starting price for the stock.
+        trend_slope (float): Slope for the linear trend component (price change per day).
+        seasonality_amplitude (float): Amplitude of the seasonal (sine wave) component.
+        seasonality_period (int): Period in days for the seasonality.
+        noise_level (float): Standard deviation of the random noise added to the price.
+        base_volume (int): Base daily trading volume.
+        volume_volatility (float): Percentage volatility for volume (e.g., 0.3 for +/-30%).
+        start_date_str (str): Start date for the data in 'YYYY-MM-DD' format.
+
+    Returns:
+        pd.DataFrame: A DataFrame with columns ['Date', 'Close', 'Volume'].
+                      'Date' is datetime, 'Close' is float, 'Volume' is int.
+    """
+
+    # 1. Create date range
+    try:
+        start_date = datetime.datetime.strptime(start_date_str, "%Y-%m-%d")
+    except ValueError:
+        raise ValueError("start_date_str must be in 'YYYY-MM-DD' format")
+    
+    # Using pandas date_range for business days if needed, or simple days
+    # For simplicity, using consecutive days. For actual trading days, use bdate_range.
+    dates = pd.date_range(start_date, periods=days, freq='D')
+    
+    # Time component for trend and seasonality
+    time_component = np.arange(days)
+
+    # 2. Generate linear trend component
+    trend = trend_slope * time_component
+
+    # 3. Generate seasonal component (sine wave)
+    seasonal = seasonality_amplitude * np.sin(2 * np.pi * time_component / seasonality_period)
+
+    # 4. Generate random noise
+    noise = np.random.normal(loc=0, scale=noise_level, size=days)
+
+    # 5. Calculate Close price
+    close_prices = initial_price + trend + seasonal + noise
+    # Ensure prices don't go below a certain minimum (e.g., 1.0)
+    close_prices = np.maximum(close_prices, 1.0) 
+
+    # 6. Generate Volume data
+    volume_random_factor = np.random.uniform(-volume_volatility, volume_volatility, size=days)
+    volumes = base_volume * (1 + volume_random_factor)
+    # Ensure volume is positive and integer
+    volumes = np.maximum(volumes, 0).astype(int)
+
+    # 7. Create DataFrame
+    df = pd.DataFrame({
+        'Date': dates,
+        'Close': close_prices,
+        'Volume': volumes
+    })
+
+    # For OHLC (Optional - can be expanded later)
+    # Open: Could be previous day's close, or close +/- small random factor
+    # High: Close + small positive random factor
+    # Low: Close - small positive random factor (ensure Low <= Open/Close and High >= Open/Close)
+    # For now, these are omitted as per requirements.
+
+    return df
+
+if __name__ == "__main__":
+    # Generate data with default parameters
+    print("Generating synthetic financial data with default parameters...")
+    financial_df = generate_financial_data()
+
+    # Print head and tail
+    print("\nDataFrame Head:")
+    print(financial_df.head())
+    print("\nDataFrame Tail:")
+    print(financial_df.tail())
+    print(f"\nGenerated {len(financial_df)} days of data.")
+
+    # Save to CSV
+    output_filename = "synthetic_financial_data.csv"
+    try:
+        financial_df.to_csv(output_filename, index=False)
+        print(f"\nSuccessfully saved data to {output_filename}")
+    except Exception as e:
+        print(f"\nError saving data to CSV: {e}")
+
+    # Example with custom parameters
+    print("\nGenerating synthetic financial data with custom parameters...")
+    custom_financial_df = generate_financial_data(
+        days=100,
+        initial_price=50.0,
+        trend_slope=-0.1, # Downward trend
+        seasonality_amplitude=5.0,
+        seasonality_period=30,
+        noise_level=1.0,
+        base_volume=50000,
+        start_date_str="2023-01-01"
+    )
+    print("\nCustom DataFrame Head:")
+    print(custom_financial_df.head())
+    custom_output_filename = "custom_synthetic_financial_data.csv"
+    try:
+        custom_financial_df.to_csv(custom_output_filename, index=False)
+        print(f"\nSuccessfully saved custom data to {custom_output_filename}")
+    except Exception as e:
+        print(f"\nError saving custom data to CSV: {e}")

tensorus/ingestion_agent.py

@@ -0,0 +1,409 @@
+# ingestion_agent.py
+"""
+Implements the Autonomous Data Ingestion Agent for Tensorus.
+
+This agent monitors a source directory for new data files (e.g., CSV, images),
+preprocesses them into tensors using configurable functions, performs basic
+validation, and inserts them into a specified dataset in TensorStorage.
+
+Future Enhancements:
+- Monitor cloud storage (S3, GCS) and APIs.
+- More robust error handling for malformed files.
+- More sophisticated duplicate detection (e.g., file hashing).
+- Support for streaming data sources.
+- Asynchronous processing for higher throughput.
+- More complex and configurable preprocessing pipelines.
+- Schema validation against predefined dataset schemas.
+- Resource management controls.
+"""
+
+import os
+import time
+import glob
+import logging
+import threading
+import csv
+from PIL import Image
+import torch
+import torchvision.transforms as T # Use torchvision for image transforms
+import collections # Added import
+
+from typing import Dict, Callable, Optional, Tuple, List, Any
+from .tensor_storage import TensorStorage # Import our storage module
+
+# Configure basic logging (can be customized further)
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+# --- Custom Log Handler ---
+class AgentMemoryLogHandler(logging.Handler):
+    """
+    A custom logging handler that stores log records in a deque.
+    """
+    def __init__(self, deque: collections.deque):
+        super().__init__()
+        self.deque = deque
+
+    def emit(self, record: logging.LogRecord) -> None:
+        self.deque.append(self.format(record))
+
+
+# --- Default Preprocessing Functions ---
+
+def preprocess_csv(file_path: str) -> Tuple[Optional[torch.Tensor], Dict[str, Any]]:
+    """
+    Basic CSV preprocessor. Assumes numeric data.
+    Reads a CSV, converts rows to tensors (one tensor per row or one tensor for the whole file).
+    Returns a single tensor representing the whole file for simplicity here.
+    """
+    metadata = {"source_file": file_path, "type": "csv"}
+    data = []
+    try:
+        with open(file_path, 'r', newline='') as csvfile:
+            reader = csv.reader(csvfile)
+            header = next(reader, None) # Skip header row
+            metadata["header"] = header
+            for row in reader:
+                # Attempt to convert row elements to floats
+                try:
+                    numeric_row = [float(item) for item in row]
+                    data.append(numeric_row)
+                except ValueError:
+                    logger.warning(f"Skipping non-numeric row in {file_path}: {row}")
+                    continue # Skip rows that can't be fully converted to float
+
+        if not data:
+            logger.warning(f"No numeric data found or processed in CSV file: {file_path}")
+            return None, metadata
+
+        tensor = torch.tensor(data, dtype=torch.float32)
+        logger.debug(f"Successfully processed {file_path} into tensor shape {tensor.shape}")
+        return tensor, metadata
+    except Exception as e:
+        logger.error(f"Failed to process CSV file {file_path}: {e}")
+        return None, metadata
+
+
+def preprocess_image(file_path: str) -> Tuple[Optional[torch.Tensor], Dict[str, Any]]:
+    """
+    Basic Image preprocessor using Pillow and Torchvision transforms.
+    Opens an image, applies standard transformations (resize, normalize),
+    and returns it as a tensor.
+    """
+    metadata = {"source_file": file_path, "type": "image"}
+    try:
+        img = Image.open(file_path).convert('RGB') # Ensure 3 channels (RGB)
+
+        # Example transform: Resize, convert to tensor, normalize
+        # These should ideally be configurable
+        transform = T.Compose([
+            T.Resize((128, 128)), # Example fixed size
+            T.ToTensor(), # Converts PIL image (H, W, C) [0,255] to Tensor (C, H, W) [0,1]
+            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # ImageNet stats
+        ])
+
+        tensor = transform(img)
+        metadata["original_size"] = img.size # (width, height)
+        logger.debug(f"Successfully processed {file_path} into tensor shape {tensor.shape}")
+        return tensor, metadata
+    except FileNotFoundError:
+         logger.error(f"Image file not found: {file_path}")
+         return None, metadata
+    except Exception as e:
+        logger.error(f"Failed to process image file {file_path}: {e}")
+        return None, metadata
+
+# --- Data Ingestion Agent Class ---
+
+class DataIngestionAgent:
+    """
+    Monitors a directory for new files and ingests them into TensorStorage.
+    """
+
+    def __init__(self,
+                 tensor_storage: TensorStorage,
+                 dataset_name: str,
+                 source_directory: str,
+                 polling_interval_sec: int = 10,
+                 preprocessing_rules: Optional[Dict[str, Callable[[str], Tuple[Optional[torch.Tensor], Dict[str, Any]]]]] = None):
+        """
+        Initializes the DataIngestionAgent.
+
+        Args:
+            tensor_storage: An instance of the TensorStorage class.
+            dataset_name: The name of the dataset in TensorStorage to ingest into.
+            source_directory: The path to the local directory to monitor.
+            polling_interval_sec: How often (in seconds) to check the directory.
+            preprocessing_rules: A dictionary mapping lowercase file extensions
+                                 (e.g., '.csv', '.png') to preprocessing functions.
+                                 Each function takes a file path and returns a
+                                 Tuple containing the processed Tensor (or None on failure)
+                                 and a metadata dictionary. If None, default rules are used.
+        """
+        if not isinstance(tensor_storage, TensorStorage):
+            raise TypeError("tensor_storage must be an instance of TensorStorage")
+        if not os.path.isdir(source_directory):
+            raise ValueError(f"Source directory '{source_directory}' does not exist or is not a directory.")
+
+        self.tensor_storage = tensor_storage
+        self.dataset_name = dataset_name
+        self.source_directory = source_directory
+        self.polling_interval = polling_interval_sec
+
+        # Ensure dataset exists
+        try:
+            self.tensor_storage.get_dataset(self.dataset_name)
+            logger.info(f"Agent targeting existing dataset '{self.dataset_name}'.")
+        except ValueError:
+            logger.info(f"Dataset '{self.dataset_name}' not found. Creating it.")
+            self.tensor_storage.create_dataset(self.dataset_name)
+
+        # Default preprocessing rules if none provided
+        if preprocessing_rules is None:
+            self.preprocessing_rules = {
+                '.csv': preprocess_csv,
+                '.png': preprocess_image,
+                '.jpg': preprocess_image,
+                '.jpeg': preprocess_image,
+                '.tif': preprocess_image,
+                '.tiff': preprocess_image,
+            }
+            logger.info("Using default preprocessing rules for CSV and common image formats.")
+        else:
+            self.preprocessing_rules = preprocessing_rules
+            logger.info(f"Using custom preprocessing rules for extensions: {list(preprocessing_rules.keys())}")
+
+        self.processed_files = set() # Keep track of files already processed in this session
+        self._stop_event = threading.Event()
+        self._monitor_thread = None
+        self.status = "stopped" # Status reporting
+        self.logs = collections.deque(maxlen=100) # Log capturing
+
+        # Setup custom log handler
+        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+        memory_handler = AgentMemoryLogHandler(self.logs)
+        memory_handler.setFormatter(formatter)
+        logger.addHandler(memory_handler) # Add handler to the specific logger instance
+
+        logger.info(f"DataIngestionAgent initialized for dataset '{self.dataset_name}' monitoring '{self.source_directory}'.")
+
+    def get_status(self) -> str:
+        """Returns the current status of the agent."""
+        return self.status
+
+    def get_logs(self, max_lines: Optional[int] = None) -> List[str]:
+        """Returns recent log messages from the agent."""
+        if max_lines is None:
+            return list(self.logs)
+        else:
+            return list(self.logs)[-max_lines:]
+
+    def _validate_data(self, tensor: Optional[torch.Tensor], metadata: Dict[str, Any]) -> bool:
+        """
+        Performs basic validation on the preprocessed tensor.
+        Returns True if valid, False otherwise.
+        """
+        if tensor is None:
+            logger.warning(f"Validation failed: Tensor is None for {metadata.get('source_file', 'N/A')}")
+            return False
+        if not isinstance(tensor, torch.Tensor):
+             logger.warning(f"Validation failed: Output is not a tensor for {metadata.get('source_file', 'N/A')}")
+             return False
+        # Add more specific checks if needed (e.g., tensor.numel() > 0)
+        return True
+
+    def _process_file(self, file_path: str) -> None:
+        """Processes a single detected file."""
+        logger.info(f"Detected new file: {file_path}")
+        _, file_extension = os.path.splitext(file_path)
+        file_extension = file_extension.lower()
+
+        preprocessor = self.preprocessing_rules.get(file_extension)
+
+        if preprocessor:
+            logger.debug(f"Applying preprocessor for '{file_extension}' to {file_path}")
+            try:
+                tensor, metadata = preprocessor(file_path)
+
+                if self._validate_data(tensor, metadata):
+                    # Ensure tensor is not None before insertion
+                    if tensor is not None:
+                        metadata["created_by"] = "IngestionAgent" # Add agent source
+                        record_id = self.tensor_storage.insert(self.dataset_name, tensor, metadata)
+                        logger.info(f"Successfully ingested '{file_path}' into dataset '{self.dataset_name}' with record ID: {record_id} (created_by: IngestionAgent)")
+                        self.processed_files.add(file_path) # Mark as processed only on success
+                    else:
+                         # Should have been caught by validation, but as safeguard:
+                         logger.error(f"Validation passed but tensor is None for {file_path}. Skipping insertion.")
+
+                else:
+                    logger.warning(f"Data validation failed for {file_path}. Skipping insertion.")
+
+            except Exception as e:
+                logger.error(f"Unhandled error during preprocessing or insertion for {file_path}: {e}", exc_info=True)
+        else:
+            logger.debug(f"No preprocessor configured for file extension '{file_extension}'. Skipping file: {file_path}")
+
+
+    def _scan_source_directory(self) -> None:
+        """Scans the source directory for new files matching the rules."""
+        logger.debug(f"Scanning directory: {self.source_directory}")
+        supported_extensions = self.preprocessing_rules.keys()
+
+        try:
+             # Use glob to find all files, then filter
+             # This might be inefficient for huge directories, consider os.scandir
+             all_files = glob.glob(os.path.join(self.source_directory, '*'), recursive=False) # Non-recursive
+
+             for file_path in all_files:
+                 if not os.path.isfile(file_path):
+                      continue # Skip directories
+
+                 _, file_extension = os.path.splitext(file_path)
+                 file_extension = file_extension.lower()
+
+                 if file_extension in supported_extensions and file_path not in self.processed_files:
+                     self._process_file(file_path)
+
+        except Exception as e:
+             logger.error(f"Error scanning source directory '{self.source_directory}': {e}", exc_info=True)
+
+
+    def _monitor_loop(self) -> None:
+        """The main loop executed by the background thread."""
+        logger.info(f"Starting monitoring loop for '{self.source_directory}'. Polling interval: {self.polling_interval} seconds.")
+        while not self._stop_event.is_set():
+            self._scan_source_directory()
+            # Wait for the specified interval or until stop event is set
+            self._stop_event.wait(self.polling_interval)
+        logger.info("Monitoring loop stopped.")
+
+
+    def start(self) -> None:
+        """Starts the monitoring process in a background thread."""
+        if self._monitor_thread is not None and self._monitor_thread.is_alive():
+            logger.warning("Agent monitoring is already running.")
+            return
+
+        self._stop_event.clear()
+        self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
+        self._monitor_thread.start()
+        if self._monitor_thread.is_alive(): # Check if thread actually started
+            self.status = "running"
+            logger.info("Data Ingestion Agent started monitoring.")
+        else:
+            self.status = "error" # Or some other appropriate error state
+            logger.error("Data Ingestion Agent failed to start monitoring thread.")
+
+
+    def stop(self) -> None:
+        """Signals the monitoring thread to stop."""
+        if self._monitor_thread is None or not self._monitor_thread.is_alive():
+            logger.info("Agent monitoring is not running.")
+            return
+
+        logger.info("Stopping Data Ingestion Agent monitoring...")
+        self._stop_event.set()
+        self._monitor_thread.join(timeout=self.polling_interval + 5) # Wait for thread to finish
+
+        if self._monitor_thread.is_alive():
+             logger.warning("Monitoring thread did not stop gracefully after timeout.")
+             # self.status remains "running" or could be set to "stopping_error"
+        else:
+             logger.info("Data Ingestion Agent monitoring stopped successfully.")
+             self.status = "stopped"
+        self._monitor_thread = None
+
+
+# --- Example Usage ---
+if __name__ == "__main__":
+    run_example = os.getenv("RUN_INGESTION_AGENT_EXAMPLE", "False").lower() == "true"
+
+    if run_example:
+        logger.info("--- Starting Ingestion Agent Example (RUN_INGESTION_AGENT_EXAMPLE=True) ---")
+
+        # 1. Setup TensorStorage
+    storage = TensorStorage()
+
+    # 2. Setup a temporary directory for the agent to monitor
+    source_dir = "temp_ingestion_source"
+    if not os.path.exists(source_dir):
+        os.makedirs(source_dir)
+        logger.info(f"Created temporary source directory: {source_dir}")
+
+    # 3. Create the Ingestion Agent
+    # We'll use a short polling interval for demonstration
+    agent = DataIngestionAgent(
+        tensor_storage=storage,
+        dataset_name="raw_data",
+        source_directory=source_dir,
+        polling_interval_sec=5
+    )
+
+    # 4. Start the agent (runs in the background)
+    agent.start()
+
+    # 5. Simulate adding files to the source directory
+    print("\nSimulating file creation...")
+    time.sleep(2) # Give agent time to start initial scan
+
+    # Create a dummy CSV file
+    csv_path = os.path.join(source_dir, "data_1.csv")
+    with open(csv_path, 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerow(["Timestamp", "Value1", "Value2"])
+        writer.writerow(["1678886400", "10.5", "20.1"])
+        writer.writerow(["1678886460", "11.2", "20.5"])
+        writer.writerow(["1678886520", "invalid", "20.9"]) # Test non-numeric row
+        writer.writerow(["1678886580", "10.9", "21.0"])
+    print(f"Created CSV: {csv_path}")
+
+    # Create a dummy image file (requires Pillow)
+    try:
+        img_path = os.path.join(source_dir, "image_1.png")
+        dummy_img = Image.new('RGB', (60, 30), color = 'red')
+        dummy_img.save(img_path)
+        print(f"Created Image: {img_path}")
+    except ImportError:
+        print("Pillow not installed, skipping image creation. Install with: pip install Pillow")
+    except Exception as e:
+        print(f"Could not create dummy image: {e}")
+
+
+    # Create an unsupported file type
+    txt_path = os.path.join(source_dir, "notes.txt")
+    with open(txt_path, 'w') as f:
+        f.write("This is a test file.")
+    print(f"Created TXT: {txt_path} (should be skipped)")
+
+
+    # 6. Let the agent run for a couple of polling cycles
+    print(f"\nWaiting for agent to process files (polling interval {agent.polling_interval}s)...")
+    time.sleep(agent.polling_interval * 2 + 1) # Wait for 2 cycles + buffer
+
+
+    # 7. Check the contents of TensorStorage
+    print("\n--- Checking TensorStorage contents ---")
+    try:
+        ingested_data = storage.get_dataset_with_metadata("raw_data")
+        print(f"Found {len(ingested_data)} items in dataset 'raw_data':")
+        for item in ingested_data:
+            print(f"  Record ID: {item['metadata'].get('record_id')}, Source: {item['metadata'].get('source_file')}, Shape: {item['tensor'].shape}, Dtype: {item['tensor'].dtype}")
+            # print(f" Tensor: {item['tensor']}") # Can be verbose
+    except ValueError as e:
+        print(f"Could not retrieve dataset 'raw_data': {e}")
+
+
+    # 8. Stop the agent
+    print("\n--- Stopping Agent ---")
+    agent.stop()
+
+    # 9. Clean up the temporary directory (optional)
+    # print(f"\nCleaning up temporary directory: {source_dir}")
+    # import shutil
+    # shutil.rmtree(source_dir)
+
+    logger.info("--- Ingestion Agent Example Finished ---")
+else:
+    logger.info("--- Ingestion Agent Example SKIPPED (RUN_INGESTION_AGENT_EXAMPLE not set to 'true') ---")

tensorus/mcp_client.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, AsyncIterator, List, Optional, Sequence, Type, TypeVar, Union
+
+from pydantic import BaseModel, ValidationError
+from fastmcp.client import Client as FastMCPClient
+from fastmcp.exceptions import FastMCPError
+from fastmcp.client.transports import StreamableHttpTransport # Changed import path and class name
+
+# Import the TextContent type that the server is actually using
+from mcp.types import TextContent
+
+# Type variable for Pydantic models
+T = TypeVar('T', bound=BaseModel)
+
+logger = logging.getLogger("tensorus.mcp.client")
+logger.setLevel(logging.INFO)
+
+class MCPResponseError(Exception):
+    """Generic exception for MCP response errors"""
+    pass
+
+class TensorusMCPClient:
+    """High-level client for the Tensorus MCP server with typed, prompt, sync, and streaming support."""
+
+    def __init__(self, transport: Any) -> None:
+        self._client = FastMCPClient(transport)
+
+    @staticmethod
+    def from_http(url: str) -> TensorusMCPClient:
+        """Factory using Streamable HTTP transport"""
+        transport = StreamableHttpTransport(url=url) # Updated class name usage
+        return TensorusMCPClient(transport)
+
+    async def __aenter__(self) -> TensorusMCPClient:
+        await self._client.__aenter__()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Any,
+        exc_val: Any,
+        exc_tb: Any
+    ) -> None:
+        await self._client.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def _call_json(
+        self,
+        name: str,
+        args: Optional[dict] = None,
+        response_model: Type[T] | None = None
+    ) -> Union[dict, T, None]:
+        """
+        Internal helper to call a tool and parse JSON or Pydantic model.
+        """
+        try:
+            result = await self._client.call_tool(name, args or {})
+        except FastMCPError as e:
+            logger.error(f"Tool call failed: {name} args={args} error={e}")
+            raise MCPResponseError(str(e))
+
+        if not result:
+            return None
+        content = result[0]
+        if not isinstance(content, TextContent):
+            raise TypeError(f"Unexpected content type: {type(content)}")
+
+        data = json.loads(content.text)
+        if response_model:
+            try:
+                # Handle cases where the actual data for the model is nested
+                if isinstance(data, dict) and 'data' in data and response_model == DatasetListResponse :
+                    # Specific handling for DatasetListResponse if data is nested under 'data'
+                    # and the model itself expects a flat structure like {"datasets": []}
+                    # This might indicate an API inconsistency or a client model mismatch.
+                    # For now, let's assume the API returns {..., "data": actual_list_for_datasets_field}
+                    # and DatasetListResponse expects {"datasets": actual_list_for_datasets_field}
+                    # So, we need to re-wrap it:
+                    return response_model.parse_obj({"datasets": data['data']})
+                elif response_model == IngestTensorResponse and isinstance(data, dict) and data.get('success') is True and isinstance(data.get('data'), dict) and 'record_id' in data['data']:
+                    # Specific handling for IngestTensorResponse if data is nested and needs id mapping
+                    return IngestTensorResponse(id=data['data']['record_id'], status="ingested") # Assuming "ingested" is the status on success
+                elif response_model == TensorDetailsResponse and isinstance(data, dict) and 'record_id' in data:
+                    # Map API's 'record_id' to model's 'id' field
+                    data_for_model = data.copy()
+                    data_for_model['id'] = data_for_model.pop('record_id')
+                    return TensorDetailsResponse.parse_obj(data_for_model)
+
+                # For other models (like CreateDatasetResponse, DeleteDatasetResponse, etc.),
+                # parse the data directly. This assumes 'data' dictionary as a whole matches the response_model.
+                return response_model.parse_obj(data)
+            except ValidationError as ve:
+                logger.error(f"Response validation failed for {name}: {ve}. Data: {data}")
+                raise
+        return data
+
+    async def call_tool_stream(
+        self,
+        name: str,
+        args: Optional[dict] = None
+    ) -> AsyncIterator[str]:
+        """
+        Stream partial TextContent.text chunks from a long-running tool.
+        """
+        try:
+            async for msg in self._client.call_tool_stream(name, args or {}):
+                yield msg.text
+        except FastMCPError as e:
+            logger.error(f"Streaming call failed: {name} error={e}")
+            raise MCPResponseError(str(e))
+
+    # --- Prompt support ---
+    async def call_prompt(
+        self,
+        prompt_name: str,
+        args: Optional[dict] = None
+    ) -> List[TextContent]:
+        """
+        Invoke a prompt template defined on the MCP server.
+        """
+        try:
+            return await self._client.call_prompt(prompt_name, args or {})
+        except FastMCPError as e:
+            logger.error(f"Prompt call failed: {prompt_name} args={args} error={e}")
+            raise MCPResponseError(str(e))
+
+    # --- Dataset management ---
+    async def list_datasets(self) -> list[str]:
+        return await self._call_json(
+            "tensorus_list_datasets",
+            response_model=DatasetListResponse
+        )
+
+    def list_datasets_sync(self) -> list[str]:
+        """Sync wrapper for list_datasets"""
+        return asyncio.run(self.list_datasets())
+
+    async def create_dataset(self, dataset_name: str) -> CreateDatasetResponse:
+        return await self._call_json(
+            "tensorus_create_dataset",
+            {"dataset_name": dataset_name},
+            response_model=CreateDatasetResponse
+        )
+
+    def create_dataset_sync(self, dataset_name: str) -> CreateDatasetResponse:
+        return asyncio.run(self.create_dataset(dataset_name))
+
+    async def delete_dataset(self, dataset_name: str) -> DeleteDatasetResponse:
+        return await self._call_json(
+            "tensorus_delete_dataset",
+            {"dataset_name": dataset_name},
+            response_model=DeleteDatasetResponse
+        )
+
+    def delete_dataset_sync(self, dataset_name: str) -> DeleteDatasetResponse:
+        return asyncio.run(self.delete_dataset(dataset_name))
+
+    # --- Tensor management ---
+    async def ingest_tensor(
+        self,
+        dataset_name: str,
+        tensor_shape: Sequence[int],
+        tensor_dtype: str,
+        tensor_data: Any,
+        metadata: Optional[dict] = None,
+    ) -> IngestTensorResponse:
+        payload = {
+            "dataset_name": dataset_name,
+            "tensor_shape": list(tensor_shape),
+            "tensor_dtype": tensor_dtype,
+            "tensor_data": tensor_data,
+            "metadata": metadata,
+        }
+        return await self._call_json(
+            "tensorus_ingest_tensor",
+            payload,
+            response_model=IngestTensorResponse
+        )
+
+    async def get_tensor_details(self, dataset_name: str, record_id: str) -> TensorDetailsResponse:
+        return await self._call_json(
+            "tensorus_get_tensor_details",
+            {"dataset_name": dataset_name, "record_id": record_id},
+            response_model=TensorDetailsResponse
+        )
+
+    async def delete_tensor(self, dataset_name: str, record_id: str) -> DeleteTensorResponse:
+        return await self._call_json(
+            "tensorus_delete_tensor",
+            {"dataset_name": dataset_name, "record_id": record_id},
+            response_model=DeleteTensorResponse
+        )
+
+    async def update_tensor_metadata(
+        self,
+        dataset_name: str,
+        record_id: str,
+        new_metadata: dict
+    ) -> UpdateMetadataResponse:
+        return await self._call_json(
+            "tensorus_update_tensor_metadata",
+            {"dataset_name": dataset_name, "record_id": record_id, "new_metadata": new_metadata},
+            response_model=UpdateMetadataResponse
+        )
+
+    # --- Tensor operations ---
+    async def apply_unary_operation(self, operation: str, payload: dict) -> OperationResponse:
+        return await self._call_json(
+            "tensorus_apply_unary_operation",
+            {"operation": operation, **payload},
+            response_model=OperationResponse
+        )
+
+    async def apply_binary_operation(self, operation: str, payload: dict) -> OperationResponse:
+        return await self._call_json(
+            "tensorus_apply_binary_operation",
+            {"operation": operation, **payload},
+            response_model=OperationResponse
+        )
+
+    async def apply_list_operation(self, operation: str, payload: dict) -> OperationResponse:
+        return await self._call_json(
+            "tensorus_apply_list_operation",
+            {"operation": operation, **payload},
+            response_model=OperationResponse
+        )
+
+    async def apply_einsum(self, payload: dict) -> OperationResponse:
+        return await self._call_json(
+            "tensorus_apply_einsum",
+            payload,
+            response_model=OperationResponse
+        )
+
+    # --- Miscellaneous ---
+    async def execute_nql_query(self, query: str) -> NQLQueryResponse:
+        return await self._call_json(
+            "execute_nql_query",
+            {"query": query},
+            response_model=NQLQueryResponse
+        )
+
+# --- Pydantic response models ---
+class DatasetListResponse(BaseModel):
+    datasets: list[str]
+
+class CreateDatasetResponse(BaseModel):
+    success: bool
+    message: Optional[str]
+
+class DeleteDatasetResponse(BaseModel):
+    success: bool
+
+class IngestTensorResponse(BaseModel):
+    id: str
+    status: str
+
+class TensorDetailsResponse(BaseModel):
+    id: str
+    shape: list[int]
+    dtype: str
+    data: Any
+    metadata: Optional[dict]
+
+class DeleteTensorResponse(BaseModel):
+    success: bool
+
+class UpdateMetadataResponse(BaseModel):
+    success: bool
+
+class OperationResponse(BaseModel):
+    result: Any
+
+class NQLQueryResponse(BaseModel):
+    results: Any
+
+# End of TensorusMCPClient