Upload 35 files

src/__init__.py

@@ -0,0 +1 @@
+"""OXON Technologies / CANedge Data Lake application and utilities."""

src/examples/batch_example.py

@@ -5,11 +5,18 @@ This script demonstrates memory-efficient batch processing across
 the entire data lake using SQL queries.
 """
 
-from datalake.config import DataLakeConfig
-from datalake.athena import AthenaQuery
-from datalake.catalog import DataLakeCatalog
-from datalake.query import DataLakeQuery
-from datalake.batch import BatchProcessor
+import sys
+from pathlib import Path
+
+_project_root = Path(__file__).resolve().parent.parent.parent
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
+from src.datalake.config import DataLakeConfig
+from src.datalake.athena import AthenaQuery
+from src.datalake.catalog import DataLakeCatalog
+from src.datalake.query import DataLakeQuery
+from src.datalake.batch import BatchProcessor
 import pandas as pd
 
 

src/examples/explore_example.py

@@ -5,9 +5,16 @@ This script demonstrates how to discover devices, messages, dates,
 and schemas in the CANedge Athena data lake.
 """
 
-from datalake.config import DataLakeConfig
-from datalake.athena import AthenaQuery
-from datalake.catalog import DataLakeCatalog
+import sys
+from pathlib import Path
+
+_project_root = Path(__file__).resolve().parent.parent.parent
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
+from src.datalake.config import DataLakeConfig
+from src.datalake.athena import AthenaQuery
+from src.datalake.catalog import DataLakeCatalog
 
 
 def main():

src/examples/query_example.py

@@ -5,10 +5,17 @@ This script demonstrates how to read data for specific devices/messages,
 perform time series queries, and filter by date ranges using SQL.
 """
 
-from datalake.config import DataLakeConfig
-from datalake.athena import AthenaQuery
-from datalake.catalog import DataLakeCatalog
-from datalake.query import DataLakeQuery
+import sys
+from pathlib import Path
+
+_project_root = Path(__file__).resolve().parent.parent.parent
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
+from src.datalake.config import DataLakeConfig
+from src.datalake.athena import AthenaQuery
+from src.datalake.catalog import DataLakeCatalog
+from src.datalake.query import DataLakeQuery
 import pandas as pd
 
 

src/streamlit_app.py

@@ -4,10 +4,16 @@ OXON Technologies - Professional Streamlit Dashboard
 A comprehensive dashboard for analyzing device data from AWS Athena data lake.
 """
 
+import sys
+from pathlib import Path
+
+_project_root = Path(__file__).resolve().parent.parent
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
 import streamlit as st
 from warnings import filterwarnings
 import base64
-from pathlib import Path
 from PIL import Image
 import pandas as pd
 import numpy as np
@@ -19,15 +25,18 @@ from typing import Dict, Optional, List, Tuple
 from ydata_profiling import ProfileReport
 import plotly.express as px
 
-from datalake.config import DataLakeConfig
-from datalake.athena import AthenaQuery
-from datalake.catalog import DataLakeCatalog
-from datalake.query import DataLakeQuery
-from datalake.batch import BatchProcessor
+from src.datalake.config import DataLakeConfig
+from src.datalake.athena import AthenaQuery
+from src.datalake.catalog import DataLakeCatalog
+from src.datalake.query import DataLakeQuery
+from src.datalake.batch import BatchProcessor
+
+from src.utils.correlation import CorrelationMatrixGenerator
+from src.utils.dimension_reduction import DimensionReduction
+from src.utils.feature_class import DetectFeatureClasses
 
-from utils.correlation import CorrelationMatrixGenerator
-from utils.dimension_reduction import DimensionReduction
-from utils.feature_class import DetectFeatureClasses
+# Base directory for config/images (relative to this file)
+_SRC_DIR = Path(__file__).resolve().parent
 
 # Ignore warnings
 filterwarnings("ignore")
@@ -36,20 +45,22 @@ filterwarnings("ignore")
 # Configuration Management
 # ============================================================================
 
-def load_config(config_path: str = "config.yaml") -> Dict:
+def load_config(config_path: Optional[str] = None) -> Dict:
     """
     Load configuration from YAML file.
-    
+
     Args:
-        config_path: Path to the configuration YAML file
-        
+        config_path: Path to the configuration YAML file (default: src/config.yaml)
+
     Returns:
         Dictionary containing configuration settings
-        
+
     Raises:
         FileNotFoundError: If config file doesn't exist
         yaml.YAMLError: If config file is invalid YAML
     """
+    if config_path is None:
+        config_path = _SRC_DIR / "config.yaml"
     config_file = Path(config_path)
     if not config_file.exists():
         raise FileNotFoundError(f"Configuration file not found: {config_path}")
@@ -245,20 +256,21 @@ def display_header(logo_path: str, title: str):
             <div style="display: flex; align-items: center;">
                 <img src="data:image/png;base64,{logo_base64}" alt="Logo" 
                      style="height: 200px; margin-right: 10px;">
-                <h1 style="display: inline; margin: 0;">{title} 🔍</h1>
+                <h1 style="display: inline; margin: 0;">{title} ??</h1>
             </div>
             """,
             unsafe_allow_html=True,
         )
     else:
-        st.title(f"{title} 🔍")
+        st.title(f"{title} ??")
 
 
 def display_sidebar():
     """Display sidebar with device selection."""
     with st.sidebar:
         # Logo
-        logo_path = st.session_state['app_config'].get('dashboard', {}).get('logo_path', 'images/logo.png')
+        logo_rel = st.session_state['app_config'].get('dashboard', {}).get('logo_path', 'images/logo.png')
+        logo_path = _SRC_DIR / logo_rel
         try:
             st.image(Image.open(logo_path), width='stretch')
         except Exception:
@@ -270,7 +282,7 @@ def display_sidebar():
         
         # Check if AWS services are initialized
         if not st.session_state['aws_initialized']:
-            st.warning("⚠️ AWS services not initialized. Please check configuration.")
+            st.warning("?? AWS services not initialized. Please check configuration.")
             return
         
         # Load device list if not cached
@@ -320,7 +332,7 @@ def display_sidebar():
         
         # Show selected device info only after user has confirmed
         if st.session_state['selected_device']:
-            st.success(f"✓ Selected: {st.session_state['selected_device']}")
+            st.success(f"? Selected: {st.session_state['selected_device']}")
 
 
 # ============================================================================
@@ -394,7 +406,7 @@ def render_message_viewer_tab():
         return
     
     if not st.session_state['selected_device']:
-        st.info("👈 Please select a device from the sidebar and click **Go!** to begin.")
+        st.info("?? Please select a device from the sidebar and click **Go!** to begin.")
         return
     
     device_id = st.session_state['selected_device']
@@ -467,7 +479,7 @@ def render_message_viewer_tab():
         st.rerun()
     
     if st.session_state['selected_message']:
-        st.info(f"📊 Selected message: `{st.session_state['selected_message']}` ({selected_message_name})")
+        st.info(f"?? Selected message: `{st.session_state['selected_message']}` ({selected_message_name})")
         
         # Date range selection (optional filter)
         st.divider()
@@ -527,7 +539,7 @@ def render_message_viewer_tab():
             if apply_filter_clicked:
                 # Validate date range before applying
                 if date_start > date_end:
-                    st.error("⚠️ Start date must be before or equal to end date.")
+                    st.error("?? Start date must be before or equal to end date.")
                 else:
                     st.session_state['applied_date_range_start'] = date_start
                     st.session_state['applied_date_range_end'] = date_end
@@ -536,14 +548,14 @@ def render_message_viewer_tab():
             # Show current applied filter status
             if st.session_state.get('applied_date_range_start') and st.session_state.get('applied_date_range_end'):
                 st.success(
-                    f"📅 **Applied filter:** {st.session_state['applied_date_range_start']} to "
+                    f"?? **Applied filter:** {st.session_state['applied_date_range_start']} to "
                     f"{st.session_state['applied_date_range_end']}"
                 )
             elif date_start and date_end:
                 if date_start <= date_end:
-                    st.info("ℹ️ Select dates and click **Apply Filter** to filter the data.")
+                    st.info("?? Select dates and click **Apply Filter** to filter the data.")
                 else:
-                    st.error("⚠️ Start date must be before or equal to end date.")
+                    st.error("?? Start date must be before or equal to end date.")
         else:
             # Clear applied date range when disabled
             if st.session_state.get('date_range_enabled'):
@@ -612,13 +624,13 @@ def render_message_data(device_id: str, message: str):
         
         if len(df) == 0:
             st.warning(
-                f"⚠️ No data found in the selected date range "
+                f"?? No data found in the selected date range "
                 f"({st.session_state['applied_date_range_start']} to {st.session_state['applied_date_range_end']})."
             )
             st.info("Try selecting a different date range or disable the filter to see all data.")
             return
         elif len(df) < original_row_count:
-            st.info(f"📊 Showing {len(df):,} of {original_row_count:,} records (filtered by date range).")
+            st.info(f"?? Showing {len(df):,} of {original_row_count:,} records (filtered by date range).")
     
     # Display statistics
     # st.subheader("Statistics")
@@ -859,7 +871,7 @@ def render_correlations_tab():
         return
     
     if not st.session_state['selected_device']:
-        st.info("👈 Please select a device from the sidebar and click **Go!** to begin.")
+        st.info("?? Please select a device from the sidebar and click **Go!** to begin.")
         return
     
     device_id = st.session_state['selected_device']
@@ -899,7 +911,7 @@ def render_correlations_tab():
         st.error("No features available for correlation analysis.")
         return
     
-    st.info(f"📊 Analyzing {len(df_features.columns)} features from {len(df)} days of data.")
+    st.info(f"?? Analyzing {len(df_features.columns)} features from {len(df)} days of data.")
     
     # Detect feature classes
     st.subheader("1. Feature Classification")
@@ -1000,7 +1012,7 @@ def render_correlations_tab():
             for idx, cluster in enumerate(cluster_list):
                 with st.expander(f"Cohort {idx + 1}: {len(cluster)} features (all pairs in {band_label})"):
                     for feature in cluster:
-                        st.write(f"  • {feature}")
+                        st.write(f"  � {feature}")
                     if len(cluster) > 1:
                         st.markdown("**Pairwise correlations (values lie in " + band_label + "):**")
                         cluster_corr = corr_matrix.loc[cluster, cluster]
@@ -1038,7 +1050,7 @@ def render_correlations_tab():
 
 def render_placeholder_tab():
     """Render placeholder tab."""
-    st.info("🚧 This feature is under development.")
+    st.info("?? This feature is under development.")
 
 
 # ============================================================================
@@ -1053,8 +1065,8 @@ def main():
     # Load configuration
     if st.session_state['app_config'] is None:
         st.error(
-            f"❌ Configuration Error: {st.session_state.get('config_error', 'Unknown error')}\n\n"
-            "Please ensure `config.yaml` exists and is properly formatted."
+            f"? Configuration Error: {st.session_state.get('config_error', 'Unknown error')}\n\n"
+            "Please ensure `src/config.yaml` exists and is properly formatted."
         )
         st.stop()
     
@@ -1062,8 +1074,8 @@ def main():
     if not initialize_aws_if_needed():
         if st.session_state['aws_error']:
             st.error(
-                f"❌ AWS Initialization Error: {st.session_state['aws_error']}\n\n"
-                "Please check your AWS credentials in `config.yaml`."
+                f"? AWS Initialization Error: {st.session_state['aws_error']}\n\n"
+                "Please check your AWS credentials in `src/config.yaml`."
             )
         st.stop()
     
@@ -1091,7 +1103,8 @@ def main():
     )
     
     # Display header
-    header_logo = dashboard_config.get('header_logo_path', 'images/analysis.png')
+    header_logo_rel = dashboard_config.get('header_logo_path', 'images/analysis.png')
+    header_logo = str(_SRC_DIR / header_logo_rel)
     header_title = dashboard_config.get('page_title', 'Analytical Dashboard')
     display_header(header_logo, header_title)
     

src/test_connection.py

@@ -2,9 +2,16 @@
 Quick test script to verify Athena connection and basic functionality.
 """
 
-from datalake.config import DataLakeConfig
-from datalake.athena import AthenaQuery
-from datalake.catalog import DataLakeCatalog
+import sys
+from pathlib import Path
+
+_project_root = Path(__file__).resolve().parent.parent
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
+from src.datalake.config import DataLakeConfig
+from src.datalake.athena import AthenaQuery
+from src.datalake.catalog import DataLakeCatalog
 
 def main():
     """Test basic connection and functionality."""

src/utils/dimension_reduction.py

@@ -5,7 +5,7 @@ import pandas as pd
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import MinMaxScaler
 
-from utils.correlation import CorrelationMatrixGenerator
+from src.utils.correlation import CorrelationMatrixGenerator
 
 
 class DimensionReduction: