Merge pull request #37 from se4ai2526-uniba/aggiornamento-doc

Refactor repository structure for Docker integration and update documentation

.github/workflows/ci.yml

@@ -86,8 +86,34 @@ jobs:
 
     - name: Pull Models with DVC
       run: |
-        dvc pull models/random_forest_embedding_gridsearch.pkl models/label_names.pkl
+        # Clean any potentially corrupted cache
+        rm -rf .dvc/cache/files/md5/e1 || true
+        
+        # Retry logic for DVC pull to handle intermittent server errors
+        max_attempts=3
+        attempt=0
+        until [ $attempt -ge $max_attempts ]
+        do
+          dvc pull models/random_forest_embedding_gridsearch.pkl models/label_names.pkl && break
+          attempt=$((attempt+1))
+          echo "DVC pull attempt $attempt failed. Retrying in 10 seconds..."
+          # Clean cache on retry
+          rm -rf .dvc/cache || true
+          sleep 10
+        done
+        if [ $attempt -ge $max_attempts ]; then
+          echo "DVC pull failed after $max_attempts attempts"
+          exit 1
+        fi
+
+    - name: Verify Models Downloaded
+      run: |
+        if [ ! -f "models/random_forest_embedding_gridsearch.pkl" ] || [ ! -f "models/label_names.pkl" ]; then
+          echo "ERROR: Required model files not found after DVC pull"
+          exit 1
+        fi
+        echo "All required model files present"
 
     - name: Build Docker Image
       run: |
-        docker build . -t hopcroft-app:latest
+        docker build -t hopcroft-app:latest .

Dockerfile

@@ -59,14 +59,14 @@ COPY --chown=user:user . .
 RUN chown -R user:user /app
 
 # Fix line endings and permissions for the start script
-RUN dos2unix scripts/start_space.sh && \
-    chmod +x scripts/start_space.sh
+RUN dos2unix docker/scripts/start_space.sh && \
+    chmod +x docker/scripts/start_space.sh
 
 # Install the project itself
 RUN pip install --no-cache-dir .
 
 # Make start script executable
-RUN chmod +x scripts/start_space.sh
+RUN chmod +x docker/scripts/start_space.sh
 
 # Switch to non-root user
 USER user
@@ -75,4 +75,4 @@ USER user
 EXPOSE 7860
 
 # Command to run the application
-CMD ["./scripts/start_space.sh"]
+CMD ["./docker/scripts/start_space.sh"]

README.md

@@ -249,7 +249,7 @@ make test-api-list          # List predictions
 #### Docker
 Build and run the API in a container:
 ```bash
-docker build -t hopcroft-api .
+docker build -f docker/Dockerfile -t hopcroft-api .
 docker run --rm --name hopcroft-api -p 8080:8080 hopcroft-api
 ```
 
@@ -257,6 +257,80 @@ Endpoints:
 - Swagger UI: [http://localhost:8080/docs](http://localhost:8080/docs)
 - Health check: [http://localhost:8080/health](http://localhost:8080/health)
 
+### Milestone 5 (Deployment)
+We implemented a complete containerized deployment pipeline for production-ready delivery:
+
+1. **Docker Containerization**
+    - `docker/Dockerfile`: Multi-stage Python 3.10 slim image with non-root user, system dependencies (git, nginx, curl), DVC integration, and automated startup script.
+    - `docker/Dockerfile.streamlit`: Lightweight container for Streamlit GUI with minimal dependencies.
+    - `docker/.dockerignore`: Optimized build context excluding unnecessary files.
+
+2. **Docker Compose Orchestration**
+    - Multi-service architecture: API backend (`hopcroft-api`), Streamlit frontend (`hopcroft-gui`), and monitoring stack.
+    - Bridge network (`hopcroft-net`) for inter-service communication.
+    - Health checks with automatic restart policies.
+    - Bind mounts for development hot-reload, named volumes for persistent storage (`hopcroft-logs`).
+
+3. **Hugging Face Spaces Deployment**
+    - Docker SDK configuration with port 7860.
+    - `docker/scripts/start_space.sh`: Automated startup script that configures DVC credentials, pulls models from DagsHub, and starts FastAPI + Streamlit + Nginx.
+    - Secrets management via HF Spaces Variables (`DAGSHUB_USERNAME`, `DAGSHUB_TOKEN`).
+    - Live deployment: `https://huggingface.co/spaces/se4ai2526-uniba/Hopcroft`
+
+4. **Nginx Reverse Proxy**
+    - `docker/nginx.conf`: Routes traffic to API (port 8000) and Streamlit (port 8501) on single port 7860.
+    - Path-based routing for API docs, metrics, and web interface.
+
+5. **Environment Configuration**
+    - `.env.example` template with MLflow and DagsHub credentials.
+    - Automatic environment variable injection via `env_file` directive.
+
+### Milestone 6 (Monitoring)
+We implemented comprehensive observability and load testing infrastructure:
+
+1. **Prometheus Metrics Collection**
+    - `prometheus.yml`: Scrape configuration for API metrics (10s interval), self-monitoring, and Pushgateway.
+    - Custom metrics: `hopcroft_requests_total`, `hopcroft_request_duration_seconds`, `hopcroft_in_progress_requests`, `hopcroft_prediction_processing_seconds`.
+    - PromQL queries for request rate, latency percentiles, and in-progress tracking.
+
+2. **Grafana Dashboards**
+    - Auto-provisioned datasources and dashboards via `provisioning/` directory.
+    - `hopcroft_dashboard.json`: Real-time visualization of API request rate, latency, drift status, and p-value metrics.
+    - Credentials: `admin/admin` on port 3000.
+
+3. **Alerting System**
+    - `alert_rules.yml`: Prometheus alert rules for `ServiceDown`, `HighErrorRate` (>10% 5xx), `SlowRequests` (p95 > 2s).
+    - Alertmanager configuration with severity-based routing and inhibition rules.
+    - Webhook integration for alert notifications.
+
+4. **Data Drift Detection**
+    - `prepare_baseline.py`: Extracts 1000-sample reference dataset from training data.
+    - `run_drift_check.py`: Kolmogorov-Smirnov two-sample test with Bonferroni correction (p < 0.05).
+    - Metrics pushed to Pushgateway: `drift_detected`, `drift_p_value`, `drift_distance`, `drift_check_timestamp`.
+    - JSON reports saved to `monitoring/drift/reports/`.
+
+5. **Locust Load Testing**
+    - `locustfile.py`: Simulated user behavior with weighted tasks (60% single prediction, 20% batch, 20% monitoring).
+    - Configurable wait times (1-5s) for realistic traffic simulation.
+    - Web UI on port 8089, headless mode support, CSV export for results.
+    - Pre-configured for HF Spaces and local Docker environments.
+
+6. **Uptime Monitoring (Better Stack)**
+    - External monitoring of production endpoints (`/health`, `/openapi.json`, `/docs`).
+    - Multi-location checks with email notifications.
+    - Incident tracking and resolution screenshots in `monitoring/screenshots/`.
+
+7. **CI/CD Pipeline**
+    - `.github/workflows/ci.yml`: GitHub Actions workflow triggered on push/PR to main and feature branches.
+    - Jobs: Ruff linting, pytest unit tests with HTML reports, DVC model pulling, Docker image build.
+    - Secrets: `DAGSHUB_USERNAME`, `DAGSHUB_TOKEN` for model access.
+    - Disk space optimization for CI runner.
+
+8. **Pushgateway Integration**
+    - Collects metrics from short-lived jobs (drift detection scripts).
+    - Persistent storage with 5-minute intervals.
+    - Scraped by Prometheus for long-term storage and Grafana visualization.
+
 ---
 
 ## Docker Compose Usage
@@ -283,7 +357,7 @@ Docker Compose orchestrates both the **API backend** and **Streamlit GUI** servi
 #### 1. Build and Start All Services
 Build both images and start the containers:
 ```bash
-docker-compose up -d --build
+docker compose -f docker/docker-compose.yml up -d --build
 ```
 
 | Flag | Description |
@@ -299,7 +373,7 @@ docker-compose up -d --build
 #### 2. Stop All Services
 Stop and remove containers and networks:
 ```bash
-docker-compose down
+docker compose -f docker/docker-compose.yml down
 ```
 
 | Flag | Description |
@@ -310,19 +384,19 @@ docker-compose down
 #### 3. Restart Services
 After updating `.env` or configuration files:
 ```bash
-docker-compose restart
+docker compose -f docker/docker-compose.yml restart
 ```
 
 Or for a full restart with environment reload:
 ```bash
-docker-compose down
-docker-compose up -d
+docker compose -f docker/docker-compose.yml down
+docker compose -f docker/docker-compose.yml up -d
 ```
 
 #### 4. Check Status
 View the status of all running services:
 ```bash
-docker-compose ps
+docker compose -f docker/docker-compose.yml ps
 ```
 
 Or use Docker commands:
@@ -333,13 +407,13 @@ docker ps
 #### 5. View Logs
 Tail logs from both services in real-time:
 ```bash
-docker-compose logs -f
+docker compose -f docker/docker-compose.yml logs -f
 ```
 
 View logs from a specific service:
 ```bash
-docker-compose logs -f hopcroft-api
-docker-compose logs -f hopcroft-gui
+docker compose -f docker/docker-compose.yml logs -f hopcroft-api
+docker compose -f docker/docker-compose.yml logs -f hopcroft-gui
 ```
 
 | Flag | Description |
@@ -350,8 +424,8 @@ docker-compose logs -f hopcroft-gui
 #### 6. Execute Commands in Container
 Open an interactive shell inside a running container:
 ```bash
-docker-compose exec hopcroft-api /bin/bash
-docker-compose exec hopcroft-gui /bin/bash
+docker compose -f docker/docker-compose.yml exec hopcroft-api /bin/bash
+docker compose -f docker/docker-compose.yml exec hopcroft-gui /bin/bash
 ```
 
 Examples of useful commands inside the API container:
@@ -375,9 +449,9 @@ printenv | grep MLFLOW
 **Docker Compose orchestrates two services:**
 
 ```
-docker-compose.yml
+docker/docker-compose.yml
 ├── hopcroft-api (FastAPI Backend)
-│   ├── Build: ./Dockerfile
+│   ├── Build: docker/Dockerfile
 │   ├── Port: 8080:8080
 │   ├── Network: hopcroft-net
 │   ├── Environment: .env (MLflow credentials)
@@ -387,7 +461,7 @@ docker-compose.yml
 │   └── Health Check: /health endpoint
 │
 ├── hopcroft-gui (Streamlit Frontend)
-│   ├── Build: ./Dockerfile.streamlit
+│   ├── Build: docker/Dockerfile.streamlit
 │   ├── Port: 8501:8501
 │   ├── Network: hopcroft-net
 │   ├── Environment: API_BASE_URL=http://hopcroft-api:8080
@@ -409,7 +483,7 @@ docker-compose.yml
 
 **hopcroft-api (FastAPI Backend)**
 - Purpose: FastAPI backend serving the ML model for skill classification
-- Image: Built from `Dockerfile`
+- Image: Built from `docker/Dockerfile`
 - Port: 8080 (maps to host 8080)
 - Features:
   - Random Forest model with embedding features
@@ -419,7 +493,7 @@ docker-compose.yml
 
 **hopcroft-gui (Streamlit Frontend)**
 - Purpose: Streamlit web interface for interactive predictions
-- Image: Built from `Dockerfile.streamlit`
+- Image: Built from `docker/Dockerfile.streamlit`
 - Port: 8501 (maps to host 8501)
 - Features:
   - User-friendly interface for skill prediction
@@ -441,7 +515,7 @@ docker-compose.yml
 - Use Dockerfile's CMD
 - GUI → API via Docker network
 
-For **production deployment**, modify `docker-compose.yml` to remove bind mounts and disable reload.
+For **production deployment**, modify `docker/docker-compose.yml` to remove bind mounts and disable reload.
 
 ### Troubleshooting
 
@@ -450,19 +524,19 @@ For **production deployment**, modify `docker-compose.yml` to remove bind mounts
 1. Wait 30-60 seconds for API to fully initialize and become healthy
 2. Refresh the GUI page (F5)
 3. Check API health: `curl http://localhost:8080/health`
-4. Check logs: `docker-compose logs hopcroft-api`
+4. Check logs: `docker compose -f docker/docker-compose.yml logs hopcroft-api`
 
 #### Issue: "500 Internal Server Error" on predictions
 **Solution:**
 1. Verify MLflow credentials in `.env` are correct
-2. Restart services: `docker-compose down && docker-compose up -d`
+2. Restart services: `docker compose -f docker/docker-compose.yml down && docker compose -f docker/docker-compose.yml up -d`
 3. Check environment variables: `docker exec hopcroft-api printenv | grep MLFLOW`
 
 #### Issue: Changes to code not reflected
 **Solution:**
 - For Python code changes: Auto-reload is enabled, wait a few seconds
-- For Dockerfile changes: Rebuild with `docker-compose up -d --build`
-- For `.env` changes: Restart with `docker-compose down && docker-compose up -d`
+- For Dockerfile changes: Rebuild with `docker compose -f docker/docker-compose.yml up -d --build`
+- For `.env` changes: Restart with `docker compose -f docker/docker-compose.yml down && docker compose -f docker/docker-compose.yml up -d`
 
 #### Issue: Port already in use
 **Solution:**
@@ -472,9 +546,9 @@ netstat -ano | findstr :8080
 netstat -ano | findstr :8501
 
 # Stop existing containers
-docker-compose down
+docker compose -f docker/docker-compose.yml down
 
-# Or change ports in docker-compose.yml
+# Or change ports in docker/docker-compose.yml
 ```
 
 
@@ -498,12 +572,12 @@ To enable the application to pull models from DagsHub via DVC, you must configur
 | `DAGSHUB_TOKEN` | Secret | Your DagsHub access token (Settings -> Tokens). |
 
 > [!IMPORTANT]
-> These secrets are injected into the container at runtime. The `scripts/start_space.sh` script uses them to authenticate DVC and pull the required model files (`.pkl`) before starting the API and GUI.
+> These secrets are injected into the container at runtime. The `docker/scripts/start_space.sh` script uses them to authenticate DVC and pull the required model files (`.pkl`) before starting the API and GUI.
 
 ### 3. Automated Startup
 The deployment follows this automated flow:
-1. **Dockerfile**: Builds the environment, installs dependencies, and sets up Nginx.
-2. **scripts/start_space.sh**: 
+1. **docker/Dockerfile**: Builds the environment, installs dependencies, and sets up Nginx.
+2. **docker/scripts/start_space.sh**: 
    - Configures DVC with your secrets.
    - Pulls models from the DagsHub remote.
    - Starts the **FastAPI** backend (port 8000).
@@ -531,7 +605,7 @@ The Streamlit GUI provides an interactive web interface for the skill classifica
 - Responsive design
 
 ### Usage
-1. Ensure both services are running: `docker-compose up -d`
+1. Ensure both services are running: `docker compose -f docker/docker-compose.yml up -d`
 2. Open the GUI in your browser: [http://localhost:8501](http://localhost:8501)
 3. Enter a GitHub issue description in the text area
 4. Click "Predict Skills" to get predictions

.dockerignore → docker/.dockerignore

@@ -27,5 +27,5 @@ notebooks/
 reports/
 docs/
 tests/
-scripts/
-!scripts/start_space.sh
+docker/scripts/
+!docker/scripts/start_space.sh

docker/README.md

@@ -0,0 +1,91 @@
+# Docker Configuration
+
+This directory contains all Docker-related configuration files for the Hopcroft Skill Classification project.
+
+## Directory Structure
+
+```
+docker/
+├── Dockerfile              # Main API container (Python 3.10, FastAPI, Nginx)
+├── Dockerfile.streamlit    # Streamlit GUI container
+├── docker-compose.yml      # Multi-service orchestration
+├── nginx.conf              # Reverse proxy configuration for HF Spaces
+├── .dockerignore           # Build context exclusions
+├── scripts/
+│   └── start_space.sh      # Startup script for Hugging Face Spaces
+└── README.md               # This file
+```
+
+## File Descriptions
+
+| File | Purpose |
+|------|---------|
+| `Dockerfile` | Main container with FastAPI API, DVC, Nginx. Used for HF Spaces deployment. |
+| `Dockerfile.streamlit` | Lightweight Streamlit-only container for GUI service. |
+| `docker-compose.yml` | Orchestrates API, GUI, Prometheus, Grafana, Alertmanager, Pushgateway. |
+| `nginx.conf` | Reverse proxy routing traffic to API (8000) and Streamlit (8501) on port 7860. |
+| `.dockerignore` | Excludes data/, tests/, docs/ etc. from Docker build context. |
+| `start_space.sh` | Configures DVC, pulls models, starts FastAPI + Streamlit + Nginx. |
+
+## Quick Start
+
+### Local Development
+
+```bash
+# From the docker/ directory
+docker compose up -d --build
+
+# Or from repository root
+docker compose -f docker/docker-compose.yml up -d --build
+```
+
+**Available Services:**
+| Service | URL |
+|---------|-----|
+| API (FastAPI) | http://localhost:8080/docs |
+| GUI (Streamlit) | http://localhost:8501 |
+| Prometheus | http://localhost:9090 |
+| Grafana | http://localhost:3000 (admin/admin) |
+| Pushgateway | http://localhost:9091 |
+| Alertmanager | http://localhost:9093 |
+
+### Stop Services
+
+```bash
+docker compose -f docker/docker-compose.yml down
+```
+
+### View Logs
+
+```bash
+docker compose -f docker/docker-compose.yml logs -f
+```
+
+## Building Individual Images
+
+```bash
+# From repository root
+docker build -f docker/Dockerfile -t hopcroft-api:latest .
+docker build -f docker/Dockerfile.streamlit -t hopcroft-gui:latest .
+```
+
+## Environment Variables
+
+The `.env` file in the repository root is automatically loaded. Required variables:
+
+```env
+MLFLOW_TRACKING_URI=https://dagshub.com/se4ai2526-uniba/Hopcroft.mlflow
+MLFLOW_TRACKING_USERNAME=your_username
+MLFLOW_TRACKING_PASSWORD=your_token
+```
+
+## Hugging Face Spaces
+
+The `Dockerfile` and `start_space.sh` are configured for HF Spaces deployment:
+- Exposes port 7860 (HF Spaces requirement)
+- Uses Nginx as reverse proxy
+- Pulls models from DagsHub via DVC on startup
+
+**Secrets required on HF Spaces:**
+- `DAGSHUB_USERNAME`
+- `DAGSHUB_TOKEN`

docker-compose.yml → docker/docker-compose.yml

@@ -1,18 +1,18 @@
 services:
   hopcroft-api:
     build:
-      context: .
+      context: ..
       dockerfile: Dockerfile
     container_name: hopcroft-api
     ports:
       - "8080:8080"
     env_file:
-      - .env
+      - ../.env
     environment:
       - PROJECT_NAME=Hopcroft
     volumes:
       # Bind mount: enables live code reloading for development
-      - ./hopcroft_skill_classification_tool_competition:/app/hopcroft_skill_classification_tool_competition
+      - ../hopcroft_skill_classification_tool_competition:/app/hopcroft_skill_classification_tool_competition
       # Named volume: persistent storage for application logs
       - hopcroft-logs:/app/logs
     networks:
@@ -30,8 +30,8 @@ services:
 
   hopcroft-gui:
     build:
-      context: .
-      dockerfile: Dockerfile.streamlit
+      context: ..
+      dockerfile: docker/Dockerfile.streamlit
     container_name: hopcroft-gui
     ports:
       - "8501:8501"
@@ -39,7 +39,7 @@ services:
       - API_BASE_URL=http://hopcroft-api:8080
     volumes:
       # Bind mount for development hot-reload
-      - ./hopcroft_skill_classification_tool_competition/streamlit_app.py:/app/streamlit_app.py
+      - ../hopcroft_skill_classification_tool_competition/streamlit_app.py:/app/streamlit_app.py
     networks:
       - hopcroft-net
     depends_on:
@@ -51,8 +51,8 @@ services:
     image: prom/prometheus:latest
     container_name: prometheus
     volumes:
-      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-      - ./monitoring/prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml
+      - ../monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ../monitoring/prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml
     ports:
       - "9090:9090"
     networks:
@@ -65,7 +65,7 @@ services:
     image: prom/alertmanager:latest
     container_name: alertmanager
     volumes:
-      - ./monitoring/alertmanager/config.yml:/etc/alertmanager/config.yml
+      - ../monitoring/alertmanager/config.yml:/etc/alertmanager/config.yml
     ports:
       - "9093:9093"
     networks:
@@ -84,9 +84,9 @@ services:
       - GF_SERVER_ROOT_URL=http://localhost:3000
     volumes:
       # Provisioning: auto-configure datasources and dashboards
-      - ./monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
-      - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
-      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
+      - ../monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ../monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ../monitoring/grafana/dashboards:/var/lib/grafana/dashboards
       # Persistent storage for Grafana data
       - grafana-data:/var/lib/grafana
     networks:
@@ -95,7 +95,7 @@ services:
       - prometheus
     restart: unless-stopped
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:3000/api/health || exit 1"]
+      test: [ "CMD-SHELL", "curl -f http://localhost:3000/api/health || exit 1" ]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -115,7 +115,6 @@ services:
     volumes:
       - pushgateway-data:/data
 
-
 networks:
   hopcroft-net:
     driver: bridge

{scripts → docker/scripts}/start_space.sh

@@ -106,7 +106,7 @@ if ! command -v nginx &> /dev/null; then
     echo "$(date) - ERROR: nginx not found in PATH"
     exit 1
 fi
-nginx -c /app/nginx.conf -g "daemon off;" >> /tmp/nginx_startup.log 2>&1 &
+nginx -c /app/docker/nginx.conf -g "daemon off;" >> /tmp/nginx_startup.log 2>&1 &
 
 echo "$(date) - Waiting for Nginx to initialize..."
 sleep 5

tests/integration/test_feature_pipeline.py

@@ -130,7 +130,8 @@ class TestDataFlowConsistency:
             "Normal clean text",
         ]
         
-        features_dirty, _ = extract_tfidf_features(dirty_df, max_features=50)
+        # Use min_df=1 and max_df=1.0 for small test datasets to avoid empty vocabulary
+        features_dirty, _ = extract_tfidf_features(dirty_df, max_features=50, min_df=1, max_df=1.0)
         
         # Clean version
         clean_df = sample_dataframe.copy()
@@ -142,7 +143,8 @@ class TestDataFlowConsistency:
             "Normal clean text",
         ]
         
-        features_clean, _ = extract_tfidf_features(clean_df, max_features=50)
+        # Use min_df=1 and max_df=1.0 for small test datasets
+        features_clean, _ = extract_tfidf_features(clean_df, max_features=50, min_df=1, max_df=1.0)
         
         # Features should be similar (cleaning is applied to both)
         # But not necessarily identical due to stemming
@@ -283,19 +285,20 @@ class TestErrorHandlingInPipeline:
             extract_tfidf_features(df)
     
     def test_pipeline_with_all_nan_text(self):
-        """Test pipeline with all NaN text values."""
+        """Test pipeline with all NaN text values raises appropriate error.
+        
+        TF-IDF cannot build a vocabulary from empty/NaN documents,
+        so it should raise a ValueError with a descriptive message.
+        """
         df = pd.DataFrame({
             'issue text': [None, None, None],
             'issue description': [None, None, None],
             'Label1': [1, 0, 1],
         })
         
-        # Should handle NaN values without crashing
-        features, _ = extract_tfidf_features(df, max_features=50)
-        
-        # May result in zero features for all samples
-        assert features.shape[0] == 3
-        assert not np.any(np.isnan(features))
+        # TF-IDF should raise ValueError for empty vocabulary
+        with pytest.raises(ValueError, match="empty vocabulary"):
+            extract_tfidf_features(df, max_features=50)
     
     def test_pipeline_with_empty_labels(self):
         """Test pipeline when no labels are present."""