Modification Config et Squelette

.dockerignore

@@ -0,0 +1,10 @@
+# Dossiers d'environnement virtuel
+.venv/
+env/
+venv/
+.env/
+
+# Cache Python
+__pycache__/
+*.pyc
+.pytest_cache/

.gitignore

@@ -0,0 +1,4 @@
+venv/
+env/
+__pycache__/
+*.pyc

Dockerfile

@@ -8,10 +8,11 @@ WORKDIR /app
 COPY . /app
 
 # Install any needed packages specified in requirements.txt
-RUN pip install pandas
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r requirements.txt
 
 # Make a volume mount point for the input/output CSV files
 VOLUME ["/app/input_data.csv", "/app/output_data.csv"]
 
 # Run the application (by default, run the main ETL process)
-CMD ["python", "etl_process.py"]
+CMD ["python", "main.py"]

data/output_data.csv

@@ -0,0 +1,6 @@
+id,name,age,city,salary,tax,net_salary
+1,John Doe,28.0,New York,70000,7000.0,63000.0
+2,Jane Smith,34.0,Los Angeles,80000,8000.0,72000.0
+3,Bob Johnson,45.0,Chicago,90000,9000.0,81000.0
+4,Alice Williams,29.0,San Francisco,85000,8500.0,76500.0
+6,Eve Davis,38.0,Boston,95000,9500.0,85500.0

jenkins/Jenkinsfile

@@ -2,8 +2,8 @@ pipeline {
     agent any
 
     environment {
-        TEST_IMAGE = 'paycare-tests'
-        ETL_IMAGE = 'paycare-etl'
+        TEST_IMAGE = 'air-quality-tests'
+        APP_IMAGE = 'air-quality-app'
         AWS_ACCESS_KEY_ID = credentials('aws-access-key')
         AWS_SECRET_ACCESS_KEY = credentials('aws-secret-key')
         AWS_DEFAULT_REGION = 'eu-central-1' 
@@ -32,24 +32,24 @@ pipeline {
                         --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
                         --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
                         --env AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \
-                        paycare-tests
+                        ${TEST_IMAGE}
             '''
             }
         }
 
-        stage('Build ETL Container') {
+        stage('Build App Container') {
             steps {
-                sh 'docker build -t ${ETL_IMAGE} .'
+                sh 'docker build -t ${APP_IMAGE} .'
             }
         }
 
-        stage('Run ETL in Docker') {
+        stage('Run App in Docker') {
             steps {
                 script {
                     sh '''
                         docker run --rm \
                         -v ${WORKSPACE}/data:/app/data \
-                        ${ETL_IMAGE}
+                        ${APP_IMAGE}
                     '''
 
                     sh 'ls -l ${WORKSPACE}/data'
@@ -61,11 +61,11 @@ pipeline {
 
     post {
         success {
-            echo '✅ ETL Pipeline completed successfully!'
+            echo '✅ App Pipeline completed successfully!'
             archiveArtifacts artifacts: 'data/output_data.csv', fingerprint: true
         }
         failure {
-            echo '❌ ETL Pipeline failed.'
+            echo '❌ App Pipeline failed.'
         }
     }
 }

tests/.dockerignore

@@ -0,0 +1,10 @@
+# Dossiers d'environnement virtuel
+.venv/
+env/
+venv/
+.env/
+
+# Cache Python
+__pycache__/
+*.pyc
+.pytest_cache/

tests/Dockerfile

@@ -6,12 +6,13 @@ WORKDIR /app
 
 # Copier uniquement les fichiers nécessaires
 COPY tests/requirements.txt requirements.txt
-RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r requirements.txt
 
 # Copier le code source et les tests (avec le bon chemin)
-COPY etl_process.py /app/
-COPY tests/test_etl.py tests/upload_s3.py /app/
+COPY main.py /app/
+COPY tests/test_main.py tests/upload_s3.py /app/
 
 # Exécuter les tests Pytest au démarrage
 # CMD ["pytest", "test_etl.py", "--junitxml=results.xml"]
-CMD pytest test_etl.py --junitxml=results.xml && python upload_s3.py
+CMD pytest test_main.py --junitxml=results.xml && python upload_s3.py

tests/requirements.txt

@@ -1,4 +1,4 @@
-pandas==2.0.3
-numpy==1.24.4
+pandas
+numpy
 pytest
 boto3

tests/{test_etl.py → test_main.py}

@@ -1,7 +1,7 @@
 import pytest
 import pandas as pd
 from io import StringIO
-from etl_process import extract_data, transform_data, load_data
+from main import extract_data, transform_data, load_data
 
 
 # Test for data extraction