Suppression de fichiers obsolètes + modif requirements

app/jedha_final_project.ipynb

@@ -1,580 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Libs",
-   "id": "dae9db5e62cec5e9"
-  },
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-07-09T19:43:39.841918Z",
-     "start_time": "2025-07-09T19:43:39.401113Z"
-    }
-   },
-   "source": [
-    "import os\n",
-    "\n",
-    "import boto3\n",
-    "import pandas as pd\n",
-    "# Charger les variables\n",
-    "from dotenv import load_dotenv\n"
-   ],
-   "outputs": [],
-   "execution_count": 1
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# All",
-   "id": "8c0c6c3d85f13653"
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-09T19:38:42.289222Z",
-     "start_time": "2025-07-09T19:38:16.883228Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "# df_traffic = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/comptages-routiers-permanents.csv',\n",
-    "#                          sep=';', on_bad_lines='skip')\n",
-    "# df_nox = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/2024_NOX.csv', sep=',', on_bad_lines='skip')\n",
-    "# df_O3 = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/2024_O3.csv', sep=',', on_bad_lines='skip')\n",
-    "# df_pm10 = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/2024_pm10.csv', sep=',', on_bad_lines='skip')\n",
-    "# df_pm25 = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/2024_pm25.csv', sep=',', on_bad_lines='skip')\n",
-    "# df_meteo = pd.read_csv('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/H_75_latest-2024-2025.csv', sep=';')\n"
-   ],
-   "id": "96738dbb6b0524b6",
-   "outputs": [],
-   "execution_count": 2
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Meteo",
-   "id": "8a0a89e2100fc626"
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Clean",
-   "id": "84ec54a1e60f633"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "# Convertir en format Date et renommer la colonne AAAAMMJJHH\n",
-    "df_meteo['AAAAMMJJHH'] = pd.to_datetime(df_meteo[\"AAAAMMJJHH\"], format=\"%Y%m%d%H\", utc=True)\n",
-    "df_meteo = df_meteo.rename(columns={\"AAAAMMJJHH\": \"Timestamp\"})\n",
-    "\n",
-    "# Supprimer toutes les colonnes où toutes les valeurs sont NaN\n",
-    "# Permet de passer de 204 colonnes a 98\n",
-    "df_meteo = df_meteo.dropna(how=\"all\", axis=1)\n",
-    "\n",
-    "# Supprimer les lignes où \"PARIS-MONTSOURIS-DOUBLE\" est dans la colonne \"NOM_USUEL\"\n",
-    "# Permet de passer de 80 k columns a 65 k\n",
-    "df_meteo = df_meteo[~df_meteo['NOM_USUEL'].str.contains(\"PARIS-MONTSOURIS-DOUBLE\", na=False)]\n",
-    "\n",
-    "df_meteo.reset_index(inplace=True)\n",
-    "df_meteo = df_meteo.sort_values(by=['Timestamp'])"
-   ],
-   "id": "11f81e08321616c7",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Pivot",
-   "id": "c4c59f29f647cd51"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "# Pivoter le DataFrame\n",
-    "df_meteo_pivoted = df_meteo.set_index(['Timestamp', 'NOM_USUEL']).unstack()\n",
-    "df_meteo_pivoted = df_meteo_pivoted.drop(['index', 'NUM_POSTE', 'LAT', 'LON', 'ALTI'], axis=1)\n",
-    "\n",
-    "df_meteo_pivoted.columns = [f\"{station}_{var}\" for var, station in df_meteo_pivoted.columns]\n",
-    "df_meteo_pivoted = df_meteo_pivoted.reset_index()\n",
-    "\n",
-    "# Extraire les identifiants de station uniques\n",
-    "station_ids = sorted({col.split('_')[0] for col in df_meteo_pivoted.columns if '_' in col})\n",
-    "\n",
-    "# Réorganiser les colonnes\n",
-    "sorted_columns = ['Timestamp'] + [col for station in station_ids for col in df_meteo_pivoted.columns if\n",
-    "                                  col.startswith(station)]\n",
-    "\n",
-    "# Réorganiser le DataFrame\n",
-    "df_meteo_pivoted = df_meteo_pivoted[sorted_columns]\n",
-    "\n",
-    "df_meteo_pivoted"
-   ],
-   "id": "a0d4f42370a2cdca",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "df_meteo_pivoted.to_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/meteo_cleaned_pivoted.parquet',\n",
-    "                            index=False)"
-   ],
-   "id": "196b3e20978976ec",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Pollutants",
-   "id": "c075b0ecc2339caa"
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Clean",
-   "id": "f0ea5ee496220a8c"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "##################################################\n",
-    "# NOX\n",
-    "##################################################\n",
-    "\n",
-    "# Rename col Unnamed: 0 et convertir en format Date\n",
-    "df_nox = df_nox.rename(columns={\"Unnamed: 0\": \"Timestamp\"})\n",
-    "df_nox['Timestamp'] = pd.to_datetime(df_nox[\"Timestamp\"], utc=True)\n",
-    "\n",
-    "# 8 800 ; 40 columns vers 7 columns\n",
-    "# Liste des chaînes à rechercher dans les noms de colonnes\n",
-    "colonnes_a_garder = ['Timestamp', 'PA18', 'EIFF3', 'PA13', 'NEUIL', 'BONAP']\n",
-    "\n",
-    "# Filtrer les colonnes du DataFrame\n",
-    "df_nox = df_nox.loc[:,\n",
-    "         df_nox.columns.isin(colonnes_a_garder) | df_nox.columns.str.contains('|'.join(colonnes_a_garder))]\n",
-    "\n",
-    "# Supprimer les lignes contenant NaN dans la colonne \"Timestamp\"\n",
-    "df_nox = df_nox.dropna(subset=['Timestamp'])\n",
-    "\n",
-    "# df_nox.reset_index(inplace=True)\n",
-    "df_nox = df_nox.sort_values(by=['Timestamp'])\n",
-    "\n",
-    "##################################################\n",
-    "# O3\n",
-    "##################################################\n",
-    "# Rename col Unnamed: 0 et convertir en format Date\n",
-    "df_O3 = df_O3.rename(columns={\"Unnamed: 0\": \"Timestamp\"})\n",
-    "df_O3['Timestamp'] = pd.to_datetime(df_O3[\"Timestamp\"], utc=True)\n",
-    "\n",
-    "# Liste des chaînes à rechercher dans les noms de colonnes\n",
-    "colonnes_a_garder = ['Timestamp', 'PA18', 'EIFF3', 'PA13', 'NEUIL', 'PA01H']\n",
-    "\n",
-    "# Filtrer les colonnes du DataFrame\n",
-    "df_O3 = df_O3.loc[:, df_O3.columns.isin(colonnes_a_garder) | df_O3.columns.str.contains('|'.join(colonnes_a_garder))]\n",
-    "\n",
-    "# Supprimer les lignes contenant NaN dans la colonne \"Timestamp\"\n",
-    "df_O3 = df_O3.dropna(subset=['Timestamp'])\n",
-    "\n",
-    "# df_O3.reset_index(inplace=True)\n",
-    "df_O3 = df_O3.sort_values(by=['Timestamp'])\n",
-    "\n",
-    "##################################################\n",
-    "# pm10\n",
-    "##################################################\n",
-    "# Rename col Unnamed: 0 et convertir en format Date\n",
-    "df_pm10 = df_pm10.rename(columns={\"Unnamed: 0\": \"Timestamp\"})\n",
-    "df_pm10['Timestamp'] = pd.to_datetime(df_pm10[\"Timestamp\"], utc=True)\n",
-    "\n",
-    "# Liste des chaînes à rechercher dans les noms de colonnes\n",
-    "colonnes_a_garder = ['Timestamp', 'PA18', 'ELYS', 'BASCH', 'AUT', 'PA01H']\n",
-    "\n",
-    "# Filtrer les colonnes du DataFrame\n",
-    "df_pm10 = df_pm10.loc[:,\n",
-    "          df_pm10.columns.isin(colonnes_a_garder) | df_pm10.columns.str.contains('|'.join(colonnes_a_garder))]\n",
-    "\n",
-    "# Supprimer les lignes contenant NaN dans la colonne \"Timestamp\"\n",
-    "df_pm10 = df_pm10.dropna(subset=['Timestamp'])\n",
-    "\n",
-    "# df_pm10.reset_index(inplace=True)\n",
-    "df_pm10 = df_pm10.sort_values(by=['Timestamp'])\n",
-    "\n",
-    "##################################################\n",
-    "# pm25\n",
-    "##################################################\n",
-    "# Rename col Unnamed: 0 et convertir en format Date\n",
-    "df_pm25 = df_pm25.rename(columns={\"Unnamed: 0\": \"Timestamp\"})\n",
-    "df_pm25['Timestamp'] = pd.to_datetime(df_pm25[\"Timestamp\"], utc=True)\n",
-    "\n",
-    "# Liste des chaînes à rechercher dans les noms de colonnes\n",
-    "colonnes_a_garder = ['Timestamp', 'PA18', 'ELYS', 'AUT', 'PA01H']\n",
-    "\n",
-    "# Filtrer les colonnes du DataFrame\n",
-    "df_pm25 = df_pm25.loc[:,\n",
-    "          df_pm25.columns.isin(colonnes_a_garder) | df_pm25.columns.str.contains('|'.join(colonnes_a_garder))]\n",
-    "\n",
-    "# Supprimer les lignes contenant NaN dans la colonne \"Timestamp\"\n",
-    "df_pm25 = df_pm25.dropna(subset=['Timestamp'])\n",
-    "\n",
-    "# df_pm25.reset_index(inplace=True)\n",
-    "df_pm25 = df_pm25.sort_values(by=['Timestamp'])\n"
-   ],
-   "id": "20e9485dea763097",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Merge",
-   "id": "96cf48a9f7521fcd"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "df_merged = pd.merge_asof(df_nox,\n",
-    "                          df_O3,\n",
-    "                          left_on='Timestamp',\n",
-    "                          right_on='Timestamp',\n",
-    "                          direction='nearest')\n",
-    "\n",
-    "df_merged = pd.merge_asof(df_merged,\n",
-    "                          df_pm10,\n",
-    "                          left_on='Timestamp',\n",
-    "                          right_on='Timestamp',\n",
-    "                          direction='nearest')\n",
-    "\n",
-    "df_merged = pd.merge_asof(df_merged,\n",
-    "                          df_pm25,\n",
-    "                          left_on='Timestamp',\n",
-    "                          right_on='Timestamp',\n",
-    "                          direction='nearest')\n"
-   ],
-   "id": "2db2ed91c9efda4b",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Extract",
-   "id": "f13105d20628b7b0"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "df_merged.to_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/df_pollutants_cleaned_pivoted.parquet',\n",
-    "                     index=False)\n"
-   ],
-   "id": "eaccdaee3f90298a",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Traffic",
-   "id": "bb30b5e28f65c9bc"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "# Convertir la colonne \"Date et heure de comptage\" en format Date\n",
-    "df_traffic['Date et heure de comptage'] = pd.to_datetime(df_traffic[\"Date et heure de comptage\"], utc=True)\n",
-    "df_traffic = df_traffic.rename(columns={\"Date et heure de comptage\": \"Timestamp\"})"
-   ],
-   "id": "de7fc1da2bf02136",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Clean",
-   "id": "126558e93cf2c2a6"
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-09T19:38:46.212441Z",
-     "start_time": "2025-07-09T19:38:42.317055Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "# Convertir la colonne \"Date et heure de comptage\" en format Date\n",
-    "df_traffic['Date et heure de comptage'] = pd.to_datetime(df_traffic[\"Date et heure de comptage\"], utc=True)\n",
-    "df_traffic = df_traffic.rename(columns={\"Date et heure de comptage\": \"Timestamp\"})\n",
-    "\n",
-    "# Filtrer les lignes contenant certaines valeurs dans la colonne \"Identifiant arc\"\n",
-    "ids = [1572, 1573, 4434, 4440, 728, 737, 5442, 5455, 615, 616]\n",
-    "\n",
-    "# Filtrer uniquement sur les identifiants\n",
-    "df_traffic = df_traffic[df_traffic['Identifiant arc'].isin(ids)]\n",
-    "\n",
-    "df_traffic = df_traffic.sort_values(by=['Timestamp'])"
-   ],
-   "id": "9c0ea39992c0f566",
-   "outputs": [],
-   "execution_count": 3
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "# Création d’un identifiant unique par site\n",
-    "df_traffic_filtered[\"ID_Libelle\"] = df_traffic_filtered[\"Identifiant arc\"].astype(str) + \"_\" + df_traffic_filtered[\"Libelle\"]\n",
-    "\n",
-    "df_traffic_filtered_pivoted = df_traffic_filtered.set_index(['Timestamp', 'ID_Libelle']).unstack()\n",
-    "\n",
-    "df_traffic_filtered_pivoted.columns = [f\"{station}_{var}\" for var, station in df_traffic_filtered_pivoted.columns]\n",
-    "df_traffic_filtered_pivoted = df_traffic_filtered_pivoted.reset_index()\n",
-    "\n",
-    "# Extraire les identifiants de station uniques\n",
-    "ids_libelles = sorted({col.split('_')[0] for col in df_meteo_pivoted.columns if '_' in col})\n",
-    "\n",
-    "# Réorganiser les colonnes\n",
-    "sorted_columns = ['Timestamp'] + [col for station in station_ids for col in df_meteo_pivoted.columns if\n",
-    "                                  col.startswith(station)]\n",
-    "\n",
-    "# Réorganiser le DataFrame\n",
-    "df_meteo_pivoted = df_meteo_pivoted[sorted_columns]\n",
-    "\n",
-    "\n",
-    "# # On \"pivot\" le DataFrame pour avoir une seule ligne par timestamp\n",
-    "# df_traffic_filtered_pivot = df_traffic_filtered.melt(id_vars=[\"Timestamp\", \"site_id\"],\n",
-    "#                    value_vars=[col for col in df_traffic_filtered.columns if col not in [\"Timestamp\", \"code_site\", \"Libelle\", \"Identifiant arc\"]],\n",
-    "#                    var_name=\"variable\", value_name=\"valeur\")\n",
-    "#\n",
-    "# # Création des noms de colonnes finaux\n",
-    "# df_traffic_filtered_pivot[\"colonne_finale\"] = df_traffic_filtered_pivot[\"site_id\"] + \"_\" + df_traffic_filtered_pivot[\"variable\"]\n",
-    "#\n",
-    "# # Restructuration du tableau\n",
-    "# df_traffic_filtered_final = df_traffic_filtered_pivot.pivot_table(index=\"Timestamp\", columns=\"colonne_finale\", values=\"valeur\").reset_index()\n"
-   ],
-   "id": "af9f5b3120eeb1d",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "# Création d’un identifiant unique par site\n",
-    "df_traffic[\"ID_Libelle\"] = df_traffic[\"Identifiant arc\"].astype(str) + \"_\" + df_traffic[\n",
-    "    \"Libelle\"]\n",
-    "df_traffic = df_traffic.drop(['Identifiant arc', 'Libelle'], axis=1)\n",
-    "\n",
-    "# Pivoter le DataFrame\n",
-    "df_traffic_pivoted = df_traffic.set_index(['Timestamp', 'ID_Libelle']).unstack()"
-   ],
-   "id": "70ccccef23c73b1b",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": "df_traffic_pivoted.columns",
-   "id": "d37d22c8734776fe",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "\n",
-    "df_traffic_pivoted.columns = [f\"{station}_{var}\" for var, station in df_traffic_pivoted.columns]\n",
-    "df_traffic_pivoted = df_traffic_pivoted.reset_index()"
-   ],
-   "id": "e911bbf7d54cf3c8",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "\n",
-    "# Extraire les identifiants de station uniques sans couper à chaque underscore\n",
-    "ids_libelles = sorted({col.rsplit('_', 1)[0] for col in df_traffic_pivoted.columns if col != 'Timestamp'})\n",
-    "\n",
-    "# Réorganiser les colonnes par station\n",
-    "sorted_columns = ['Timestamp'] + [\n",
-    "    col for station in ids_libelles\n",
-    "    for col in df_traffic_pivoted.columns\n",
-    "    if col.startswith(station + \"_\")\n",
-    "]\n",
-    "\n",
-    "df_traffic_pivoted = df_traffic_pivoted[sorted_columns]\n"
-   ],
-   "id": "ad16251433d93b49",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## Extract",
-   "id": "973e0774ef72f46"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "from collections import Counter\n",
-    "\n",
-    "# Liste des colonnes en double\n",
-    "col_counts = Counter(df_traffic_pivoted.columns)\n",
-    "duplicate_cols = [col for col, count in col_counts.items() if count > 1]\n",
-    "\n",
-    "print(\"Colonnes dupliquées :\", duplicate_cols)\n"
-   ],
-   "id": "c67fce5edffdd474",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": "df_traffic_pivoted = df_traffic_pivoted.loc[:, ~df_traffic_pivoted.columns.duplicated()]\n",
-   "id": "7f1085cb636fca55",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "df_traffic_pivoted.to_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/traffic_cleaned_pivoted.parquet',\n",
-    "                              index=False)"
-   ],
-   "id": "c9d8fc584837b7cb",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": "df_traffic_pivoted.shape",
-   "id": "bc51ff55f46b1d09",
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Merge final",
-   "id": "9971bbc11c27dbdf"
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-09T19:44:09.673695Z",
-     "start_time": "2025-07-09T19:44:09.463956Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "df_traffic = pd.read_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/traffic_cleaned_pivoted.parquet')\n",
-    "df_meteo = pd.read_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/meteo_cleaned_pivoted.parquet')\n",
-    "df_pollutants = pd.read_parquet(\n",
-    "    '/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/pollutants_cleaned_pivoted.parquet')\n",
-    "\n",
-    "df_traffic = df_traffic.sort_values(by=['Timestamp'])\n",
-    "df_meteo = df_meteo.sort_values(by=['Timestamp'])\n",
-    "df_pollutants = df_pollutants.sort_values(by=['Timestamp'])\n",
-    "\n",
-    "# Merge on the nearest time values\n",
-    "df_merged = pd.merge_asof(df_traffic,\n",
-    "                          df_meteo,\n",
-    "                          left_on='Timestamp',\n",
-    "                          right_on='Timestamp',\n",
-    "                          direction='nearest')\n",
-    "\n",
-    "df_merged = pd.merge_asof(df_merged,\n",
-    "                          df_pollutants,\n",
-    "                          left_on='Timestamp',\n",
-    "                          right_on='Timestamp',\n",
-    "                          direction='nearest')\n",
-    "\n",
-    "df_merged = df_merged.sort_values(by=['Timestamp'])"
-   ],
-   "id": "ed106c330d7fe155",
-   "outputs": [],
-   "execution_count": 2
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# Upload to S3",
-   "id": "72d1e27b43e8f51"
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-09T19:44:54.064492Z",
-     "start_time": "2025-07-09T19:44:53.921133Z"
-    }
-   },
-   "cell_type": "code",
-   "source": "df_merged.to_parquet('2024_semester2_merged_v2.parquet', engine='pyarrow')\n",
-   "id": "c5f2ca648dc532e0",
-   "outputs": [],
-   "execution_count": 3
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "# CURIOSITY",
-   "id": "e83dca08dee6a881"
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "source": [
-    "meteo = pd.read_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/meteo_cleaned_pivoted.parquet')\n",
-    "pollutants = pd.read_parquet('/Users/a.lasnier/Desktop/dsl_ft_32/quality-air/data/pollutants_cleaned_pivoted.parquet')"
-   ],
-   "id": "346726ba01317db",
-   "outputs": [],
-   "execution_count": null
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

etl/traffic_rennes.py

@@ -1,63 +0,0 @@
-import requests
-import pandas as pd
-import logging
-
-# Configure le logger au niveau du module
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-def fetch_trafic_data():
-    """Récupère les données de trafic de Rennes Métropole"""
-    url = "https://data.rennesmetropole.fr/api/explore/v2.1/catalog/datasets/etat-du-trafic-en-temps-reel/records"
-    params = {
-        "select": "datetime,denomination,averagevehiclespeed,traveltime,trafficstatus",
-        "where": "averagevehiclespeed > 0 and trafficstatus != 'unknown'",
-        "order_by": "datetime desc",
-        "limit": 100,
-        "timezone": "Europe/Paris"
-    }
-    try:
-        response = requests.get(url, params=params)
-        response.raise_for_status()
-        logger.info("✅ Données récupérées avec succès depuis l'API Rennes Métropole.")
-        return response.json()["results"]
-    except Exception as e:
-        logger.error(f"❌ Erreur lors de la récupération des données : {e}")
-        raise
-
-def process_data(data):
-    """Nettoie les données sans les agréger"""
-    df = pd.DataFrame(data)
-    df["datetime"] = pd.to_datetime(df["datetime"])
-    df["averagevehiclespeed"] = pd.to_numeric(df["averagevehiclespeed"], errors="coerce")
-    df["traveltime"] = pd.to_numeric(df["traveltime"], errors="coerce")
-
-    latest_datetime = df["datetime"].max()
-    df_latest = df[df["datetime"] == latest_datetime]
-
-    agg_df = (
-        df_latest.groupby(["denomination", "datetime"], as_index=False)
-        .agg({
-            "averagevehiclespeed": "mean",
-            "traveltime": "mean",
-            "trafficstatus": "first"
-        })
-        .sort_values(by="trafficstatus", ascending=False)
-        .reset_index(drop=True)  # <-- reset index ici
-    )
-    logger.info(f"✅ Données de {latest_datetime} traitées avec succès. {len(agg_df)} lignes.")
-    return agg_df, latest_datetime
-
-def main():
-    try:
-        data = fetch_trafic_data()
-        agg_df, latest_datetime = process_data(data)
-        logger.info("Aperçu des données traitées :")
-        logger.info(agg_df.head().to_string(index=False))
-        return agg_df, latest_datetime
-    except Exception as e:
-        logger.error(f"❌ Échec du pipeline : {e}")
-        return None, None
-
-if __name__ == "__main__":
-    main()

requirements.txt

@@ -1,3 +1,8 @@
 pandas
 pytest
-requests
+requests
+apache-airflow-providers-postgres
+apache-airflow-providers-amazon
+scikit-learn
+psycopg[binary]
+python-dotenv