diff --git "a/notebooks/01_data_exploration.ipynb" "b/notebooks/01_data_exploration.ipynb"
new file mode 100644--- /dev/null
+++ "b/notebooks/01_data_exploration.ipynb"
@@ -0,0 +1,1055 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# DVF Data Exploration & Analysis\n",
+ "\n",
+ "French property price analysis using DVF (Demandes de Valeurs Foncières) geolocalized data.\n",
+ "\n",
+ "**Pipeline**: Download → Clean → Compute Price/m² → Aggregate → Visualize\n",
+ "\n",
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 0. Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Setup complete\n"
+ ]
+ }
+ ],
+ "source": [
+ "import sys\n",
+ "sys.path.insert(0, '..')\n",
+ "\n",
+ "import polars as pl\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns\n",
+ "import logging\n",
+ "\n",
+ "logging.basicConfig(level=logging.INFO)\n",
+ "\n",
+ "from src.config import (\n",
+ " RAW_DIR, PROCESSED_DIR, AGGREGATED_DIR,\n",
+ " DVF_YEARS, TOP_10_CITIES, REGION_NAMES,\n",
+ " DEPT_TO_REGION, TYPE_LOCAL_SHORT, NO_DVF_DEPARTMENTS,\n",
+ ")\n",
+ "\n",
+ "plt.rcParams['figure.figsize'] = (14, 6)\n",
+ "plt.rcParams['figure.dpi'] = 100\n",
+ "sns.set_theme(style='whitegrid')\n",
+ "\n",
+ "print('Setup complete')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 1. Download DVF Data\n",
+ "\n",
+ "Download the geolocalized DVF data for 2020-2024 (~500MB/year compressed)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:src.downloader:Cached: dvf_2020.csv.gz\n",
+ "INFO:src.downloader:Already decompressed: dvf_2020.csv\n",
+ "INFO:src.downloader:Cached: dvf_2021.csv.gz\n",
+ "INFO:src.downloader:Already decompressed: dvf_2021.csv\n",
+ "INFO:src.downloader:Cached: dvf_2022.csv.gz\n",
+ "INFO:src.downloader:Already decompressed: dvf_2022.csv\n",
+ "INFO:src.downloader:Cached: dvf_2023.csv.gz\n",
+ "INFO:src.downloader:Already decompressed: dvf_2023.csv\n",
+ "INFO:src.downloader:Cached: dvf_2024.csv.gz\n",
+ "INFO:src.downloader:Already decompressed: dvf_2024.csv\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Downloaded 5 files:\n",
+ " dvf_2020.csv: 358.2 MB\n",
+ " dvf_2021.csv: 814.5 MB\n",
+ " dvf_2022.csv: 815.2 MB\n",
+ " dvf_2023.csv: 662.1 MB\n",
+ " dvf_2024.csv: 605.8 MB\n"
+ ]
+ }
+ ],
+ "source": [
+ "from src.downloader import download_all\n",
+ "\n",
+ "csv_paths = download_all()\n",
+ "print(f'\\nDownloaded {len(csv_paths)} files:')\n",
+ "for p in csv_paths:\n",
+ " size_mb = p.stat().st_size / 1e6\n",
+ " print(f' {p.name}: {size_mb:.1f} MB')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 2. Load & Explore Raw Data\n",
+ "\n",
+ "Load one year first to understand the data structure."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Exploring: dvf_2024.csv\n",
+ "\n",
+ "Shape: (100000, 16)\n",
+ "\n",
+ "Columns (16):\n",
+ " id_mutation String nulls: 0 (0.0%)\n",
+ " date_mutation String nulls: 0 (0.0%)\n",
+ " nature_mutation String nulls: 0 (0.0%)\n",
+ " valeur_fonciere Float64 nulls: 691 (0.7%)\n",
+ " code_postal String nulls: 183 (0.2%)\n",
+ " code_commune String nulls: 0 (0.0%)\n",
+ " nom_commune String nulls: 0 (0.0%)\n",
+ " code_departement String nulls: 0 (0.0%)\n",
+ " id_parcelle String nulls: 0 (0.0%)\n",
+ " code_type_local String nulls: 50,152 (50.2%)\n",
+ " type_local String nulls: 50,152 (50.2%)\n",
+ " surface_reelle_bati Int64 nulls: 72,828 (72.8%)\n",
+ " nombre_pieces_principales Int64 nulls: 50,266 (50.3%)\n",
+ " nombre_lots Int64 nulls: 0 (0.0%)\n",
+ " longitude Float64 nulls: 726 (0.7%)\n",
+ " latitude Float64 nulls: 726 (0.7%)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from src.cleaner import load_raw_csv\n",
+ "\n",
+ "# Load latest year for exploration\n",
+ "latest = sorted(csv_paths)[-1]\n",
+ "print(f'Exploring: {latest.name}')\n",
+ "\n",
+ "df_sample = load_raw_csv(latest).head(100_000).collect()\n",
+ "print(f'\\nShape: {df_sample.shape}')\n",
+ "print(f'\\nColumns ({len(df_sample.columns)}):')\n",
+ "for col in df_sample.columns:\n",
+ " dtype = df_sample[col].dtype\n",
+ " nulls = df_sample[col].null_count()\n",
+ " pct = nulls / len(df_sample) * 100\n",
+ " print(f' {col:30s} {str(dtype):15s} nulls: {nulls:,} ({pct:.1f}%)')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
shape: (5, 16)| id_mutation | date_mutation | nature_mutation | valeur_fonciere | code_postal | code_commune | nom_commune | code_departement | id_parcelle | code_type_local | type_local | surface_reelle_bati | nombre_pieces_principales | nombre_lots | longitude | latitude |
|---|
| str | str | str | f64 | str | str | str | str | str | str | str | i64 | i64 | i64 | f64 | f64 |
| "2024-1" | "2024-01-02" | "Vente" | 346.5 | "01230" | "01076" | "Chaley" | "01" | "010760000B0514" | null | null | null | null | 0 | 5.530952 | 45.952439 |
| "2024-2" | "2024-01-03" | "Vente" | 10000.0 | "01170" | "01103" | "Chevry" | "01" | "011030000B1782" | null | null | null | null | 0 | 6.043339 | 46.282256 |
| "2024-3" | "2024-01-08" | "Vente" | 249000.0 | "01290" | "01203" | "Laiz" | "01" | "012030000C1065" | null | null | null | null | 0 | 4.911143 | 46.247235 |
| "2024-4" | "2024-01-03" | "Vente" | 329500.0 | "01170" | "01173" | "Gex" | "01" | "01173000AI0551" | "2" | "Appartement" | 89 | 4 | 2 | 6.058695 | 46.332212 |
| "2024-4" | "2024-01-03" | "Vente" | 329500.0 | "01170" | "01173" | "Gex" | "01" | "01173000AI0551" | "3" | "Dépendance" | null | 0 | 1 | 6.058695 | 46.332212 |
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Distribution of valeur_fonciere (raw)\n",
+ "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
+ "\n",
+ "vals = df_sample.filter(\n",
+ " pl.col('valeur_fonciere').is_not_null() & (pl.col('valeur_fonciere') > 0)\n",
+ ")['valeur_fonciere'].to_list()\n",
+ "\n",
+ "axes[0].hist(vals, bins=100, edgecolor='black', alpha=0.7)\n",
+ "axes[0].set_title('Raw valeur_fonciere distribution')\n",
+ "axes[0].set_xlabel('Price (EUR)')\n",
+ "\n",
+ "# Log scale\n",
+ "axes[1].hist(vals, bins=100, edgecolor='black', alpha=0.7, log=True)\n",
+ "axes[1].set_title('Raw valeur_fonciere (log scale)')\n",
+ "axes[1].set_xlabel('Price (EUR)')\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 3. Run Cleaning Pipeline\n",
+ "\n",
+ "Apply the full cleaning pipeline on all years."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Loading all years...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/tmp/ipykernel_34103/3610840135.py:6: DeprecationWarning: `pl.count()` is deprecated. Please use `pl.len()` instead.\n",
+ "(Deprecated in version 0.20.5)\n",
+ " n_raw = lf_raw.select(pl.count()).collect().item()\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Raw rows: 18,715,662\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/tmp/ipykernel_34103/3610840135.py:11: DeprecationWarning: `pl.count()` is deprecated. Please use `pl.len()` instead.\n",
+ "(Deprecated in version 0.20.5)\n",
+ " n_sales = lf_sales.select(pl.count()).collect().item()\n",
+ "INFO:src.cleaner:Loading 5 raw files...\n",
+ "INFO:src.cleaner:Filtering sales...\n",
+ "INFO:src.cleaner:Deduplicating mutations...\n",
+ "INFO:src.cleaner:Adding derived columns...\n",
+ "INFO:src.cleaner:Removing outliers...\n",
+ "INFO:src.cleaner:Normalizing commune codes...\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "After filtering to sales: 6,183,929 (33.0%)\n",
+ "\n",
+ "Running full cleaning pipeline...\n",
+ "\n",
+ "Final clean dataset: 4,410,655 transactions\n",
+ "Columns: ['id_mutation', 'date_mutation', 'nature_mutation', 'valeur_fonciere', 'code_postal', 'code_commune', 'nom_commune', 'code_departement', 'id_parcelle', 'type_local', 'surface_reelle_bati', 'nombre_pieces_principales', 'nombre_lots', 'longitude', 'latitude', 'prix_m2', 'code_section', 'code_region', 'year', 'code_commune_city', 'code_commune_original']\n"
+ ]
+ }
+ ],
+ "source": [
+ "from src.cleaner import clean, load_multiple_csvs, filter_sales\n",
+ "\n",
+ "# Load all years\n",
+ "print('Loading all years...')\n",
+ "lf_raw = load_multiple_csvs(csv_paths)\n",
+ "n_raw = lf_raw.select(pl.count()).collect().item()\n",
+ "print(f'Raw rows: {n_raw:,}')\n",
+ "\n",
+ "# After filtering sales\n",
+ "lf_sales = filter_sales(lf_raw)\n",
+ "n_sales = lf_sales.select(pl.count()).collect().item()\n",
+ "print(f'After filtering to sales: {n_sales:,} ({n_sales/n_raw*100:.1f}%)')\n",
+ "\n",
+ "# Full pipeline\n",
+ "print('\\nRunning full cleaning pipeline...')\n",
+ "lf_clean = clean(csv_paths)\n",
+ "df_clean = lf_clean.collect()\n",
+ "print(f'\\nFinal clean dataset: {len(df_clean):,} transactions')\n",
+ "print(f'Columns: {df_clean.columns}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Saved: dvf_clean.parquet (227.1 MB)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Save cleaned data for reuse\n",
+ "PROCESSED_DIR.mkdir(parents=True, exist_ok=True)\n",
+ "parquet_path = PROCESSED_DIR / 'dvf_clean.parquet'\n",
+ "df_clean.write_parquet(parquet_path)\n",
+ "size_mb = parquet_path.stat().st_size / 1e6\n",
+ "print(f'Saved: {parquet_path.name} ({size_mb:.1f} MB)')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
shape: (5, 21)| id_mutation | date_mutation | nature_mutation | valeur_fonciere | code_postal | code_commune | nom_commune | code_departement | id_parcelle | type_local | surface_reelle_bati | nombre_pieces_principales | nombre_lots | longitude | latitude | prix_m2 | code_section | code_region | year | code_commune_city | code_commune_original |
|---|
| str | str | str | f64 | str | str | str | str | str | str | i64 | i64 | i64 | f64 | f64 | f64 | str | str | str | str | str |
| "2023-418148" | "2023-12-19" | "Vente" | 171000.0 | "33170" | "33192" | "Gradignan" | "33" | "33192000BH0158" | "Appartement" | 44 | 2 | 1 | -0.600309 | 44.785246 | 3886.363636 | "33192000BH" | "75" | "2023" | "33192" | "33192" |
| "2024-17433" | "2024-07-15" | "Vente" | 100000.0 | "02290" | "02326" | "Fontenoy" | "02" | "02326000AH0046" | "Maison" | 97 | 4 | 0 | 3.205291 | 49.408683 | 1030.927835 | "02326000AH" | "32" | "2024" | "02326" | "02326" |
| "2021-504860" | "2021-02-18" | "Vente" | 254500.0 | "33700" | "33281" | "Mérignac" | "33" | "33281000BL0757" | "Maison" | 56 | 3 | 0 | -0.650228 | 44.856402 | 4544.642857 | "33281000BL" | "75" | "2021" | "33281" | "33281" |
| "2022-882228" | "2022-06-24" | "Vente" | 407000.0 | "54136" | "54090" | "Bouxières-aux-Dames" | "54" | "540900000B0683" | "Appartement" | 207 | 4 | 4 | 6.170229 | 48.750534 | 1966.183575 | "540900000B" | "44" | "2022" | "54090" | "54090" |
| "2020-246142" | "2020-11-25" | "Vente" | 333500.0 | "33000" | "33063" | "Bordeaux" | "33" | "33063000KE0074" | "Appartement" | 60 | 2 | 1 | -0.58141 | 44.838453 | 5558.333333 | "33063000KE" | "75" | "2020" | "33063" | "33063" |