{ "cells": [ { "cell_type": "markdown", "id": "2936bcdd", "metadata": {}, "source": [ "## Analyse Exploratoire" ] }, { "cell_type": "markdown", "id": "d07f903e", "metadata": {}, "source": [ "### Import des modules" ] }, { "cell_type": "code", "execution_count": 1, "id": "675bc3c9", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import joblib\n", "import warnings\n", "warnings.filterwarnings(\"ignore\")" ] }, { "cell_type": "markdown", "id": "cce73eca", "metadata": {}, "source": [ "### Vision globale des données Train et Test" ] }, { "cell_type": "markdown", "id": "4abc770c", "metadata": {}, "source": [ "#### Chargement des données brutes" ] }, { "cell_type": "code", "execution_count": 2, "id": "caf19121", "metadata": {}, "outputs": [], "source": [ "app_test = pd.read_csv(\"./Donnees_Brutes/application_test.csv\", sep=',')\n", "app_train = pd.read_csv(\"./Donnees_Brutes/application_train.csv\", sep=',')\n" ] }, { "cell_type": "markdown", "id": "d97b4955", "metadata": {}, "source": [ "#### Découverte des données train et test" ] }, { "cell_type": "code", "execution_count": 3, "id": "7cd04f8c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Train ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_CURRTARGETNAME_CONTRACT_TYPECODE_GENDERFLAG_OWN_CARFLAG_OWN_REALTYCNT_CHILDRENAMT_INCOME_TOTALAMT_CREDITAMT_ANNUITY...FLAG_DOCUMENT_18FLAG_DOCUMENT_19FLAG_DOCUMENT_20FLAG_DOCUMENT_21AMT_REQ_CREDIT_BUREAU_HOURAMT_REQ_CREDIT_BUREAU_DAYAMT_REQ_CREDIT_BUREAU_WEEKAMT_REQ_CREDIT_BUREAU_MONAMT_REQ_CREDIT_BUREAU_QRTAMT_REQ_CREDIT_BUREAU_YEAR
01000021Cash loansMNY0202500.0406597.524700.5...00000.00.00.00.00.01.0
11000030Cash loansFNN0270000.01293502.535698.5...00000.00.00.00.00.00.0
21000040Revolving loansMYY067500.0135000.06750.0...00000.00.00.00.00.00.0
31000060Cash loansFNY0135000.0312682.529686.5...0000NaNNaNNaNNaNNaNNaN
41000070Cash loansMNY0121500.0513000.021865.5...00000.00.00.00.00.00.0
\n", "

5 rows × 122 columns

\n", "
" ], "text/plain": [ " SK_ID_CURR TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR \\\n", "0 100002 1 Cash loans M N \n", "1 100003 0 Cash loans F N \n", "2 100004 0 Revolving loans M Y \n", "3 100006 0 Cash loans F N \n", "4 100007 0 Cash loans M N \n", "\n", " FLAG_OWN_REALTY CNT_CHILDREN AMT_INCOME_TOTAL AMT_CREDIT AMT_ANNUITY \\\n", "0 Y 0 202500.0 406597.5 24700.5 \n", "1 N 0 270000.0 1293502.5 35698.5 \n", "2 Y 0 67500.0 135000.0 6750.0 \n", "3 Y 0 135000.0 312682.5 29686.5 \n", "4 Y 0 121500.0 513000.0 21865.5 \n", "\n", " ... FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 \\\n", "0 ... 0 0 0 0 \n", "1 ... 0 0 0 0 \n", "2 ... 0 0 0 0 \n", "3 ... 0 0 0 0 \n", "4 ... 0 0 0 0 \n", "\n", " AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 NaN NaN \n", "4 0.0 0.0 \n", "\n", " AMT_REQ_CREDIT_BUREAU_WEEK AMT_REQ_CREDIT_BUREAU_MON \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 NaN NaN \n", "4 0.0 0.0 \n", "\n", " AMT_REQ_CREDIT_BUREAU_QRT AMT_REQ_CREDIT_BUREAU_YEAR \n", "0 0.0 1.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 NaN NaN \n", "4 0.0 0.0 \n", "\n", "[5 rows x 122 columns]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Test ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_CURRNAME_CONTRACT_TYPECODE_GENDERFLAG_OWN_CARFLAG_OWN_REALTYCNT_CHILDRENAMT_INCOME_TOTALAMT_CREDITAMT_ANNUITYAMT_GOODS_PRICE...FLAG_DOCUMENT_18FLAG_DOCUMENT_19FLAG_DOCUMENT_20FLAG_DOCUMENT_21AMT_REQ_CREDIT_BUREAU_HOURAMT_REQ_CREDIT_BUREAU_DAYAMT_REQ_CREDIT_BUREAU_WEEKAMT_REQ_CREDIT_BUREAU_MONAMT_REQ_CREDIT_BUREAU_QRTAMT_REQ_CREDIT_BUREAU_YEAR
0100001Cash loansFNY0135000.0568800.020560.5450000.0...00000.00.00.00.00.00.0
1100005Cash loansMNY099000.0222768.017370.0180000.0...00000.00.00.00.00.03.0
2100013Cash loansMYY0202500.0663264.069777.0630000.0...00000.00.00.00.01.04.0
3100028Cash loansFNY2315000.01575000.049018.51575000.0...00000.00.00.00.00.03.0
4100038Cash loansMYN1180000.0625500.032067.0625500.0...0000NaNNaNNaNNaNNaNNaN
\n", "

5 rows × 121 columns

\n", "
" ], "text/plain": [ " SK_ID_CURR NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR FLAG_OWN_REALTY \\\n", "0 100001 Cash loans F N Y \n", "1 100005 Cash loans M N Y \n", "2 100013 Cash loans M Y Y \n", "3 100028 Cash loans F N Y \n", "4 100038 Cash loans M Y N \n", "\n", " CNT_CHILDREN AMT_INCOME_TOTAL AMT_CREDIT AMT_ANNUITY AMT_GOODS_PRICE \\\n", "0 0 135000.0 568800.0 20560.5 450000.0 \n", "1 0 99000.0 222768.0 17370.0 180000.0 \n", "2 0 202500.0 663264.0 69777.0 630000.0 \n", "3 2 315000.0 1575000.0 49018.5 1575000.0 \n", "4 1 180000.0 625500.0 32067.0 625500.0 \n", "\n", " ... FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 \\\n", "0 ... 0 0 0 0 \n", "1 ... 0 0 0 0 \n", "2 ... 0 0 0 0 \n", "3 ... 0 0 0 0 \n", "4 ... 0 0 0 0 \n", "\n", " AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 NaN NaN \n", "\n", " AMT_REQ_CREDIT_BUREAU_WEEK AMT_REQ_CREDIT_BUREAU_MON \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 NaN NaN \n", "\n", " AMT_REQ_CREDIT_BUREAU_QRT AMT_REQ_CREDIT_BUREAU_YEAR \n", "0 0.0 0.0 \n", "1 0.0 3.0 \n", "2 1.0 4.0 \n", "3 0.0 3.0 \n", "4 NaN NaN \n", "\n", "[5 rows x 121 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Vue globale des données\n", "pd.set_option(\"display.max_info_columns\", 200)\n", "jeu_donnee = [app_train, app_test]\n", "noms = [\"Train\", \"Test\"]\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Résumé {nom} ---\")\n", " display(df.head())\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "7e32a932", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Train ---\n" ] }, { "data": { "text/plain": [ "(307511, 122)" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Test ---\n" ] }, { "data": { "text/plain": [ "(48744, 121)" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Dimensions des données\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Résumé {nom} ---\")\n", " display(df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "id": "aa309245", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Informations sur Train ---\n", "\n", "RangeIndex: 307511 entries, 0 to 307510\n", "Data columns (total 122 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 307511 non-null int64 \n", " 1 TARGET 307511 non-null int64 \n", " 2 NAME_CONTRACT_TYPE 307511 non-null object \n", " 3 CODE_GENDER 307511 non-null object \n", " 4 FLAG_OWN_CAR 307511 non-null object \n", " 5 FLAG_OWN_REALTY 307511 non-null object \n", " 6 CNT_CHILDREN 307511 non-null int64 \n", " 7 AMT_INCOME_TOTAL 307511 non-null float64\n", " 8 AMT_CREDIT 307511 non-null float64\n", " 9 AMT_ANNUITY 307499 non-null float64\n", " 10 AMT_GOODS_PRICE 307233 non-null float64\n", " 11 NAME_TYPE_SUITE 306219 non-null object \n", " 12 NAME_INCOME_TYPE 307511 non-null object \n", " 13 NAME_EDUCATION_TYPE 307511 non-null object \n", " 14 NAME_FAMILY_STATUS 307511 non-null object \n", " 15 NAME_HOUSING_TYPE 307511 non-null object \n", " 16 REGION_POPULATION_RELATIVE 307511 non-null float64\n", " 17 DAYS_BIRTH 307511 non-null int64 \n", " 18 DAYS_EMPLOYED 307511 non-null int64 \n", " 19 DAYS_REGISTRATION 307511 non-null float64\n", " 20 DAYS_ID_PUBLISH 307511 non-null int64 \n", " 21 OWN_CAR_AGE 104582 non-null float64\n", " 22 FLAG_MOBIL 307511 non-null int64 \n", " 23 FLAG_EMP_PHONE 307511 non-null int64 \n", " 24 FLAG_WORK_PHONE 307511 non-null int64 \n", " 25 FLAG_CONT_MOBILE 307511 non-null int64 \n", " 26 FLAG_PHONE 307511 non-null int64 \n", " 27 FLAG_EMAIL 307511 non-null int64 \n", " 28 OCCUPATION_TYPE 211120 non-null object \n", " 29 CNT_FAM_MEMBERS 307509 non-null float64\n", " 30 REGION_RATING_CLIENT 307511 non-null int64 \n", " 31 REGION_RATING_CLIENT_W_CITY 307511 non-null int64 \n", " 32 WEEKDAY_APPR_PROCESS_START 307511 non-null object \n", " 33 HOUR_APPR_PROCESS_START 307511 non-null int64 \n", " 34 REG_REGION_NOT_LIVE_REGION 307511 non-null int64 \n", " 35 REG_REGION_NOT_WORK_REGION 307511 non-null int64 \n", " 36 LIVE_REGION_NOT_WORK_REGION 307511 non-null int64 \n", " 37 REG_CITY_NOT_LIVE_CITY 307511 non-null int64 \n", " 38 REG_CITY_NOT_WORK_CITY 307511 non-null int64 \n", " 39 LIVE_CITY_NOT_WORK_CITY 307511 non-null int64 \n", " 40 ORGANIZATION_TYPE 307511 non-null object \n", " 41 EXT_SOURCE_1 134133 non-null float64\n", " 42 EXT_SOURCE_2 306851 non-null float64\n", " 43 EXT_SOURCE_3 246546 non-null float64\n", " 44 APARTMENTS_AVG 151450 non-null float64\n", " 45 BASEMENTAREA_AVG 127568 non-null float64\n", " 46 YEARS_BEGINEXPLUATATION_AVG 157504 non-null float64\n", " 47 YEARS_BUILD_AVG 103023 non-null float64\n", " 48 COMMONAREA_AVG 92646 non-null float64\n", " 49 ELEVATORS_AVG 143620 non-null float64\n", " 50 ENTRANCES_AVG 152683 non-null float64\n", " 51 FLOORSMAX_AVG 154491 non-null float64\n", " 52 FLOORSMIN_AVG 98869 non-null float64\n", " 53 LANDAREA_AVG 124921 non-null float64\n", " 54 LIVINGAPARTMENTS_AVG 97312 non-null float64\n", " 55 LIVINGAREA_AVG 153161 non-null float64\n", " 56 NONLIVINGAPARTMENTS_AVG 93997 non-null float64\n", " 57 NONLIVINGAREA_AVG 137829 non-null float64\n", " 58 APARTMENTS_MODE 151450 non-null float64\n", " 59 BASEMENTAREA_MODE 127568 non-null float64\n", " 60 YEARS_BEGINEXPLUATATION_MODE 157504 non-null float64\n", " 61 YEARS_BUILD_MODE 103023 non-null float64\n", " 62 COMMONAREA_MODE 92646 non-null float64\n", " 63 ELEVATORS_MODE 143620 non-null float64\n", " 64 ENTRANCES_MODE 152683 non-null float64\n", " 65 FLOORSMAX_MODE 154491 non-null float64\n", " 66 FLOORSMIN_MODE 98869 non-null float64\n", " 67 LANDAREA_MODE 124921 non-null float64\n", " 68 LIVINGAPARTMENTS_MODE 97312 non-null float64\n", " 69 LIVINGAREA_MODE 153161 non-null float64\n", " 70 NONLIVINGAPARTMENTS_MODE 93997 non-null float64\n", " 71 NONLIVINGAREA_MODE 137829 non-null float64\n", " 72 APARTMENTS_MEDI 151450 non-null float64\n", " 73 BASEMENTAREA_MEDI 127568 non-null float64\n", " 74 YEARS_BEGINEXPLUATATION_MEDI 157504 non-null float64\n", " 75 YEARS_BUILD_MEDI 103023 non-null float64\n", " 76 COMMONAREA_MEDI 92646 non-null float64\n", " 77 ELEVATORS_MEDI 143620 non-null float64\n", " 78 ENTRANCES_MEDI 152683 non-null float64\n", " 79 FLOORSMAX_MEDI 154491 non-null float64\n", " 80 FLOORSMIN_MEDI 98869 non-null float64\n", " 81 LANDAREA_MEDI 124921 non-null float64\n", " 82 LIVINGAPARTMENTS_MEDI 97312 non-null float64\n", " 83 LIVINGAREA_MEDI 153161 non-null float64\n", " 84 NONLIVINGAPARTMENTS_MEDI 93997 non-null float64\n", " 85 NONLIVINGAREA_MEDI 137829 non-null float64\n", " 86 FONDKAPREMONT_MODE 97216 non-null object \n", " 87 HOUSETYPE_MODE 153214 non-null object \n", " 88 TOTALAREA_MODE 159080 non-null float64\n", " 89 WALLSMATERIAL_MODE 151170 non-null object \n", " 90 EMERGENCYSTATE_MODE 161756 non-null object \n", " 91 OBS_30_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 92 DEF_30_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 93 OBS_60_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 94 DEF_60_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 95 DAYS_LAST_PHONE_CHANGE 307510 non-null float64\n", " 96 FLAG_DOCUMENT_2 307511 non-null int64 \n", " 97 FLAG_DOCUMENT_3 307511 non-null int64 \n", " 98 FLAG_DOCUMENT_4 307511 non-null int64 \n", " 99 FLAG_DOCUMENT_5 307511 non-null int64 \n", " 100 FLAG_DOCUMENT_6 307511 non-null int64 \n", " 101 FLAG_DOCUMENT_7 307511 non-null int64 \n", " 102 FLAG_DOCUMENT_8 307511 non-null int64 \n", " 103 FLAG_DOCUMENT_9 307511 non-null int64 \n", " 104 FLAG_DOCUMENT_10 307511 non-null int64 \n", " 105 FLAG_DOCUMENT_11 307511 non-null int64 \n", " 106 FLAG_DOCUMENT_12 307511 non-null int64 \n", " 107 FLAG_DOCUMENT_13 307511 non-null int64 \n", " 108 FLAG_DOCUMENT_14 307511 non-null int64 \n", " 109 FLAG_DOCUMENT_15 307511 non-null int64 \n", " 110 FLAG_DOCUMENT_16 307511 non-null int64 \n", " 111 FLAG_DOCUMENT_17 307511 non-null int64 \n", " 112 FLAG_DOCUMENT_18 307511 non-null int64 \n", " 113 FLAG_DOCUMENT_19 307511 non-null int64 \n", " 114 FLAG_DOCUMENT_20 307511 non-null int64 \n", " 115 FLAG_DOCUMENT_21 307511 non-null int64 \n", " 116 AMT_REQ_CREDIT_BUREAU_HOUR 265992 non-null float64\n", " 117 AMT_REQ_CREDIT_BUREAU_DAY 265992 non-null float64\n", " 118 AMT_REQ_CREDIT_BUREAU_WEEK 265992 non-null float64\n", " 119 AMT_REQ_CREDIT_BUREAU_MON 265992 non-null float64\n", " 120 AMT_REQ_CREDIT_BUREAU_QRT 265992 non-null float64\n", " 121 AMT_REQ_CREDIT_BUREAU_YEAR 265992 non-null float64\n", "dtypes: float64(65), int64(41), object(16)\n", "memory usage: 286.2+ MB\n" ] }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Informations sur Test ---\n", "\n", "RangeIndex: 48744 entries, 0 to 48743\n", "Data columns (total 121 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 48744 non-null int64 \n", " 1 NAME_CONTRACT_TYPE 48744 non-null object \n", " 2 CODE_GENDER 48744 non-null object \n", " 3 FLAG_OWN_CAR 48744 non-null object \n", " 4 FLAG_OWN_REALTY 48744 non-null object \n", " 5 CNT_CHILDREN 48744 non-null int64 \n", " 6 AMT_INCOME_TOTAL 48744 non-null float64\n", " 7 AMT_CREDIT 48744 non-null float64\n", " 8 AMT_ANNUITY 48720 non-null float64\n", " 9 AMT_GOODS_PRICE 48744 non-null float64\n", " 10 NAME_TYPE_SUITE 47833 non-null object \n", " 11 NAME_INCOME_TYPE 48744 non-null object \n", " 12 NAME_EDUCATION_TYPE 48744 non-null object \n", " 13 NAME_FAMILY_STATUS 48744 non-null object \n", " 14 NAME_HOUSING_TYPE 48744 non-null object \n", " 15 REGION_POPULATION_RELATIVE 48744 non-null float64\n", " 16 DAYS_BIRTH 48744 non-null int64 \n", " 17 DAYS_EMPLOYED 48744 non-null int64 \n", " 18 DAYS_REGISTRATION 48744 non-null float64\n", " 19 DAYS_ID_PUBLISH 48744 non-null int64 \n", " 20 OWN_CAR_AGE 16432 non-null float64\n", " 21 FLAG_MOBIL 48744 non-null int64 \n", " 22 FLAG_EMP_PHONE 48744 non-null int64 \n", " 23 FLAG_WORK_PHONE 48744 non-null int64 \n", " 24 FLAG_CONT_MOBILE 48744 non-null int64 \n", " 25 FLAG_PHONE 48744 non-null int64 \n", " 26 FLAG_EMAIL 48744 non-null int64 \n", " 27 OCCUPATION_TYPE 33139 non-null object \n", " 28 CNT_FAM_MEMBERS 48744 non-null float64\n", " 29 REGION_RATING_CLIENT 48744 non-null int64 \n", " 30 REGION_RATING_CLIENT_W_CITY 48744 non-null int64 \n", " 31 WEEKDAY_APPR_PROCESS_START 48744 non-null object \n", " 32 HOUR_APPR_PROCESS_START 48744 non-null int64 \n", " 33 REG_REGION_NOT_LIVE_REGION 48744 non-null int64 \n", " 34 REG_REGION_NOT_WORK_REGION 48744 non-null int64 \n", " 35 LIVE_REGION_NOT_WORK_REGION 48744 non-null int64 \n", " 36 REG_CITY_NOT_LIVE_CITY 48744 non-null int64 \n", " 37 REG_CITY_NOT_WORK_CITY 48744 non-null int64 \n", " 38 LIVE_CITY_NOT_WORK_CITY 48744 non-null int64 \n", " 39 ORGANIZATION_TYPE 48744 non-null object \n", " 40 EXT_SOURCE_1 28212 non-null float64\n", " 41 EXT_SOURCE_2 48736 non-null float64\n", " 42 EXT_SOURCE_3 40076 non-null float64\n", " 43 APARTMENTS_AVG 24857 non-null float64\n", " 44 BASEMENTAREA_AVG 21103 non-null float64\n", " 45 YEARS_BEGINEXPLUATATION_AVG 25888 non-null float64\n", " 46 YEARS_BUILD_AVG 16926 non-null float64\n", " 47 COMMONAREA_AVG 15249 non-null float64\n", " 48 ELEVATORS_AVG 23555 non-null float64\n", " 49 ENTRANCES_AVG 25165 non-null float64\n", " 50 FLOORSMAX_AVG 25423 non-null float64\n", " 51 FLOORSMIN_AVG 16278 non-null float64\n", " 52 LANDAREA_AVG 20490 non-null float64\n", " 53 LIVINGAPARTMENTS_AVG 15964 non-null float64\n", " 54 LIVINGAREA_AVG 25192 non-null float64\n", " 55 NONLIVINGAPARTMENTS_AVG 15397 non-null float64\n", " 56 NONLIVINGAREA_AVG 22660 non-null float64\n", " 57 APARTMENTS_MODE 24857 non-null float64\n", " 58 BASEMENTAREA_MODE 21103 non-null float64\n", " 59 YEARS_BEGINEXPLUATATION_MODE 25888 non-null float64\n", " 60 YEARS_BUILD_MODE 16926 non-null float64\n", " 61 COMMONAREA_MODE 15249 non-null float64\n", " 62 ELEVATORS_MODE 23555 non-null float64\n", " 63 ENTRANCES_MODE 25165 non-null float64\n", " 64 FLOORSMAX_MODE 25423 non-null float64\n", " 65 FLOORSMIN_MODE 16278 non-null float64\n", " 66 LANDAREA_MODE 20490 non-null float64\n", " 67 LIVINGAPARTMENTS_MODE 15964 non-null float64\n", " 68 LIVINGAREA_MODE 25192 non-null float64\n", " 69 NONLIVINGAPARTMENTS_MODE 15397 non-null float64\n", " 70 NONLIVINGAREA_MODE 22660 non-null float64\n", " 71 APARTMENTS_MEDI 24857 non-null float64\n", " 72 BASEMENTAREA_MEDI 21103 non-null float64\n", " 73 YEARS_BEGINEXPLUATATION_MEDI 25888 non-null float64\n", " 74 YEARS_BUILD_MEDI 16926 non-null float64\n", " 75 COMMONAREA_MEDI 15249 non-null float64\n", " 76 ELEVATORS_MEDI 23555 non-null float64\n", " 77 ENTRANCES_MEDI 25165 non-null float64\n", " 78 FLOORSMAX_MEDI 25423 non-null float64\n", " 79 FLOORSMIN_MEDI 16278 non-null float64\n", " 80 LANDAREA_MEDI 20490 non-null float64\n", " 81 LIVINGAPARTMENTS_MEDI 15964 non-null float64\n", " 82 LIVINGAREA_MEDI 25192 non-null float64\n", " 83 NONLIVINGAPARTMENTS_MEDI 15397 non-null float64\n", " 84 NONLIVINGAREA_MEDI 22660 non-null float64\n", " 85 FONDKAPREMONT_MODE 15947 non-null object \n", " 86 HOUSETYPE_MODE 25125 non-null object \n", " 87 TOTALAREA_MODE 26120 non-null float64\n", " 88 WALLSMATERIAL_MODE 24851 non-null object \n", " 89 EMERGENCYSTATE_MODE 26535 non-null object \n", " 90 OBS_30_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 91 DEF_30_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 92 OBS_60_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 93 DEF_60_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 94 DAYS_LAST_PHONE_CHANGE 48744 non-null float64\n", " 95 FLAG_DOCUMENT_2 48744 non-null int64 \n", " 96 FLAG_DOCUMENT_3 48744 non-null int64 \n", " 97 FLAG_DOCUMENT_4 48744 non-null int64 \n", " 98 FLAG_DOCUMENT_5 48744 non-null int64 \n", " 99 FLAG_DOCUMENT_6 48744 non-null int64 \n", " 100 FLAG_DOCUMENT_7 48744 non-null int64 \n", " 101 FLAG_DOCUMENT_8 48744 non-null int64 \n", " 102 FLAG_DOCUMENT_9 48744 non-null int64 \n", " 103 FLAG_DOCUMENT_10 48744 non-null int64 \n", " 104 FLAG_DOCUMENT_11 48744 non-null int64 \n", " 105 FLAG_DOCUMENT_12 48744 non-null int64 \n", " 106 FLAG_DOCUMENT_13 48744 non-null int64 \n", " 107 FLAG_DOCUMENT_14 48744 non-null int64 \n", " 108 FLAG_DOCUMENT_15 48744 non-null int64 \n", " 109 FLAG_DOCUMENT_16 48744 non-null int64 \n", " 110 FLAG_DOCUMENT_17 48744 non-null int64 \n", " 111 FLAG_DOCUMENT_18 48744 non-null int64 \n", " 112 FLAG_DOCUMENT_19 48744 non-null int64 \n", " 113 FLAG_DOCUMENT_20 48744 non-null int64 \n", " 114 FLAG_DOCUMENT_21 48744 non-null int64 \n", " 115 AMT_REQ_CREDIT_BUREAU_HOUR 42695 non-null float64\n", " 116 AMT_REQ_CREDIT_BUREAU_DAY 42695 non-null float64\n", " 117 AMT_REQ_CREDIT_BUREAU_WEEK 42695 non-null float64\n", " 118 AMT_REQ_CREDIT_BUREAU_MON 42695 non-null float64\n", " 119 AMT_REQ_CREDIT_BUREAU_QRT 42695 non-null float64\n", " 120 AMT_REQ_CREDIT_BUREAU_YEAR 42695 non-null float64\n", "dtypes: float64(65), int64(40), object(16)\n", "memory usage: 45.0+ MB\n" ] }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Résumé types de données\n", "pd.set_option(\"display.max_info_columns\", 200)\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Informations sur {nom} ---\")\n", " display(df.info())\n", " " ] }, { "cell_type": "code", "execution_count": 6, "id": "21785fb5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DataFrame 0 - 0\n", "DataFrame 1 - 0\n" ] } ], "source": [ "# Vérification de doublons\n", "for i, df in enumerate(jeu_donnee, start=0):\n", " print(f\"DataFrame {i} - {df.duplicated().sum()}\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "5938dbc3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Statistiques descriptives - Train ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countmeanstdmin25%50%75%max
SK_ID_CURR307511.0278180.518577102790.175348100002.0189145.5278202.0367142.5456255.0
TARGET307511.00.0807290.2724190.00.00.00.01.0
CNT_CHILDREN307511.00.4170520.7221210.00.00.01.019.0
AMT_INCOME_TOTAL307511.0168797.919297237123.14627925650.0112500.0147150.0202500.0117000000.0
AMT_CREDIT307511.0599025.999706402490.77699645000.0270000.0513531.0808650.04050000.0
...........................
AMT_REQ_CREDIT_BUREAU_DAY265992.00.0070000.1107570.00.00.00.09.0
AMT_REQ_CREDIT_BUREAU_WEEK265992.00.0343620.2046850.00.00.00.08.0
AMT_REQ_CREDIT_BUREAU_MON265992.00.2673950.9160020.00.00.00.027.0
AMT_REQ_CREDIT_BUREAU_QRT265992.00.2654740.7940560.00.00.00.0261.0
AMT_REQ_CREDIT_BUREAU_YEAR265992.01.8999741.8692950.00.01.03.025.0
\n", "

106 rows × 8 columns

\n", "
" ], "text/plain": [ " count mean std min \\\n", "SK_ID_CURR 307511.0 278180.518577 102790.175348 100002.0 \n", "TARGET 307511.0 0.080729 0.272419 0.0 \n", "CNT_CHILDREN 307511.0 0.417052 0.722121 0.0 \n", "AMT_INCOME_TOTAL 307511.0 168797.919297 237123.146279 25650.0 \n", "AMT_CREDIT 307511.0 599025.999706 402490.776996 45000.0 \n", "... ... ... ... ... \n", "AMT_REQ_CREDIT_BUREAU_DAY 265992.0 0.007000 0.110757 0.0 \n", "AMT_REQ_CREDIT_BUREAU_WEEK 265992.0 0.034362 0.204685 0.0 \n", "AMT_REQ_CREDIT_BUREAU_MON 265992.0 0.267395 0.916002 0.0 \n", "AMT_REQ_CREDIT_BUREAU_QRT 265992.0 0.265474 0.794056 0.0 \n", "AMT_REQ_CREDIT_BUREAU_YEAR 265992.0 1.899974 1.869295 0.0 \n", "\n", " 25% 50% 75% max \n", "SK_ID_CURR 189145.5 278202.0 367142.5 456255.0 \n", "TARGET 0.0 0.0 0.0 1.0 \n", "CNT_CHILDREN 0.0 0.0 1.0 19.0 \n", "AMT_INCOME_TOTAL 112500.0 147150.0 202500.0 117000000.0 \n", "AMT_CREDIT 270000.0 513531.0 808650.0 4050000.0 \n", "... ... ... ... ... \n", "AMT_REQ_CREDIT_BUREAU_DAY 0.0 0.0 0.0 9.0 \n", "AMT_REQ_CREDIT_BUREAU_WEEK 0.0 0.0 0.0 8.0 \n", "AMT_REQ_CREDIT_BUREAU_MON 0.0 0.0 0.0 27.0 \n", "AMT_REQ_CREDIT_BUREAU_QRT 0.0 0.0 0.0 261.0 \n", "AMT_REQ_CREDIT_BUREAU_YEAR 0.0 1.0 3.0 25.0 \n", "\n", "[106 rows x 8 columns]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Statistiques descriptives - Test ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countmeanstdmin25%50%75%max
SK_ID_CURR48744.0277796.676350103169.547296100001.0188557.75277549.0367555.5456250.0
CNT_CHILDREN48744.00.3970540.7090470.00.000.01.020.0
AMT_INCOME_TOTAL48744.0178431.805855101522.59153626941.5112500.00157500.0225000.04410000.0
AMT_CREDIT48744.0516740.435561365397.00417445000.0260640.00450000.0675000.02245500.0
AMT_ANNUITY48720.029426.24020916016.3683152295.017973.0026199.037390.5180576.0
...........................
AMT_REQ_CREDIT_BUREAU_DAY42695.00.0018030.0461320.00.000.00.02.0
AMT_REQ_CREDIT_BUREAU_WEEK42695.00.0027870.0540370.00.000.00.02.0
AMT_REQ_CREDIT_BUREAU_MON42695.00.0092990.1109240.00.000.00.06.0
AMT_REQ_CREDIT_BUREAU_QRT42695.00.5469020.6933050.00.000.01.07.0
AMT_REQ_CREDIT_BUREAU_YEAR42695.01.9837691.8388730.00.002.03.017.0
\n", "

105 rows × 8 columns

\n", "
" ], "text/plain": [ " count mean std min \\\n", "SK_ID_CURR 48744.0 277796.676350 103169.547296 100001.0 \n", "CNT_CHILDREN 48744.0 0.397054 0.709047 0.0 \n", "AMT_INCOME_TOTAL 48744.0 178431.805855 101522.591536 26941.5 \n", "AMT_CREDIT 48744.0 516740.435561 365397.004174 45000.0 \n", "AMT_ANNUITY 48720.0 29426.240209 16016.368315 2295.0 \n", "... ... ... ... ... \n", "AMT_REQ_CREDIT_BUREAU_DAY 42695.0 0.001803 0.046132 0.0 \n", "AMT_REQ_CREDIT_BUREAU_WEEK 42695.0 0.002787 0.054037 0.0 \n", "AMT_REQ_CREDIT_BUREAU_MON 42695.0 0.009299 0.110924 0.0 \n", "AMT_REQ_CREDIT_BUREAU_QRT 42695.0 0.546902 0.693305 0.0 \n", "AMT_REQ_CREDIT_BUREAU_YEAR 42695.0 1.983769 1.838873 0.0 \n", "\n", " 25% 50% 75% max \n", "SK_ID_CURR 188557.75 277549.0 367555.5 456250.0 \n", "CNT_CHILDREN 0.00 0.0 1.0 20.0 \n", "AMT_INCOME_TOTAL 112500.00 157500.0 225000.0 4410000.0 \n", "AMT_CREDIT 260640.00 450000.0 675000.0 2245500.0 \n", "AMT_ANNUITY 17973.00 26199.0 37390.5 180576.0 \n", "... ... ... ... ... \n", "AMT_REQ_CREDIT_BUREAU_DAY 0.00 0.0 0.0 2.0 \n", "AMT_REQ_CREDIT_BUREAU_WEEK 0.00 0.0 0.0 2.0 \n", "AMT_REQ_CREDIT_BUREAU_MON 0.00 0.0 0.0 6.0 \n", "AMT_REQ_CREDIT_BUREAU_QRT 0.00 0.0 1.0 7.0 \n", "AMT_REQ_CREDIT_BUREAU_YEAR 0.00 2.0 3.0 17.0 \n", "\n", "[105 rows x 8 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Statistiques descriptives (transposées pour une meilleure lecture)\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Statistiques descriptives - {nom} ---\")\n", " display(df.describe().T)\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "37ac734c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Valeurs manquantes BD Train ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colonnevaleurs_manquantespourcentage
76COMMONAREA_MEDI21486569.9
48COMMONAREA_AVG21486569.9
62COMMONAREA_MODE21486569.9
84NONLIVINGAPARTMENTS_MEDI21351469.4
70NONLIVINGAPARTMENTS_MODE21351469.4
56NONLIVINGAPARTMENTS_AVG21351469.4
86FONDKAPREMONT_MODE21029568.4
68LIVINGAPARTMENTS_MODE21019968.4
82LIVINGAPARTMENTS_MEDI21019968.4
54LIVINGAPARTMENTS_AVG21019968.4
66FLOORSMIN_MODE20864267.8
80FLOORSMIN_MEDI20864267.8
52FLOORSMIN_AVG20864267.8
61YEARS_BUILD_MODE20448866.5
75YEARS_BUILD_MEDI20448866.5
47YEARS_BUILD_AVG20448866.5
21OWN_CAR_AGE20292966.0
53LANDAREA_AVG18259059.4
81LANDAREA_MEDI18259059.4
67LANDAREA_MODE18259059.4
73BASEMENTAREA_MEDI17994358.5
45BASEMENTAREA_AVG17994358.5
59BASEMENTAREA_MODE17994358.5
41EXT_SOURCE_117337856.4
85NONLIVINGAREA_MEDI16968255.2
71NONLIVINGAREA_MODE16968255.2
57NONLIVINGAREA_AVG16968255.2
77ELEVATORS_MEDI16389153.3
63ELEVATORS_MODE16389153.3
49ELEVATORS_AVG16389153.3
89WALLSMATERIAL_MODE15634150.8
58APARTMENTS_MODE15606150.7
72APARTMENTS_MEDI15606150.7
44APARTMENTS_AVG15606150.7
64ENTRANCES_MODE15482850.3
50ENTRANCES_AVG15482850.3
78ENTRANCES_MEDI15482850.3
83LIVINGAREA_MEDI15435050.2
69LIVINGAREA_MODE15435050.2
55LIVINGAREA_AVG15435050.2
87HOUSETYPE_MODE15429750.2
79FLOORSMAX_MEDI15302049.8
51FLOORSMAX_AVG15302049.8
65FLOORSMAX_MODE15302049.8
46YEARS_BEGINEXPLUATATION_AVG15000748.8
74YEARS_BEGINEXPLUATATION_MEDI15000748.8
60YEARS_BEGINEXPLUATATION_MODE15000748.8
88TOTALAREA_MODE14843148.3
90EMERGENCYSTATE_MODE14575547.4
28OCCUPATION_TYPE9639131.3
43EXT_SOURCE_36096519.8
118AMT_REQ_CREDIT_BUREAU_WEEK4151913.5
117AMT_REQ_CREDIT_BUREAU_DAY4151913.5
119AMT_REQ_CREDIT_BUREAU_MON4151913.5
120AMT_REQ_CREDIT_BUREAU_QRT4151913.5
116AMT_REQ_CREDIT_BUREAU_HOUR4151913.5
121AMT_REQ_CREDIT_BUREAU_YEAR4151913.5
11NAME_TYPE_SUITE12920.4
92DEF_30_CNT_SOCIAL_CIRCLE10210.3
93OBS_60_CNT_SOCIAL_CIRCLE10210.3
94DEF_60_CNT_SOCIAL_CIRCLE10210.3
91OBS_30_CNT_SOCIAL_CIRCLE10210.3
42EXT_SOURCE_26600.2
10AMT_GOODS_PRICE2780.1
9AMT_ANNUITY120.0
29CNT_FAM_MEMBERS20.0
95DAYS_LAST_PHONE_CHANGE10.0
\n", "
" ], "text/plain": [ " colonne valeurs_manquantes pourcentage\n", "76 COMMONAREA_MEDI 214865 69.9\n", "48 COMMONAREA_AVG 214865 69.9\n", "62 COMMONAREA_MODE 214865 69.9\n", "84 NONLIVINGAPARTMENTS_MEDI 213514 69.4\n", "70 NONLIVINGAPARTMENTS_MODE 213514 69.4\n", "56 NONLIVINGAPARTMENTS_AVG 213514 69.4\n", "86 FONDKAPREMONT_MODE 210295 68.4\n", "68 LIVINGAPARTMENTS_MODE 210199 68.4\n", "82 LIVINGAPARTMENTS_MEDI 210199 68.4\n", "54 LIVINGAPARTMENTS_AVG 210199 68.4\n", "66 FLOORSMIN_MODE 208642 67.8\n", "80 FLOORSMIN_MEDI 208642 67.8\n", "52 FLOORSMIN_AVG 208642 67.8\n", "61 YEARS_BUILD_MODE 204488 66.5\n", "75 YEARS_BUILD_MEDI 204488 66.5\n", "47 YEARS_BUILD_AVG 204488 66.5\n", "21 OWN_CAR_AGE 202929 66.0\n", "53 LANDAREA_AVG 182590 59.4\n", "81 LANDAREA_MEDI 182590 59.4\n", "67 LANDAREA_MODE 182590 59.4\n", "73 BASEMENTAREA_MEDI 179943 58.5\n", "45 BASEMENTAREA_AVG 179943 58.5\n", "59 BASEMENTAREA_MODE 179943 58.5\n", "41 EXT_SOURCE_1 173378 56.4\n", "85 NONLIVINGAREA_MEDI 169682 55.2\n", "71 NONLIVINGAREA_MODE 169682 55.2\n", "57 NONLIVINGAREA_AVG 169682 55.2\n", "77 ELEVATORS_MEDI 163891 53.3\n", "63 ELEVATORS_MODE 163891 53.3\n", "49 ELEVATORS_AVG 163891 53.3\n", "89 WALLSMATERIAL_MODE 156341 50.8\n", "58 APARTMENTS_MODE 156061 50.7\n", "72 APARTMENTS_MEDI 156061 50.7\n", "44 APARTMENTS_AVG 156061 50.7\n", "64 ENTRANCES_MODE 154828 50.3\n", "50 ENTRANCES_AVG 154828 50.3\n", "78 ENTRANCES_MEDI 154828 50.3\n", "83 LIVINGAREA_MEDI 154350 50.2\n", "69 LIVINGAREA_MODE 154350 50.2\n", "55 LIVINGAREA_AVG 154350 50.2\n", "87 HOUSETYPE_MODE 154297 50.2\n", "79 FLOORSMAX_MEDI 153020 49.8\n", "51 FLOORSMAX_AVG 153020 49.8\n", "65 FLOORSMAX_MODE 153020 49.8\n", "46 YEARS_BEGINEXPLUATATION_AVG 150007 48.8\n", "74 YEARS_BEGINEXPLUATATION_MEDI 150007 48.8\n", "60 YEARS_BEGINEXPLUATATION_MODE 150007 48.8\n", "88 TOTALAREA_MODE 148431 48.3\n", "90 EMERGENCYSTATE_MODE 145755 47.4\n", "28 OCCUPATION_TYPE 96391 31.3\n", "43 EXT_SOURCE_3 60965 19.8\n", "118 AMT_REQ_CREDIT_BUREAU_WEEK 41519 13.5\n", "117 AMT_REQ_CREDIT_BUREAU_DAY 41519 13.5\n", "119 AMT_REQ_CREDIT_BUREAU_MON 41519 13.5\n", "120 AMT_REQ_CREDIT_BUREAU_QRT 41519 13.5\n", "116 AMT_REQ_CREDIT_BUREAU_HOUR 41519 13.5\n", "121 AMT_REQ_CREDIT_BUREAU_YEAR 41519 13.5\n", "11 NAME_TYPE_SUITE 1292 0.4\n", "92 DEF_30_CNT_SOCIAL_CIRCLE 1021 0.3\n", "93 OBS_60_CNT_SOCIAL_CIRCLE 1021 0.3\n", "94 DEF_60_CNT_SOCIAL_CIRCLE 1021 0.3\n", "91 OBS_30_CNT_SOCIAL_CIRCLE 1021 0.3\n", "42 EXT_SOURCE_2 660 0.2\n", "10 AMT_GOODS_PRICE 278 0.1\n", "9 AMT_ANNUITY 12 0.0\n", "29 CNT_FAM_MEMBERS 2 0.0\n", "95 DAYS_LAST_PHONE_CHANGE 1 0.0" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Valeurs manquantes BD Test ---\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
colonnevaleurs_manquantespourcentage
61COMMONAREA_MODE3349568.7
75COMMONAREA_MEDI3349568.7
47COMMONAREA_AVG3349568.7
83NONLIVINGAPARTMENTS_MEDI3334768.4
55NONLIVINGAPARTMENTS_AVG3334768.4
69NONLIVINGAPARTMENTS_MODE3334768.4
85FONDKAPREMONT_MODE3279767.3
67LIVINGAPARTMENTS_MODE3278067.2
81LIVINGAPARTMENTS_MEDI3278067.2
53LIVINGAPARTMENTS_AVG3278067.2
79FLOORSMIN_MEDI3246666.6
65FLOORSMIN_MODE3246666.6
51FLOORSMIN_AVG3246666.6
20OWN_CAR_AGE3231266.3
46YEARS_BUILD_AVG3181865.3
74YEARS_BUILD_MEDI3181865.3
60YEARS_BUILD_MODE3181865.3
66LANDAREA_MODE2825458.0
52LANDAREA_AVG2825458.0
80LANDAREA_MEDI2825458.0
72BASEMENTAREA_MEDI2764156.7
44BASEMENTAREA_AVG2764156.7
58BASEMENTAREA_MODE2764156.7
84NONLIVINGAREA_MEDI2608453.5
70NONLIVINGAREA_MODE2608453.5
56NONLIVINGAREA_AVG2608453.5
76ELEVATORS_MEDI2518951.7
62ELEVATORS_MODE2518951.7
48ELEVATORS_AVG2518951.7
88WALLSMATERIAL_MODE2389349.0
57APARTMENTS_MODE2388749.0
71APARTMENTS_MEDI2388749.0
43APARTMENTS_AVG2388749.0
86HOUSETYPE_MODE2361948.5
63ENTRANCES_MODE2357948.4
77ENTRANCES_MEDI2357948.4
49ENTRANCES_AVG2357948.4
54LIVINGAREA_AVG2355248.3
82LIVINGAREA_MEDI2355248.3
68LIVINGAREA_MODE2355248.3
78FLOORSMAX_MEDI2332147.8
64FLOORSMAX_MODE2332147.8
50FLOORSMAX_AVG2332147.8
73YEARS_BEGINEXPLUATATION_MEDI2285646.9
59YEARS_BEGINEXPLUATATION_MODE2285646.9
45YEARS_BEGINEXPLUATATION_AVG2285646.9
87TOTALAREA_MODE2262446.4
89EMERGENCYSTATE_MODE2220945.6
40EXT_SOURCE_12053242.1
27OCCUPATION_TYPE1560532.0
42EXT_SOURCE_3866817.8
118AMT_REQ_CREDIT_BUREAU_MON604912.4
115AMT_REQ_CREDIT_BUREAU_HOUR604912.4
119AMT_REQ_CREDIT_BUREAU_QRT604912.4
117AMT_REQ_CREDIT_BUREAU_WEEK604912.4
116AMT_REQ_CREDIT_BUREAU_DAY604912.4
120AMT_REQ_CREDIT_BUREAU_YEAR604912.4
10NAME_TYPE_SUITE9111.9
93DEF_60_CNT_SOCIAL_CIRCLE290.1
92OBS_60_CNT_SOCIAL_CIRCLE290.1
91DEF_30_CNT_SOCIAL_CIRCLE290.1
90OBS_30_CNT_SOCIAL_CIRCLE290.1
8AMT_ANNUITY240.0
41EXT_SOURCE_280.0
\n", "
" ], "text/plain": [ " colonne valeurs_manquantes pourcentage\n", "61 COMMONAREA_MODE 33495 68.7\n", "75 COMMONAREA_MEDI 33495 68.7\n", "47 COMMONAREA_AVG 33495 68.7\n", "83 NONLIVINGAPARTMENTS_MEDI 33347 68.4\n", "55 NONLIVINGAPARTMENTS_AVG 33347 68.4\n", "69 NONLIVINGAPARTMENTS_MODE 33347 68.4\n", "85 FONDKAPREMONT_MODE 32797 67.3\n", "67 LIVINGAPARTMENTS_MODE 32780 67.2\n", "81 LIVINGAPARTMENTS_MEDI 32780 67.2\n", "53 LIVINGAPARTMENTS_AVG 32780 67.2\n", "79 FLOORSMIN_MEDI 32466 66.6\n", "65 FLOORSMIN_MODE 32466 66.6\n", "51 FLOORSMIN_AVG 32466 66.6\n", "20 OWN_CAR_AGE 32312 66.3\n", "46 YEARS_BUILD_AVG 31818 65.3\n", "74 YEARS_BUILD_MEDI 31818 65.3\n", "60 YEARS_BUILD_MODE 31818 65.3\n", "66 LANDAREA_MODE 28254 58.0\n", "52 LANDAREA_AVG 28254 58.0\n", "80 LANDAREA_MEDI 28254 58.0\n", "72 BASEMENTAREA_MEDI 27641 56.7\n", "44 BASEMENTAREA_AVG 27641 56.7\n", "58 BASEMENTAREA_MODE 27641 56.7\n", "84 NONLIVINGAREA_MEDI 26084 53.5\n", "70 NONLIVINGAREA_MODE 26084 53.5\n", "56 NONLIVINGAREA_AVG 26084 53.5\n", "76 ELEVATORS_MEDI 25189 51.7\n", "62 ELEVATORS_MODE 25189 51.7\n", "48 ELEVATORS_AVG 25189 51.7\n", "88 WALLSMATERIAL_MODE 23893 49.0\n", "57 APARTMENTS_MODE 23887 49.0\n", "71 APARTMENTS_MEDI 23887 49.0\n", "43 APARTMENTS_AVG 23887 49.0\n", "86 HOUSETYPE_MODE 23619 48.5\n", "63 ENTRANCES_MODE 23579 48.4\n", "77 ENTRANCES_MEDI 23579 48.4\n", "49 ENTRANCES_AVG 23579 48.4\n", "54 LIVINGAREA_AVG 23552 48.3\n", "82 LIVINGAREA_MEDI 23552 48.3\n", "68 LIVINGAREA_MODE 23552 48.3\n", "78 FLOORSMAX_MEDI 23321 47.8\n", "64 FLOORSMAX_MODE 23321 47.8\n", "50 FLOORSMAX_AVG 23321 47.8\n", "73 YEARS_BEGINEXPLUATATION_MEDI 22856 46.9\n", "59 YEARS_BEGINEXPLUATATION_MODE 22856 46.9\n", "45 YEARS_BEGINEXPLUATATION_AVG 22856 46.9\n", "87 TOTALAREA_MODE 22624 46.4\n", "89 EMERGENCYSTATE_MODE 22209 45.6\n", "40 EXT_SOURCE_1 20532 42.1\n", "27 OCCUPATION_TYPE 15605 32.0\n", "42 EXT_SOURCE_3 8668 17.8\n", "118 AMT_REQ_CREDIT_BUREAU_MON 6049 12.4\n", "115 AMT_REQ_CREDIT_BUREAU_HOUR 6049 12.4\n", "119 AMT_REQ_CREDIT_BUREAU_QRT 6049 12.4\n", "117 AMT_REQ_CREDIT_BUREAU_WEEK 6049 12.4\n", "116 AMT_REQ_CREDIT_BUREAU_DAY 6049 12.4\n", "120 AMT_REQ_CREDIT_BUREAU_YEAR 6049 12.4\n", "10 NAME_TYPE_SUITE 911 1.9\n", "93 DEF_60_CNT_SOCIAL_CIRCLE 29 0.1\n", "92 OBS_60_CNT_SOCIAL_CIRCLE 29 0.1\n", "91 DEF_30_CNT_SOCIAL_CIRCLE 29 0.1\n", "90 OBS_30_CNT_SOCIAL_CIRCLE 29 0.1\n", "8 AMT_ANNUITY 24 0.0\n", "41 EXT_SOURCE_2 8 0.0" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Nombre de valeurs manquantes par colonne - ratio en %\n", "pd.set_option(\"display.max_rows\", None)\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Valeurs manquantes BD {nom} ---\")\n", " missing_values = df.isna().sum().reset_index()\n", " missing_values.columns = [\"colonne\", \"valeurs_manquantes\"]\n", " \n", " missing_values[\"pourcentage\"] = (missing_values[\"valeurs_manquantes\"] / df.shape[0] * 100).round(1)\n", "\n", " missing_values = missing_values[missing_values[\"valeurs_manquantes\"] > 0]\n", " missing_values.sort_values(\"valeurs_manquantes\", ascending=False, inplace=True) \n", " display(missing_values)" ] }, { "cell_type": "code", "execution_count": 9, "id": "e71375dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Nombre de valeurs uniques - Train ---\n" ] }, { "data": { "text/plain": [ "Series([], dtype: int64)" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Nombre de valeurs uniques - Test ---\n" ] }, { "data": { "text/plain": [ "FLAG_DOCUMENT_2 1\n", "FLAG_DOCUMENT_10 1\n", "FLAG_DOCUMENT_12 1\n", "FLAG_DOCUMENT_13 1\n", "FLAG_DOCUMENT_14 1\n", "FLAG_DOCUMENT_15 1\n", "FLAG_DOCUMENT_16 1\n", "FLAG_DOCUMENT_17 1\n", "FLAG_DOCUMENT_19 1\n", "FLAG_DOCUMENT_20 1\n", "FLAG_DOCUMENT_21 1\n", "dtype: int64" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Vérification des colonnes constantes\n", "pd.set_option(\"display.max_rows\", None)\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Nombre de valeurs uniques - {nom} ---\")\n", " constantes = df.nunique()\n", " constantes = constantes[constantes == 1]\n", " display(constantes)" ] }, { "cell_type": "code", "execution_count": 10, "id": "02cf46b1", "metadata": {}, "outputs": [], "source": [ "# Modifier les formats des colonnes de type \"bool\" qui sont des int64 \n", "def convertir_bool(df):\n", " df = df.copy()\n", " for col in df.select_dtypes(include=[\"int64\", \"float64\"]).columns:\n", " valeurs_uniques = set(df[col].dropna().unique())\n", " if valeurs_uniques.issubset({0, 1}):\n", " df[col] = df[col].astype(bool)\n", " return df\n", "\n", "app_train1 = convertir_bool(app_train)\n", "app_test1 = convertir_bool(app_test)" ] }, { "cell_type": "code", "execution_count": 11, "id": "5b0f290d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Valeurs manquantes BD Train ---\n", "\n", "RangeIndex: 307511 entries, 0 to 307510\n", "Data columns (total 122 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 307511 non-null int64 \n", " 1 TARGET 307511 non-null bool \n", " 2 NAME_CONTRACT_TYPE 307511 non-null object \n", " 3 CODE_GENDER 307511 non-null object \n", " 4 FLAG_OWN_CAR 307511 non-null object \n", " 5 FLAG_OWN_REALTY 307511 non-null object \n", " 6 CNT_CHILDREN 307511 non-null int64 \n", " 7 AMT_INCOME_TOTAL 307511 non-null float64\n", " 8 AMT_CREDIT 307511 non-null float64\n", " 9 AMT_ANNUITY 307499 non-null float64\n", " 10 AMT_GOODS_PRICE 307233 non-null float64\n", " 11 NAME_TYPE_SUITE 306219 non-null object \n", " 12 NAME_INCOME_TYPE 307511 non-null object \n", " 13 NAME_EDUCATION_TYPE 307511 non-null object \n", " 14 NAME_FAMILY_STATUS 307511 non-null object \n", " 15 NAME_HOUSING_TYPE 307511 non-null object \n", " 16 REGION_POPULATION_RELATIVE 307511 non-null float64\n", " 17 DAYS_BIRTH 307511 non-null int64 \n", " 18 DAYS_EMPLOYED 307511 non-null int64 \n", " 19 DAYS_REGISTRATION 307511 non-null float64\n", " 20 DAYS_ID_PUBLISH 307511 non-null int64 \n", " 21 OWN_CAR_AGE 104582 non-null float64\n", " 22 FLAG_MOBIL 307511 non-null bool \n", " 23 FLAG_EMP_PHONE 307511 non-null bool \n", " 24 FLAG_WORK_PHONE 307511 non-null bool \n", " 25 FLAG_CONT_MOBILE 307511 non-null bool \n", " 26 FLAG_PHONE 307511 non-null bool \n", " 27 FLAG_EMAIL 307511 non-null bool \n", " 28 OCCUPATION_TYPE 211120 non-null object \n", " 29 CNT_FAM_MEMBERS 307509 non-null float64\n", " 30 REGION_RATING_CLIENT 307511 non-null int64 \n", " 31 REGION_RATING_CLIENT_W_CITY 307511 non-null int64 \n", " 32 WEEKDAY_APPR_PROCESS_START 307511 non-null object \n", " 33 HOUR_APPR_PROCESS_START 307511 non-null int64 \n", " 34 REG_REGION_NOT_LIVE_REGION 307511 non-null bool \n", " 35 REG_REGION_NOT_WORK_REGION 307511 non-null bool \n", " 36 LIVE_REGION_NOT_WORK_REGION 307511 non-null bool \n", " 37 REG_CITY_NOT_LIVE_CITY 307511 non-null bool \n", " 38 REG_CITY_NOT_WORK_CITY 307511 non-null bool \n", " 39 LIVE_CITY_NOT_WORK_CITY 307511 non-null bool \n", " 40 ORGANIZATION_TYPE 307511 non-null object \n", " 41 EXT_SOURCE_1 134133 non-null float64\n", " 42 EXT_SOURCE_2 306851 non-null float64\n", " 43 EXT_SOURCE_3 246546 non-null float64\n", " 44 APARTMENTS_AVG 151450 non-null float64\n", " 45 BASEMENTAREA_AVG 127568 non-null float64\n", " 46 YEARS_BEGINEXPLUATATION_AVG 157504 non-null float64\n", " 47 YEARS_BUILD_AVG 103023 non-null float64\n", " 48 COMMONAREA_AVG 92646 non-null float64\n", " 49 ELEVATORS_AVG 143620 non-null float64\n", " 50 ENTRANCES_AVG 152683 non-null float64\n", " 51 FLOORSMAX_AVG 154491 non-null float64\n", " 52 FLOORSMIN_AVG 98869 non-null float64\n", " 53 LANDAREA_AVG 124921 non-null float64\n", " 54 LIVINGAPARTMENTS_AVG 97312 non-null float64\n", " 55 LIVINGAREA_AVG 153161 non-null float64\n", " 56 NONLIVINGAPARTMENTS_AVG 93997 non-null float64\n", " 57 NONLIVINGAREA_AVG 137829 non-null float64\n", " 58 APARTMENTS_MODE 151450 non-null float64\n", " 59 BASEMENTAREA_MODE 127568 non-null float64\n", " 60 YEARS_BEGINEXPLUATATION_MODE 157504 non-null float64\n", " 61 YEARS_BUILD_MODE 103023 non-null float64\n", " 62 COMMONAREA_MODE 92646 non-null float64\n", " 63 ELEVATORS_MODE 143620 non-null float64\n", " 64 ENTRANCES_MODE 152683 non-null float64\n", " 65 FLOORSMAX_MODE 154491 non-null float64\n", " 66 FLOORSMIN_MODE 98869 non-null float64\n", " 67 LANDAREA_MODE 124921 non-null float64\n", " 68 LIVINGAPARTMENTS_MODE 97312 non-null float64\n", " 69 LIVINGAREA_MODE 153161 non-null float64\n", " 70 NONLIVINGAPARTMENTS_MODE 93997 non-null float64\n", " 71 NONLIVINGAREA_MODE 137829 non-null float64\n", " 72 APARTMENTS_MEDI 151450 non-null float64\n", " 73 BASEMENTAREA_MEDI 127568 non-null float64\n", " 74 YEARS_BEGINEXPLUATATION_MEDI 157504 non-null float64\n", " 75 YEARS_BUILD_MEDI 103023 non-null float64\n", " 76 COMMONAREA_MEDI 92646 non-null float64\n", " 77 ELEVATORS_MEDI 143620 non-null float64\n", " 78 ENTRANCES_MEDI 152683 non-null float64\n", " 79 FLOORSMAX_MEDI 154491 non-null float64\n", " 80 FLOORSMIN_MEDI 98869 non-null float64\n", " 81 LANDAREA_MEDI 124921 non-null float64\n", " 82 LIVINGAPARTMENTS_MEDI 97312 non-null float64\n", " 83 LIVINGAREA_MEDI 153161 non-null float64\n", " 84 NONLIVINGAPARTMENTS_MEDI 93997 non-null float64\n", " 85 NONLIVINGAREA_MEDI 137829 non-null float64\n", " 86 FONDKAPREMONT_MODE 97216 non-null object \n", " 87 HOUSETYPE_MODE 153214 non-null object \n", " 88 TOTALAREA_MODE 159080 non-null float64\n", " 89 WALLSMATERIAL_MODE 151170 non-null object \n", " 90 EMERGENCYSTATE_MODE 161756 non-null object \n", " 91 OBS_30_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 92 DEF_30_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 93 OBS_60_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 94 DEF_60_CNT_SOCIAL_CIRCLE 306490 non-null float64\n", " 95 DAYS_LAST_PHONE_CHANGE 307510 non-null float64\n", " 96 FLAG_DOCUMENT_2 307511 non-null bool \n", " 97 FLAG_DOCUMENT_3 307511 non-null bool \n", " 98 FLAG_DOCUMENT_4 307511 non-null bool \n", " 99 FLAG_DOCUMENT_5 307511 non-null bool \n", " 100 FLAG_DOCUMENT_6 307511 non-null bool \n", " 101 FLAG_DOCUMENT_7 307511 non-null bool \n", " 102 FLAG_DOCUMENT_8 307511 non-null bool \n", " 103 FLAG_DOCUMENT_9 307511 non-null bool \n", " 104 FLAG_DOCUMENT_10 307511 non-null bool \n", " 105 FLAG_DOCUMENT_11 307511 non-null bool \n", " 106 FLAG_DOCUMENT_12 307511 non-null bool \n", " 107 FLAG_DOCUMENT_13 307511 non-null bool \n", " 108 FLAG_DOCUMENT_14 307511 non-null bool \n", " 109 FLAG_DOCUMENT_15 307511 non-null bool \n", " 110 FLAG_DOCUMENT_16 307511 non-null bool \n", " 111 FLAG_DOCUMENT_17 307511 non-null bool \n", " 112 FLAG_DOCUMENT_18 307511 non-null bool \n", " 113 FLAG_DOCUMENT_19 307511 non-null bool \n", " 114 FLAG_DOCUMENT_20 307511 non-null bool \n", " 115 FLAG_DOCUMENT_21 307511 non-null bool \n", " 116 AMT_REQ_CREDIT_BUREAU_HOUR 265992 non-null float64\n", " 117 AMT_REQ_CREDIT_BUREAU_DAY 265992 non-null float64\n", " 118 AMT_REQ_CREDIT_BUREAU_WEEK 265992 non-null float64\n", " 119 AMT_REQ_CREDIT_BUREAU_MON 265992 non-null float64\n", " 120 AMT_REQ_CREDIT_BUREAU_QRT 265992 non-null float64\n", " 121 AMT_REQ_CREDIT_BUREAU_YEAR 265992 non-null float64\n", "dtypes: bool(33), float64(65), int64(8), object(16)\n", "memory usage: 218.5+ MB\n" ] }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Valeurs manquantes BD Test ---\n", "\n", "RangeIndex: 48744 entries, 0 to 48743\n", "Data columns (total 121 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 48744 non-null int64 \n", " 1 NAME_CONTRACT_TYPE 48744 non-null object \n", " 2 CODE_GENDER 48744 non-null object \n", " 3 FLAG_OWN_CAR 48744 non-null object \n", " 4 FLAG_OWN_REALTY 48744 non-null object \n", " 5 CNT_CHILDREN 48744 non-null int64 \n", " 6 AMT_INCOME_TOTAL 48744 non-null float64\n", " 7 AMT_CREDIT 48744 non-null float64\n", " 8 AMT_ANNUITY 48720 non-null float64\n", " 9 AMT_GOODS_PRICE 48744 non-null float64\n", " 10 NAME_TYPE_SUITE 47833 non-null object \n", " 11 NAME_INCOME_TYPE 48744 non-null object \n", " 12 NAME_EDUCATION_TYPE 48744 non-null object \n", " 13 NAME_FAMILY_STATUS 48744 non-null object \n", " 14 NAME_HOUSING_TYPE 48744 non-null object \n", " 15 REGION_POPULATION_RELATIVE 48744 non-null float64\n", " 16 DAYS_BIRTH 48744 non-null int64 \n", " 17 DAYS_EMPLOYED 48744 non-null int64 \n", " 18 DAYS_REGISTRATION 48744 non-null float64\n", " 19 DAYS_ID_PUBLISH 48744 non-null int64 \n", " 20 OWN_CAR_AGE 16432 non-null float64\n", " 21 FLAG_MOBIL 48744 non-null bool \n", " 22 FLAG_EMP_PHONE 48744 non-null bool \n", " 23 FLAG_WORK_PHONE 48744 non-null bool \n", " 24 FLAG_CONT_MOBILE 48744 non-null bool \n", " 25 FLAG_PHONE 48744 non-null bool \n", " 26 FLAG_EMAIL 48744 non-null bool \n", " 27 OCCUPATION_TYPE 33139 non-null object \n", " 28 CNT_FAM_MEMBERS 48744 non-null float64\n", " 29 REGION_RATING_CLIENT 48744 non-null int64 \n", " 30 REGION_RATING_CLIENT_W_CITY 48744 non-null int64 \n", " 31 WEEKDAY_APPR_PROCESS_START 48744 non-null object \n", " 32 HOUR_APPR_PROCESS_START 48744 non-null int64 \n", " 33 REG_REGION_NOT_LIVE_REGION 48744 non-null bool \n", " 34 REG_REGION_NOT_WORK_REGION 48744 non-null bool \n", " 35 LIVE_REGION_NOT_WORK_REGION 48744 non-null bool \n", " 36 REG_CITY_NOT_LIVE_CITY 48744 non-null bool \n", " 37 REG_CITY_NOT_WORK_CITY 48744 non-null bool \n", " 38 LIVE_CITY_NOT_WORK_CITY 48744 non-null bool \n", " 39 ORGANIZATION_TYPE 48744 non-null object \n", " 40 EXT_SOURCE_1 28212 non-null float64\n", " 41 EXT_SOURCE_2 48736 non-null float64\n", " 42 EXT_SOURCE_3 40076 non-null float64\n", " 43 APARTMENTS_AVG 24857 non-null float64\n", " 44 BASEMENTAREA_AVG 21103 non-null float64\n", " 45 YEARS_BEGINEXPLUATATION_AVG 25888 non-null float64\n", " 46 YEARS_BUILD_AVG 16926 non-null float64\n", " 47 COMMONAREA_AVG 15249 non-null float64\n", " 48 ELEVATORS_AVG 23555 non-null float64\n", " 49 ENTRANCES_AVG 25165 non-null float64\n", " 50 FLOORSMAX_AVG 25423 non-null float64\n", " 51 FLOORSMIN_AVG 16278 non-null float64\n", " 52 LANDAREA_AVG 20490 non-null float64\n", " 53 LIVINGAPARTMENTS_AVG 15964 non-null float64\n", " 54 LIVINGAREA_AVG 25192 non-null float64\n", " 55 NONLIVINGAPARTMENTS_AVG 15397 non-null float64\n", " 56 NONLIVINGAREA_AVG 22660 non-null float64\n", " 57 APARTMENTS_MODE 24857 non-null float64\n", " 58 BASEMENTAREA_MODE 21103 non-null float64\n", " 59 YEARS_BEGINEXPLUATATION_MODE 25888 non-null float64\n", " 60 YEARS_BUILD_MODE 16926 non-null float64\n", " 61 COMMONAREA_MODE 15249 non-null float64\n", " 62 ELEVATORS_MODE 23555 non-null float64\n", " 63 ENTRANCES_MODE 25165 non-null float64\n", " 64 FLOORSMAX_MODE 25423 non-null float64\n", " 65 FLOORSMIN_MODE 16278 non-null float64\n", " 66 LANDAREA_MODE 20490 non-null float64\n", " 67 LIVINGAPARTMENTS_MODE 15964 non-null float64\n", " 68 LIVINGAREA_MODE 25192 non-null float64\n", " 69 NONLIVINGAPARTMENTS_MODE 15397 non-null float64\n", " 70 NONLIVINGAREA_MODE 22660 non-null float64\n", " 71 APARTMENTS_MEDI 24857 non-null float64\n", " 72 BASEMENTAREA_MEDI 21103 non-null float64\n", " 73 YEARS_BEGINEXPLUATATION_MEDI 25888 non-null float64\n", " 74 YEARS_BUILD_MEDI 16926 non-null float64\n", " 75 COMMONAREA_MEDI 15249 non-null float64\n", " 76 ELEVATORS_MEDI 23555 non-null float64\n", " 77 ENTRANCES_MEDI 25165 non-null float64\n", " 78 FLOORSMAX_MEDI 25423 non-null float64\n", " 79 FLOORSMIN_MEDI 16278 non-null float64\n", " 80 LANDAREA_MEDI 20490 non-null float64\n", " 81 LIVINGAPARTMENTS_MEDI 15964 non-null float64\n", " 82 LIVINGAREA_MEDI 25192 non-null float64\n", " 83 NONLIVINGAPARTMENTS_MEDI 15397 non-null float64\n", " 84 NONLIVINGAREA_MEDI 22660 non-null float64\n", " 85 FONDKAPREMONT_MODE 15947 non-null object \n", " 86 HOUSETYPE_MODE 25125 non-null object \n", " 87 TOTALAREA_MODE 26120 non-null float64\n", " 88 WALLSMATERIAL_MODE 24851 non-null object \n", " 89 EMERGENCYSTATE_MODE 26535 non-null object \n", " 90 OBS_30_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 91 DEF_30_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 92 OBS_60_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 93 DEF_60_CNT_SOCIAL_CIRCLE 48715 non-null float64\n", " 94 DAYS_LAST_PHONE_CHANGE 48744 non-null float64\n", " 95 FLAG_DOCUMENT_2 48744 non-null bool \n", " 96 FLAG_DOCUMENT_3 48744 non-null bool \n", " 97 FLAG_DOCUMENT_4 48744 non-null bool \n", " 98 FLAG_DOCUMENT_5 48744 non-null bool \n", " 99 FLAG_DOCUMENT_6 48744 non-null bool \n", " 100 FLAG_DOCUMENT_7 48744 non-null bool \n", " 101 FLAG_DOCUMENT_8 48744 non-null bool \n", " 102 FLAG_DOCUMENT_9 48744 non-null bool \n", " 103 FLAG_DOCUMENT_10 48744 non-null bool \n", " 104 FLAG_DOCUMENT_11 48744 non-null bool \n", " 105 FLAG_DOCUMENT_12 48744 non-null bool \n", " 106 FLAG_DOCUMENT_13 48744 non-null bool \n", " 107 FLAG_DOCUMENT_14 48744 non-null bool \n", " 108 FLAG_DOCUMENT_15 48744 non-null bool \n", " 109 FLAG_DOCUMENT_16 48744 non-null bool \n", " 110 FLAG_DOCUMENT_17 48744 non-null bool \n", " 111 FLAG_DOCUMENT_18 48744 non-null bool \n", " 112 FLAG_DOCUMENT_19 48744 non-null bool \n", " 113 FLAG_DOCUMENT_20 48744 non-null bool \n", " 114 FLAG_DOCUMENT_21 48744 non-null bool \n", " 115 AMT_REQ_CREDIT_BUREAU_HOUR 42695 non-null float64\n", " 116 AMT_REQ_CREDIT_BUREAU_DAY 42695 non-null float64\n", " 117 AMT_REQ_CREDIT_BUREAU_WEEK 42695 non-null float64\n", " 118 AMT_REQ_CREDIT_BUREAU_MON 42695 non-null float64\n", " 119 AMT_REQ_CREDIT_BUREAU_QRT 42695 non-null float64\n", " 120 AMT_REQ_CREDIT_BUREAU_YEAR 42695 non-null float64\n", "dtypes: bool(32), float64(65), int64(8), object(16)\n", "memory usage: 34.6+ MB\n" ] }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "pd.set_option(\"display.max_rows\", None)\n", "\n", "jeu_donnee = [app_train1, app_test1]\n", "noms = [\"Train\", \"Test\"]\n", "for nom, df in zip(noms, jeu_donnee):\n", " print(f\"\\n--- Valeurs manquantes BD {nom} ---\") \n", "\n", " display(df.info())" ] }, { "cell_type": "code", "execution_count": 12, "id": "9f03bb1e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countmean
TARGET
False282686602648.282002
True24825557778.527674
\n", "
" ], "text/plain": [ " count mean\n", "TARGET \n", "False 282686 602648.282002\n", "True 24825 557778.527674" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Distribution du montant moyen de crédit par cible\n", "credit_par_target = (\n", " app_train1\n", " .groupby(\"TARGET\")[\"AMT_CREDIT\"]\n", " .agg([\"count\", \"mean\"])\n", ")\n", "\n", "credit_par_target\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "dd8e8419", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TARGET CODE_GENDER\n", "False F 188278\n", " M 94404\n", " XNA 4\n", "True F 14170\n", " M 10655\n", "dtype: int64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Distribution du prix moyen du produit lié au crédit par cible\n", "genre_par_target = app_train1.groupby([\"TARGET\", \"CODE_GENDER\"]).size()\n", "genre_par_target" ] }, { "cell_type": "code", "execution_count": 14, "id": "6a13641c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TARGETnb_clientscredit_moyenprix_produit_moyenannuite_moyenneage_moyenage_median
0False282686602648.282002542736.79500327163.62334944.21418243.498630
1True24825557778.527674488972.41255426481.74429040.78035139.128767
\n", "
" ], "text/plain": [ " TARGET nb_clients credit_moyen prix_produit_moyen annuite_moyenne \\\n", "0 False 282686 602648.282002 542736.795003 27163.623349 \n", "1 True 24825 557778.527674 488972.412554 26481.744290 \n", "\n", " age_moyen age_median \n", "0 44.214182 43.498630 \n", "1 40.780351 39.128767 " ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "summary = (\n", " app_train1\n", " .assign(AGE = app_train1[\"DAYS_BIRTH\"] / -365)\n", " .groupby(\"TARGET\")\n", " .agg(\n", " nb_clients = (\"SK_ID_CURR\", \"count\"),\n", " credit_moyen = (\"AMT_CREDIT\", \"mean\"),\n", " prix_produit_moyen = (\"AMT_GOODS_PRICE\", \"mean\"),\n", " annuite_moyenne = (\"AMT_ANNUITY\", \"mean\"),\n", " age_moyen = (\"AGE\", \"mean\"),\n", " age_median = (\"AGE\", \"median\")\n", " )\n", " .reset_index()\n", ")\n", "\n", "summary\n" ] }, { "cell_type": "markdown", "id": "6d08ca00", "metadata": {}, "source": [ "### Fusion des fichiers avec Train & Test" ] }, { "cell_type": "markdown", "id": "bb86703b", "metadata": {}, "source": [ "#### Fusion des fichiers bureau + bureau_balance avec \"train/test\"\n", "- Fusionner bureau + bureau_balance\n", "- aggréger les données obtenues afin de pouvoir les intégrer dans le train/test\n", "- Fusionner avec train & test" ] }, { "cell_type": "code", "execution_count": 15, "id": "e7b73995", "metadata": {}, "outputs": [], "source": [ "# Charger les données bureau\n", "bureau_balance = pd.read_csv(\"./Donnees_Brutes/bureau_balance.csv\", sep=',')\n", "bureau = pd.read_csv(\"./Donnees_Brutes/bureau.csv\", sep=',')" ] }, { "cell_type": "code", "execution_count": 16, "id": "bbcaec81", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 25121815 entries, 0 to 25121814\n", "Data columns (total 19 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 SK_ID_CURR int64 \n", " 1 SK_ID_BUREAU int64 \n", " 2 CREDIT_ACTIVE object \n", " 3 CREDIT_CURRENCY object \n", " 4 DAYS_CREDIT int64 \n", " 5 CREDIT_DAY_OVERDUE int64 \n", " 6 DAYS_CREDIT_ENDDATE float64\n", " 7 DAYS_ENDDATE_FACT float64\n", " 8 AMT_CREDIT_MAX_OVERDUE float64\n", " 9 CNT_CREDIT_PROLONG int64 \n", " 10 AMT_CREDIT_SUM float64\n", " 11 AMT_CREDIT_SUM_DEBT float64\n", " 12 AMT_CREDIT_SUM_LIMIT float64\n", " 13 AMT_CREDIT_SUM_OVERDUE float64\n", " 14 CREDIT_TYPE object \n", " 15 DAYS_CREDIT_UPDATE int64 \n", " 16 AMT_ANNUITY float64\n", " 17 MONTHS_BALANCE float64\n", " 18 STATUS object \n", "dtypes: float64(9), int64(6), object(4)\n", "memory usage: 3.6+ GB\n" ] } ], "source": [ "# Fusion des fichiers bureau + bureau_balance\n", "bureau_merged = pd.merge(bureau, bureau_balance, on='SK_ID_BUREAU', how='left')\n", "bureau_merged.info()" ] }, { "cell_type": "code", "execution_count": 17, "id": "c3025aeb", "metadata": {}, "outputs": [], "source": [ "def check_columns(df_ref, df_test, name_ref=\"train\", name_test=\"test\"):\n", " cols_ref = set(df_ref.columns)\n", " cols_test = set(df_test.columns)\n", "\n", " missing = cols_ref - cols_test\n", " extra = cols_test - cols_ref\n", "\n", " print(f\"--- Vérification colonnes : {name_test} vs {name_ref} ---\")\n", " print(f\"Colonnes attendues : {len(cols_ref)}\")\n", " print(f\"Colonnes trouvées : {len(cols_test)}\")\n", "\n", " if missing:\n", " print(\"\\n❌ Colonnes manquantes dans\", name_test, \":\")\n", " for c in sorted(missing):\n", " print(\" -\", c)\n", " else:\n", " print(\"\\n✔️ Aucune colonne manquante\")\n", "\n", " if extra:\n", " print(\"\\n⚠️ Colonnes supplémentaires dans\", name_test, \":\")\n", " for c in sorted(extra):\n", " print(\" -\", c)\n", " else:\n", " print(\"\\n✔️ Aucune colonne supplémentaire\")\n", "\n", " print(\"\\n--------------------------------------------\\n\")\n" ] }, { "cell_type": "code", "execution_count": 18, "id": "1784f0d1", "metadata": {}, "outputs": [], "source": [ "# Fonction d'aggrégation previous_app\n", "def agg_fichiers(data, prefix=\"PREV\"):\n", "\n", " df = data.copy()\n", "\n", " # Sélection des colonnes\n", " num_cols = df.select_dtypes(include=[\"int64\", \"float64\"]).columns.tolist()\n", " cat_cols = df.select_dtypes(include=[\"object\"]).columns.tolist()\n", "\n", " # Exclure les identifiants des numériques lorsqu'ils existent\n", " num_cols = [c for c in num_cols if c not in df.columns.intersection([\"SK_ID_CURR\", \"SK_ID_PREV\"])]\n", "\n", "\n", " # Agrégations numériques\n", " print(\"Début agg numériques\")\n", "\n", " num_aggs = {col: [\"mean\", \"max\", \"min\", \"sum\"] for col in num_cols}\n", "\n", " num_agg = df.groupby(\"SK_ID_CURR\").agg(num_aggs)\n", " num_agg.columns = [f\"{prefix}_\" + \"_\".join(col) for col in num_agg.columns]\n", " num_agg = num_agg.reset_index()\n", "\n", " print(\"Fin agg numériques\")\n", "\n", " # Agrégations catégorielles : NUNIQUE\n", " print(\"Début nunique cat\")\n", "\n", " if cat_cols:\n", " cat_nunique = (\n", " df.groupby(\"SK_ID_CURR\")[cat_cols]\n", " .nunique()\n", " .reset_index()\n", " )\n", " cat_nunique.columns = [\"SK_ID_CURR\"] + [f\"{prefix}_{c}_NUNIQUE\" for c in cat_cols]\n", " else:\n", " cat_nunique = None\n", "\n", " print(\"Fin nunique cat\")\n", "\n", " # Agrégations catégorielles\n", " print(\"Début dummies + groupby cat\")\n", "\n", " cat_counts = None\n", "\n", " for i, col in enumerate(cat_cols, 1):\n", " print(f\"Traitement {i}/{len(cat_cols)} : {col}\")\n", "\n", " dummies = pd.get_dummies(df[col], prefix=f\"{prefix}_{col}\")\n", " dummies[\"SK_ID_CURR\"] = df[\"SK_ID_CURR\"]\n", "\n", " dummies_sum = dummies.groupby(\"SK_ID_CURR\").sum().reset_index()\n", "\n", " if cat_counts is None:\n", " cat_counts = dummies_sum\n", " else:\n", " cat_counts = cat_counts.merge(dummies_sum, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", " print(\"Fin catégoriel\")\n", "\n", " # Fusion finale\n", " print(\"Fusion finale\")\n", "\n", " df_agg = num_agg.copy()\n", "\n", " if cat_nunique is not None:\n", " df_agg = df_agg.merge(cat_nunique, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", " if cat_counts is not None:\n", " df_agg = df_agg.merge(cat_counts, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", " print(\"Agrégation terminée\")\n", " return df_agg\n", "\n" ] }, { "cell_type": "code", "execution_count": 19, "id": "50d83e6a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Début agg numériques\n", "Fin agg numériques\n", "Début nunique cat\n", "Fin nunique cat\n", "Début dummies + groupby cat\n", "Traitement 1/4 : CREDIT_ACTIVE\n", "Traitement 2/4 : CREDIT_CURRENCY\n", "Traitement 3/4 : CREDIT_TYPE\n", "Traitement 4/4 : STATUS\n", "Fin catégoriel\n", "Fusion finale\n", "Agrégation terminée\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_CURRBUREAU_SK_ID_BUREAU_meanBUREAU_SK_ID_BUREAU_maxBUREAU_SK_ID_BUREAU_minBUREAU_SK_ID_BUREAU_sumBUREAU_DAYS_CREDIT_meanBUREAU_DAYS_CREDIT_maxBUREAU_DAYS_CREDIT_minBUREAU_DAYS_CREDIT_sumBUREAU_CREDIT_DAY_OVERDUE_mean...BUREAU_CREDIT_TYPE_Real estate loanBUREAU_CREDIT_TYPE_Unknown type of loanBUREAU_STATUS_0BUREAU_STATUS_1BUREAU_STATUS_2BUREAU_STATUS_3BUREAU_STATUS_4BUREAU_STATUS_5BUREAU_STATUS_CBUREAU_STATUS_X
01000015.896632e+06589663658966301014220756-1009.284884-49-1572-1735970.0...00311000011030
11000026.152350e+0661589096113835676758452-996.781818-103-1437-1096460.0...00452700002315
21000035.885878e+065885880588587723543514-1400.750000-606-2586-56030.0...0000000000
31000046.829134e+066829134682913313658267-867.000000-408-1326-17340.0...0000000000
41000056.735201e+0667352026735200141439211-272.380952-62-373-57200.0...00140000052
\n", "

5 rows × 92 columns

\n", "
" ], "text/plain": [ " SK_ID_CURR BUREAU_SK_ID_BUREAU_mean BUREAU_SK_ID_BUREAU_max \\\n", "0 100001 5.896632e+06 5896636 \n", "1 100002 6.152350e+06 6158909 \n", "2 100003 5.885878e+06 5885880 \n", "3 100004 6.829134e+06 6829134 \n", "4 100005 6.735201e+06 6735202 \n", "\n", " BUREAU_SK_ID_BUREAU_min BUREAU_SK_ID_BUREAU_sum BUREAU_DAYS_CREDIT_mean \\\n", "0 5896630 1014220756 -1009.284884 \n", "1 6113835 676758452 -996.781818 \n", "2 5885877 23543514 -1400.750000 \n", "3 6829133 13658267 -867.000000 \n", "4 6735200 141439211 -272.380952 \n", "\n", " BUREAU_DAYS_CREDIT_max BUREAU_DAYS_CREDIT_min BUREAU_DAYS_CREDIT_sum \\\n", "0 -49 -1572 -173597 \n", "1 -103 -1437 -109646 \n", "2 -606 -2586 -5603 \n", "3 -408 -1326 -1734 \n", "4 -62 -373 -5720 \n", "\n", " BUREAU_CREDIT_DAY_OVERDUE_mean ... BUREAU_CREDIT_TYPE_Real estate loan \\\n", "0 0.0 ... 0 \n", "1 0.0 ... 0 \n", "2 0.0 ... 0 \n", "3 0.0 ... 0 \n", "4 0.0 ... 0 \n", "\n", " BUREAU_CREDIT_TYPE_Unknown type of loan BUREAU_STATUS_0 BUREAU_STATUS_1 \\\n", "0 0 31 1 \n", "1 0 45 27 \n", "2 0 0 0 \n", "3 0 0 0 \n", "4 0 14 0 \n", "\n", " BUREAU_STATUS_2 BUREAU_STATUS_3 BUREAU_STATUS_4 BUREAU_STATUS_5 \\\n", "0 0 0 0 0 \n", "1 0 0 0 0 \n", "2 0 0 0 0 \n", "3 0 0 0 0 \n", "4 0 0 0 0 \n", "\n", " BUREAU_STATUS_C BUREAU_STATUS_X \n", "0 110 30 \n", "1 23 15 \n", "2 0 0 \n", "3 0 0 \n", "4 5 2 \n", "\n", "[5 rows x 92 columns]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bureau_merge_agg = agg_fichiers(data=bureau_merged, prefix=\"BUREAU\")\n", "bureau_merge_agg.head()" ] }, { "cell_type": "code", "execution_count": 20, "id": "289050a5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(307511, 213) (48744, 212)\n" ] } ], "source": [ "# Fusion train/test & bureau_merge_agg\n", "app_train_clean1 = app_train1.merge(bureau_merge_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "app_test_clean1 = app_test1.merge(bureau_merge_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", "print(app_train_clean1.shape, app_test_clean1.shape)" ] }, { "cell_type": "code", "execution_count": 22, "id": "bbae6f7f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Vérification colonnes : test vs train ---\n", "Colonnes attendues : 213\n", "Colonnes trouvées : 212\n", "\n", "❌ Colonnes manquantes dans test :\n", " - TARGET\n", "\n", "✔️ Aucune colonne supplémentaire\n", "\n", "--------------------------------------------\n", "\n" ] } ], "source": [ "# Check des colonnes entre train et test\n", "check_columns(app_train_clean1, app_test_clean1, name_ref=\"train\", name_test=\"test\")\n" ] }, { "cell_type": "markdown", "id": "11900cfb", "metadata": {}, "source": [ "#### Fusion previous_app avec \"train/test\"\n", "- aggréger les données afin de pouvoir les intégrer dans le train/test\n", "- Fusionner avec train & test" ] }, { "cell_type": "code", "execution_count": 23, "id": "0a643614", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 1670214 entries, 0 to 1670213\n", "Data columns (total 37 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_PREV 1670214 non-null int64 \n", " 1 SK_ID_CURR 1670214 non-null int64 \n", " 2 NAME_CONTRACT_TYPE 1670214 non-null object \n", " 3 AMT_ANNUITY 1297979 non-null float64\n", " 4 AMT_APPLICATION 1670214 non-null float64\n", " 5 AMT_CREDIT 1670213 non-null float64\n", " 6 AMT_DOWN_PAYMENT 774370 non-null float64\n", " 7 AMT_GOODS_PRICE 1284699 non-null float64\n", " 8 WEEKDAY_APPR_PROCESS_START 1670214 non-null object \n", " 9 HOUR_APPR_PROCESS_START 1670214 non-null int64 \n", " 10 FLAG_LAST_APPL_PER_CONTRACT 1670214 non-null object \n", " 11 NFLAG_LAST_APPL_IN_DAY 1670214 non-null int64 \n", " 12 RATE_DOWN_PAYMENT 774370 non-null float64\n", " 13 RATE_INTEREST_PRIMARY 5951 non-null float64\n", " 14 RATE_INTEREST_PRIVILEGED 5951 non-null float64\n", " 15 NAME_CASH_LOAN_PURPOSE 1670214 non-null object \n", " 16 NAME_CONTRACT_STATUS 1670214 non-null object \n", " 17 DAYS_DECISION 1670214 non-null int64 \n", " 18 NAME_PAYMENT_TYPE 1670214 non-null object \n", " 19 CODE_REJECT_REASON 1670214 non-null object \n", " 20 NAME_TYPE_SUITE 849809 non-null object \n", " 21 NAME_CLIENT_TYPE 1670214 non-null object \n", " 22 NAME_GOODS_CATEGORY 1670214 non-null object \n", " 23 NAME_PORTFOLIO 1670214 non-null object \n", " 24 NAME_PRODUCT_TYPE 1670214 non-null object \n", " 25 CHANNEL_TYPE 1670214 non-null object \n", " 26 SELLERPLACE_AREA 1670214 non-null int64 \n", " 27 NAME_SELLER_INDUSTRY 1670214 non-null object \n", " 28 CNT_PAYMENT 1297984 non-null float64\n", " 29 NAME_YIELD_GROUP 1670214 non-null object \n", " 30 PRODUCT_COMBINATION 1669868 non-null object \n", " 31 DAYS_FIRST_DRAWING 997149 non-null float64\n", " 32 DAYS_FIRST_DUE 997149 non-null float64\n", " 33 DAYS_LAST_DUE_1ST_VERSION 997149 non-null float64\n", " 34 DAYS_LAST_DUE 997149 non-null float64\n", " 35 DAYS_TERMINATION 997149 non-null float64\n", " 36 NFLAG_INSURED_ON_APPROVAL 997149 non-null float64\n", "dtypes: float64(15), int64(6), object(16)\n", "memory usage: 471.5+ MB\n" ] } ], "source": [ "# Chargement des données sur les demandes précédentes\n", "previous_app = pd.read_csv(\"./Donnees_Brutes/previous_application.csv\", sep=',')\n", "previous_app.info()" ] }, { "cell_type": "code", "execution_count": 24, "id": "88ade8e5", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_PREVSK_ID_CURRNAME_CONTRACT_TYPEAMT_ANNUITYAMT_APPLICATIONAMT_CREDITAMT_DOWN_PAYMENTAMT_GOODS_PRICEWEEKDAY_APPR_PROCESS_STARTHOUR_APPR_PROCESS_START...NAME_SELLER_INDUSTRYCNT_PAYMENTNAME_YIELD_GROUPPRODUCT_COMBINATIONDAYS_FIRST_DRAWINGDAYS_FIRST_DUEDAYS_LAST_DUE_1ST_VERSIONDAYS_LAST_DUEDAYS_TERMINATIONNFLAG_INSURED_ON_APPROVAL
02030495271877Consumer loans1730.43017145.017145.00.017145.0SATURDAY15...Connectivity12.0middlePOS mobile with interest365243.0-42.0300.0-42.0-37.00.0
12802425108129Cash loans25188.615607500.0679671.0NaN607500.0THURSDAY11...XNA36.0low_actionCash X-Sell: low365243.0-134.0916.0365243.0365243.01.0
22523466122040Cash loans15060.735112500.0136444.5NaN112500.0TUESDAY11...XNA12.0highCash X-Sell: high365243.0-271.059.0365243.0365243.01.0
32819243176158Cash loans47041.335450000.0470790.0NaN450000.0MONDAY7...XNA12.0middleCash X-Sell: middle365243.0-482.0-152.0-182.0-177.01.0
41784265202054Cash loans31924.395337500.0404055.0NaN337500.0THURSDAY9...XNA24.0highCash Street: highNaNNaNNaNNaNNaNNaN
\n", "

5 rows × 37 columns

\n", "
" ], "text/plain": [ " SK_ID_PREV SK_ID_CURR NAME_CONTRACT_TYPE AMT_ANNUITY AMT_APPLICATION \\\n", "0 2030495 271877 Consumer loans 1730.430 17145.0 \n", "1 2802425 108129 Cash loans 25188.615 607500.0 \n", "2 2523466 122040 Cash loans 15060.735 112500.0 \n", "3 2819243 176158 Cash loans 47041.335 450000.0 \n", "4 1784265 202054 Cash loans 31924.395 337500.0 \n", "\n", " AMT_CREDIT AMT_DOWN_PAYMENT AMT_GOODS_PRICE WEEKDAY_APPR_PROCESS_START \\\n", "0 17145.0 0.0 17145.0 SATURDAY \n", "1 679671.0 NaN 607500.0 THURSDAY \n", "2 136444.5 NaN 112500.0 TUESDAY \n", "3 470790.0 NaN 450000.0 MONDAY \n", "4 404055.0 NaN 337500.0 THURSDAY \n", "\n", " HOUR_APPR_PROCESS_START ... NAME_SELLER_INDUSTRY CNT_PAYMENT \\\n", "0 15 ... Connectivity 12.0 \n", "1 11 ... XNA 36.0 \n", "2 11 ... XNA 12.0 \n", "3 7 ... XNA 12.0 \n", "4 9 ... XNA 24.0 \n", "\n", " NAME_YIELD_GROUP PRODUCT_COMBINATION DAYS_FIRST_DRAWING \\\n", "0 middle POS mobile with interest 365243.0 \n", "1 low_action Cash X-Sell: low 365243.0 \n", "2 high Cash X-Sell: high 365243.0 \n", "3 middle Cash X-Sell: middle 365243.0 \n", "4 high Cash Street: high NaN \n", "\n", " DAYS_FIRST_DUE DAYS_LAST_DUE_1ST_VERSION DAYS_LAST_DUE DAYS_TERMINATION \\\n", "0 -42.0 300.0 -42.0 -37.0 \n", "1 -134.0 916.0 365243.0 365243.0 \n", "2 -271.0 59.0 365243.0 365243.0 \n", "3 -482.0 -152.0 -182.0 -177.0 \n", "4 NaN NaN NaN NaN \n", "\n", " NFLAG_INSURED_ON_APPROVAL \n", "0 0.0 \n", "1 1.0 \n", "2 1.0 \n", "3 1.0 \n", "4 NaN \n", "\n", "[5 rows x 37 columns]" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "previous_app.head()" ] }, { "cell_type": "code", "execution_count": 25, "id": "ca9ac874", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Début agg numériques\n", "Fin agg numériques\n", "Début nunique cat\n", "Fin nunique cat\n", "Début dummies + groupby cat\n", "Traitement 1/16 : NAME_CONTRACT_TYPE\n", "Traitement 2/16 : WEEKDAY_APPR_PROCESS_START\n", "Traitement 3/16 : FLAG_LAST_APPL_PER_CONTRACT\n", "Traitement 4/16 : NAME_CASH_LOAN_PURPOSE\n", "Traitement 5/16 : NAME_CONTRACT_STATUS\n", "Traitement 6/16 : NAME_PAYMENT_TYPE\n", "Traitement 7/16 : CODE_REJECT_REASON\n", "Traitement 8/16 : NAME_TYPE_SUITE\n", "Traitement 9/16 : NAME_CLIENT_TYPE\n", "Traitement 10/16 : NAME_GOODS_CATEGORY\n", "Traitement 11/16 : NAME_PORTFOLIO\n", "Traitement 12/16 : NAME_PRODUCT_TYPE\n", "Traitement 13/16 : CHANNEL_TYPE\n", "Traitement 14/16 : NAME_SELLER_INDUSTRY\n", "Traitement 15/16 : NAME_YIELD_GROUP\n", "Traitement 16/16 : PRODUCT_COMBINATION\n", "Fin catégoriel\n", "Fusion finale\n", "Agrégation terminée\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_CURRPREV_AMT_ANNUITY_meanPREV_AMT_ANNUITY_maxPREV_AMT_ANNUITY_minPREV_AMT_ANNUITY_sumPREV_AMT_APPLICATION_meanPREV_AMT_APPLICATION_maxPREV_AMT_APPLICATION_minPREV_AMT_APPLICATION_sumPREV_AMT_CREDIT_mean...PREV_PRODUCT_COMBINATION_Cash X-Sell: lowPREV_PRODUCT_COMBINATION_Cash X-Sell: middlePREV_PRODUCT_COMBINATION_POS household with interestPREV_PRODUCT_COMBINATION_POS household without interestPREV_PRODUCT_COMBINATION_POS industry with interestPREV_PRODUCT_COMBINATION_POS industry without interestPREV_PRODUCT_COMBINATION_POS mobile with interestPREV_PRODUCT_COMBINATION_POS mobile without interestPREV_PRODUCT_COMBINATION_POS other with interestPREV_PRODUCT_COMBINATION_POS others without interest
01000013951.0003951.0003951.0003951.00024835.5024835.524835.524835.523787.00...0000001000
11000029251.7759251.7759251.7759251.775179055.00179055.0179055.0179055.0179055.00...0000000010
210000356553.99098356.9956737.310169661.970435436.50900000.068809.51306309.5484191.00...1010100000
31000045357.2505357.2505357.2505357.25024282.0024282.024282.024282.020106.00...0000000100
41000054813.2004813.2004813.2004813.20022308.7544617.50.044617.520076.75...0000001000
\n", "

5 rows × 236 columns

\n", "
" ], "text/plain": [ " SK_ID_CURR PREV_AMT_ANNUITY_mean PREV_AMT_ANNUITY_max \\\n", "0 100001 3951.000 3951.000 \n", "1 100002 9251.775 9251.775 \n", "2 100003 56553.990 98356.995 \n", "3 100004 5357.250 5357.250 \n", "4 100005 4813.200 4813.200 \n", "\n", " PREV_AMT_ANNUITY_min PREV_AMT_ANNUITY_sum PREV_AMT_APPLICATION_mean \\\n", "0 3951.000 3951.000 24835.50 \n", "1 9251.775 9251.775 179055.00 \n", "2 6737.310 169661.970 435436.50 \n", "3 5357.250 5357.250 24282.00 \n", "4 4813.200 4813.200 22308.75 \n", "\n", " PREV_AMT_APPLICATION_max PREV_AMT_APPLICATION_min \\\n", "0 24835.5 24835.5 \n", "1 179055.0 179055.0 \n", "2 900000.0 68809.5 \n", "3 24282.0 24282.0 \n", "4 44617.5 0.0 \n", "\n", " PREV_AMT_APPLICATION_sum PREV_AMT_CREDIT_mean ... \\\n", "0 24835.5 23787.00 ... \n", "1 179055.0 179055.00 ... \n", "2 1306309.5 484191.00 ... \n", "3 24282.0 20106.00 ... \n", "4 44617.5 20076.75 ... \n", "\n", " PREV_PRODUCT_COMBINATION_Cash X-Sell: low \\\n", "0 0 \n", "1 0 \n", "2 1 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_Cash X-Sell: middle \\\n", "0 0 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS household with interest \\\n", "0 0 \n", "1 0 \n", "2 1 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS household without interest \\\n", "0 0 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS industry with interest \\\n", "0 0 \n", "1 0 \n", "2 1 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS industry without interest \\\n", "0 0 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS mobile with interest \\\n", "0 1 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 1 \n", "\n", " PREV_PRODUCT_COMBINATION_POS mobile without interest \\\n", "0 0 \n", "1 0 \n", "2 0 \n", "3 1 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS other with interest \\\n", "0 0 \n", "1 1 \n", "2 0 \n", "3 0 \n", "4 0 \n", "\n", " PREV_PRODUCT_COMBINATION_POS others without interest \n", "0 0 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 0 \n", "\n", "[5 rows x 236 columns]" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "previous_app_agg = agg_fichiers(previous_app, prefix=\"PREV\")\n", "previous_app_agg.head()" ] }, { "cell_type": "code", "execution_count": 26, "id": "f637d70e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(307511, 448) (48744, 447)\n" ] } ], "source": [ "# Fusion avec train et test\n", "app_train_clean2 = app_train_clean1.merge(previous_app_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "app_test_clean2 = app_test_clean1.merge(previous_app_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", "print(app_train_clean2.shape, app_test_clean2.shape)" ] }, { "cell_type": "code", "execution_count": 31, "id": "a547bb41", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Vérification colonnes : test_prev vs train_prev ---\n", "Colonnes attendues : 448\n", "Colonnes trouvées : 447\n", "\n", "❌ Colonnes manquantes dans test_prev :\n", " - TARGET\n", "\n", "✔️ Aucune colonne supplémentaire\n", "\n", "--------------------------------------------\n", "\n" ] } ], "source": [ "# Check des colonnes entre train et test\n", "check_columns(app_train_clean2, app_test_clean2, name_ref=\"train_prev\", name_test=\"test_prev\")" ] }, { "cell_type": "markdown", "id": "eb34b162", "metadata": {}, "source": [ "#### Fusion du fichier installments_payments avec \"train/test\"\n", "Table mesurant le comportement de remboursement réel du client.\n", "L'objectif c'est de pouvoir en obtenir:\n", "- le retard de paiement: retard moyen/max, nombre de retards, % retards\n", "- le comportement de paiement: montant total dû, mt total payé, % payé/dû, paiements partiels??, paiements en avance\n", "- nm total d'échéances, nm de crédits différent, etc\n", "\n", "Ainsi on va procéder, comme pour les fichiers précédents, à une aggrégation des données pour une meilleure intégration dans le train/test car un client peut avoir plusieurs paiements différents" ] }, { "cell_type": "code", "execution_count": 28, "id": "76e57976", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 13605401 entries, 0 to 13605400\n", "Data columns (total 8 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 SK_ID_PREV int64 \n", " 1 SK_ID_CURR int64 \n", " 2 NUM_INSTALMENT_VERSION float64\n", " 3 NUM_INSTALMENT_NUMBER int64 \n", " 4 DAYS_INSTALMENT float64\n", " 5 DAYS_ENTRY_PAYMENT float64\n", " 6 AMT_INSTALMENT float64\n", " 7 AMT_PAYMENT float64\n", "dtypes: float64(5), int64(3)\n", "memory usage: 830.4 MB\n" ] } ], "source": [ "installments_payments = pd.read_csv(\"./Donnees_Brutes/installments_payments.csv\", sep=',')\n", "installments_payments.info()" ] }, { "cell_type": "code", "execution_count": 29, "id": "38253c8c", "metadata": {}, "outputs": [], "source": [ "# Ajout de variables qui pourraient être intéressantes\n", "\n", "# Retard = date réelle - date prévue\n", "installments_payments[\"PAYMENT_DELAY\"] = installments_payments[\"DAYS_ENTRY_PAYMENT\"] - installments_payments[\"DAYS_INSTALMENT\"]\n", "\n", "# Retard positif = paiement en retard\n", "installments_payments[\"LATE_PAYMENT\"] = (installments_payments[\"PAYMENT_DELAY\"] > 0).astype(int)\n", "\n", "# Paiement partiel\n", "installments_payments[\"PARTIAL_PAYMENT\"] = (installments_payments[\"AMT_PAYMENT\"] < installments_payments[\"AMT_INSTALMENT\"]).astype(int)\n", "\n", "# Paiement en avance\n", "installments_payments[\"EARLY_PAYMENT\"] = (installments_payments[\"PAYMENT_DELAY\"] < 0).astype(int)\n", "\n", "# Ratio payé / dû\n", "installments_payments[\"PAYMENT_RATIO\"] = installments_payments[\"AMT_PAYMENT\"] / installments_payments[\"AMT_INSTALMENT\"]\n", "installments_payments[\"PAYMENT_RATIO\"].replace([np.inf, -np.inf], np.nan, inplace=True) # lorsque le ratio est une division avec \"0\", la donnée est remplacée par \"NaN\"" ] }, { "cell_type": "code", "execution_count": 30, "id": "49f9d6b2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Début agg numériques\n", "Fin agg numériques\n", "Début nunique cat\n", "Fin nunique cat\n", "Début dummies + groupby cat\n", "Fin catégoriel\n", "Fusion finale\n", "Agrégation terminée\n", "\n", "RangeIndex: 339587 entries, 0 to 339586\n", "Data columns (total 33 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 339587 non-null int64 \n", " 1 INST_PAY_NUM_INSTALMENT_VERSION_mean 339587 non-null float64\n", " 2 INST_PAY_NUM_INSTALMENT_VERSION_max 339587 non-null float64\n", " 3 INST_PAY_NUM_INSTALMENT_VERSION_min 339587 non-null float64\n", " 4 INST_PAY_NUM_INSTALMENT_VERSION_sum 339587 non-null float64\n", " 5 INST_PAY_NUM_INSTALMENT_NUMBER_mean 339587 non-null float64\n", " 6 INST_PAY_NUM_INSTALMENT_NUMBER_max 339587 non-null int64 \n", " 7 INST_PAY_NUM_INSTALMENT_NUMBER_min 339587 non-null int64 \n", " 8 INST_PAY_NUM_INSTALMENT_NUMBER_sum 339587 non-null int64 \n", " 9 INST_PAY_DAYS_INSTALMENT_mean 339587 non-null float64\n", " 10 INST_PAY_DAYS_INSTALMENT_max 339587 non-null float64\n", " 11 INST_PAY_DAYS_INSTALMENT_min 339587 non-null float64\n", " 12 INST_PAY_DAYS_INSTALMENT_sum 339587 non-null float64\n", " 13 INST_PAY_DAYS_ENTRY_PAYMENT_mean 339578 non-null float64\n", " 14 INST_PAY_DAYS_ENTRY_PAYMENT_max 339578 non-null float64\n", " 15 INST_PAY_DAYS_ENTRY_PAYMENT_min 339578 non-null float64\n", " 16 INST_PAY_DAYS_ENTRY_PAYMENT_sum 339587 non-null float64\n", " 17 INST_PAY_AMT_INSTALMENT_mean 339587 non-null float64\n", " 18 INST_PAY_AMT_INSTALMENT_max 339587 non-null float64\n", " 19 INST_PAY_AMT_INSTALMENT_min 339587 non-null float64\n", " 20 INST_PAY_AMT_INSTALMENT_sum 339587 non-null float64\n", " 21 INST_PAY_AMT_PAYMENT_mean 339578 non-null float64\n", " 22 INST_PAY_AMT_PAYMENT_max 339578 non-null float64\n", " 23 INST_PAY_AMT_PAYMENT_min 339578 non-null float64\n", " 24 INST_PAY_AMT_PAYMENT_sum 339587 non-null float64\n", " 25 INST_PAY_PAYMENT_DELAY_mean 339578 non-null float64\n", " 26 INST_PAY_PAYMENT_DELAY_max 339578 non-null float64\n", " 27 INST_PAY_PAYMENT_DELAY_min 339578 non-null float64\n", " 28 INST_PAY_PAYMENT_DELAY_sum 339587 non-null float64\n", " 29 INST_PAY_PAYMENT_RATIO_mean 339575 non-null float64\n", " 30 INST_PAY_PAYMENT_RATIO_max 339575 non-null float64\n", " 31 INST_PAY_PAYMENT_RATIO_min 339575 non-null float64\n", " 32 INST_PAY_PAYMENT_RATIO_sum 339587 non-null float64\n", "dtypes: float64(29), int64(4)\n", "memory usage: 85.5 MB\n" ] } ], "source": [ "inst_payments_agg = agg_fichiers(installments_payments, prefix=\"INST_PAY\")\n", "inst_payments_agg.info()\n" ] }, { "cell_type": "code", "execution_count": 33, "id": "ff75f236", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(307511, 480) (48744, 479)\n" ] } ], "source": [ "# Fusion avec train/test\n", "app_train_clean3 = app_train_clean2.merge(inst_payments_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "app_test_clean3 = app_test_clean2.merge(inst_payments_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", "print(app_train_clean3.shape, app_test_clean3.shape)" ] }, { "cell_type": "code", "execution_count": 34, "id": "5f649bab", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Vérification colonnes : test_instal vs train_install ---\n", "Colonnes attendues : 480\n", "Colonnes trouvées : 479\n", "\n", "❌ Colonnes manquantes dans test_instal :\n", " - TARGET\n", "\n", "✔️ Aucune colonne supplémentaire\n", "\n", "--------------------------------------------\n", "\n" ] } ], "source": [ "# Check des colonnes entre train et test\n", "check_columns(app_train_clean3, app_test_clean3, name_ref=\"train_install\", name_test=\"test_instal\")" ] }, { "cell_type": "markdown", "id": "9aa95d89", "metadata": {}, "source": [ "#### Fusion de credit_card_balance & pos_cash_balance avec \"train/test\"" ] }, { "cell_type": "code", "execution_count": 35, "id": "4caa5e70", "metadata": {}, "outputs": [], "source": [ "# Chargement des données credit card est cash balance\n", "credit_card_balance = pd.read_csv(\"./Donnees_Brutes/credit_card_balance.csv\", sep=',')\n", "pos_cash_balance = pd.read_csv(\"./Donnees_Brutes/POS_CASH_balance.csv\", sep=',')" ] }, { "cell_type": "code", "execution_count": 36, "id": "8aa3ab73", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Credit card ---\n", "\n", "RangeIndex: 3840312 entries, 0 to 3840311\n", "Data columns (total 23 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 SK_ID_PREV int64 \n", " 1 SK_ID_CURR int64 \n", " 2 MONTHS_BALANCE int64 \n", " 3 AMT_BALANCE float64\n", " 4 AMT_CREDIT_LIMIT_ACTUAL int64 \n", " 5 AMT_DRAWINGS_ATM_CURRENT float64\n", " 6 AMT_DRAWINGS_CURRENT float64\n", " 7 AMT_DRAWINGS_OTHER_CURRENT float64\n", " 8 AMT_DRAWINGS_POS_CURRENT float64\n", " 9 AMT_INST_MIN_REGULARITY float64\n", " 10 AMT_PAYMENT_CURRENT float64\n", " 11 AMT_PAYMENT_TOTAL_CURRENT float64\n", " 12 AMT_RECEIVABLE_PRINCIPAL float64\n", " 13 AMT_RECIVABLE float64\n", " 14 AMT_TOTAL_RECEIVABLE float64\n", " 15 CNT_DRAWINGS_ATM_CURRENT float64\n", " 16 CNT_DRAWINGS_CURRENT int64 \n", " 17 CNT_DRAWINGS_OTHER_CURRENT float64\n", " 18 CNT_DRAWINGS_POS_CURRENT float64\n", " 19 CNT_INSTALMENT_MATURE_CUM float64\n", " 20 NAME_CONTRACT_STATUS object \n", " 21 SK_DPD int64 \n", " 22 SK_DPD_DEF int64 \n", "dtypes: float64(15), int64(7), object(1)\n", "memory usage: 673.9+ MB\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_PREVSK_ID_CURRMONTHS_BALANCEAMT_BALANCEAMT_CREDIT_LIMIT_ACTUALAMT_DRAWINGS_ATM_CURRENTAMT_DRAWINGS_CURRENTAMT_DRAWINGS_OTHER_CURRENTAMT_DRAWINGS_POS_CURRENTAMT_INST_MIN_REGULARITY...AMT_RECIVABLEAMT_TOTAL_RECEIVABLECNT_DRAWINGS_ATM_CURRENTCNT_DRAWINGS_CURRENTCNT_DRAWINGS_OTHER_CURRENTCNT_DRAWINGS_POS_CURRENTCNT_INSTALMENT_MATURE_CUMNAME_CONTRACT_STATUSSK_DPDSK_DPD_DEF
02562384378907-656.9701350000.0877.50.0877.51700.325...0.0000.0000.010.01.035.0Active00
12582071363914-163975.555450002250.02250.00.00.02250.000...64875.55564875.5551.010.00.069.0Active00
21740877371185-731815.2254500000.00.00.00.02250.000...31460.08531460.0850.000.00.030.0Active00
31389973337855-4236572.1102250002250.02250.00.00.011795.760...233048.970233048.9701.010.00.010.0Active00
41891521126868-1453919.4554500000.011547.00.011547.022924.890...453919.455453919.4550.010.01.0101.0Active00
\n", "

5 rows × 23 columns

\n", "
" ], "text/plain": [ " SK_ID_PREV SK_ID_CURR MONTHS_BALANCE AMT_BALANCE \\\n", "0 2562384 378907 -6 56.970 \n", "1 2582071 363914 -1 63975.555 \n", "2 1740877 371185 -7 31815.225 \n", "3 1389973 337855 -4 236572.110 \n", "4 1891521 126868 -1 453919.455 \n", "\n", " AMT_CREDIT_LIMIT_ACTUAL AMT_DRAWINGS_ATM_CURRENT AMT_DRAWINGS_CURRENT \\\n", "0 135000 0.0 877.5 \n", "1 45000 2250.0 2250.0 \n", "2 450000 0.0 0.0 \n", "3 225000 2250.0 2250.0 \n", "4 450000 0.0 11547.0 \n", "\n", " AMT_DRAWINGS_OTHER_CURRENT AMT_DRAWINGS_POS_CURRENT \\\n", "0 0.0 877.5 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 0.0 11547.0 \n", "\n", " AMT_INST_MIN_REGULARITY ... AMT_RECIVABLE AMT_TOTAL_RECEIVABLE \\\n", "0 1700.325 ... 0.000 0.000 \n", "1 2250.000 ... 64875.555 64875.555 \n", "2 2250.000 ... 31460.085 31460.085 \n", "3 11795.760 ... 233048.970 233048.970 \n", "4 22924.890 ... 453919.455 453919.455 \n", "\n", " CNT_DRAWINGS_ATM_CURRENT CNT_DRAWINGS_CURRENT CNT_DRAWINGS_OTHER_CURRENT \\\n", "0 0.0 1 0.0 \n", "1 1.0 1 0.0 \n", "2 0.0 0 0.0 \n", "3 1.0 1 0.0 \n", "4 0.0 1 0.0 \n", "\n", " CNT_DRAWINGS_POS_CURRENT CNT_INSTALMENT_MATURE_CUM NAME_CONTRACT_STATUS \\\n", "0 1.0 35.0 Active \n", "1 0.0 69.0 Active \n", "2 0.0 30.0 Active \n", "3 0.0 10.0 Active \n", "4 1.0 101.0 Active \n", "\n", " SK_DPD SK_DPD_DEF \n", "0 0 0 \n", "1 0 0 \n", "2 0 0 \n", "3 0 0 \n", "4 0 0 \n", "\n", "[5 rows x 23 columns]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "--- Résumé Cash balance ---\n", "\n", "RangeIndex: 10001358 entries, 0 to 10001357\n", "Data columns (total 8 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 SK_ID_PREV int64 \n", " 1 SK_ID_CURR int64 \n", " 2 MONTHS_BALANCE int64 \n", " 3 CNT_INSTALMENT float64\n", " 4 CNT_INSTALMENT_FUTURE float64\n", " 5 NAME_CONTRACT_STATUS object \n", " 6 SK_DPD int64 \n", " 7 SK_DPD_DEF int64 \n", "dtypes: float64(2), int64(5), object(1)\n", "memory usage: 610.4+ MB\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SK_ID_PREVSK_ID_CURRMONTHS_BALANCECNT_INSTALMENTCNT_INSTALMENT_FUTURENAME_CONTRACT_STATUSSK_DPDSK_DPD_DEF
01803195182943-3148.045.0Active00
11715348367990-3336.035.0Active00
21784872397406-3212.09.0Active00
31903291269225-3548.042.0Active00
42341044334279-3536.035.0Active00
\n", "
" ], "text/plain": [ " SK_ID_PREV SK_ID_CURR MONTHS_BALANCE CNT_INSTALMENT \\\n", "0 1803195 182943 -31 48.0 \n", "1 1715348 367990 -33 36.0 \n", "2 1784872 397406 -32 12.0 \n", "3 1903291 269225 -35 48.0 \n", "4 2341044 334279 -35 36.0 \n", "\n", " CNT_INSTALMENT_FUTURE NAME_CONTRACT_STATUS SK_DPD SK_DPD_DEF \n", "0 45.0 Active 0 0 \n", "1 35.0 Active 0 0 \n", "2 9.0 Active 0 0 \n", "3 42.0 Active 0 0 \n", "4 35.0 Active 0 0 " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "None" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Vue globale des données\n", "pd.set_option(\"display.max_info_columns\", 200)\n", "jeu_donnees = [credit_card_balance, pos_cash_balance]\n", "noms_ = [\"Credit card\", \"Cash balance\"]\n", "for nom, df in zip(noms_, jeu_donnees):\n", " print(f\"\\n--- Résumé {nom} ---\")\n", " display(df.head(), df.info())" ] }, { "cell_type": "code", "execution_count": 37, "id": "ab4718c4", "metadata": {}, "outputs": [], "source": [ "# Ajout de colonnes dérivées dans la table credit_card\n", "credit_card_balance[\"UTILIZATION\"] = credit_card_balance[\"AMT_BALANCE\"] / credit_card_balance[\"AMT_CREDIT_LIMIT_ACTUAL\"]\n", "credit_card_balance[\"UTILIZATION\"].replace([np.inf, -np.inf], np.nan, inplace=True)\n", "\n", "credit_card_balance[\"PAYMENT_RATIO\"] = credit_card_balance[\"AMT_PAYMENT_TOTAL_CURRENT\"] / credit_card_balance[\"AMT_DRAWINGS_CURRENT\"]\n", "credit_card_balance[\"PAYMENT_RATIO\"].replace([np.inf, -np.inf], np.nan, inplace=True)" ] }, { "cell_type": "code", "execution_count": 38, "id": "102b5d49", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Début agg numériques\n", "Fin agg numériques\n", "Début nunique cat\n", "Fin nunique cat\n", "Début dummies + groupby cat\n", "Traitement 1/1 : NAME_CONTRACT_STATUS\n", "Fin catégoriel\n", "Fusion finale\n", "Agrégation terminée\n", "\n", "RangeIndex: 103558 entries, 0 to 103557\n", "Data columns (total 97 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 103558 non-null int64 \n", " 1 CC_MONTHS_BALANCE_mean 103558 non-null float64\n", " 2 CC_MONTHS_BALANCE_max 103558 non-null int64 \n", " 3 CC_MONTHS_BALANCE_min 103558 non-null int64 \n", " 4 CC_MONTHS_BALANCE_sum 103558 non-null int64 \n", " 5 CC_AMT_BALANCE_mean 103558 non-null float64\n", " 6 CC_AMT_BALANCE_max 103558 non-null float64\n", " 7 CC_AMT_BALANCE_min 103558 non-null float64\n", " 8 CC_AMT_BALANCE_sum 103558 non-null float64\n", " 9 CC_AMT_CREDIT_LIMIT_ACTUAL_mean 103558 non-null float64\n", " 10 CC_AMT_CREDIT_LIMIT_ACTUAL_max 103558 non-null int64 \n", " 11 CC_AMT_CREDIT_LIMIT_ACTUAL_min 103558 non-null int64 \n", " 12 CC_AMT_CREDIT_LIMIT_ACTUAL_sum 103558 non-null int64 \n", " 13 CC_AMT_DRAWINGS_ATM_CURRENT_mean 72194 non-null float64\n", " 14 CC_AMT_DRAWINGS_ATM_CURRENT_max 72194 non-null float64\n", " 15 CC_AMT_DRAWINGS_ATM_CURRENT_min 72194 non-null float64\n", " 16 CC_AMT_DRAWINGS_ATM_CURRENT_sum 103558 non-null float64\n", " 17 CC_AMT_DRAWINGS_CURRENT_mean 103558 non-null float64\n", " 18 CC_AMT_DRAWINGS_CURRENT_max 103558 non-null float64\n", " 19 CC_AMT_DRAWINGS_CURRENT_min 103558 non-null float64\n", " 20 CC_AMT_DRAWINGS_CURRENT_sum 103558 non-null float64\n", " 21 CC_AMT_DRAWINGS_OTHER_CURRENT_mean 72194 non-null float64\n", " 22 CC_AMT_DRAWINGS_OTHER_CURRENT_max 72194 non-null float64\n", " 23 CC_AMT_DRAWINGS_OTHER_CURRENT_min 72194 non-null float64\n", " 24 CC_AMT_DRAWINGS_OTHER_CURRENT_sum 103558 non-null float64\n", " 25 CC_AMT_DRAWINGS_POS_CURRENT_mean 72194 non-null float64\n", " 26 CC_AMT_DRAWINGS_POS_CURRENT_max 72194 non-null float64\n", " 27 CC_AMT_DRAWINGS_POS_CURRENT_min 72194 non-null float64\n", " 28 CC_AMT_DRAWINGS_POS_CURRENT_sum 103558 non-null float64\n", " 29 CC_AMT_INST_MIN_REGULARITY_mean 103558 non-null float64\n", " 30 CC_AMT_INST_MIN_REGULARITY_max 103558 non-null float64\n", " 31 CC_AMT_INST_MIN_REGULARITY_min 103558 non-null float64\n", " 32 CC_AMT_INST_MIN_REGULARITY_sum 103558 non-null float64\n", " 33 CC_AMT_PAYMENT_CURRENT_mean 72120 non-null float64\n", " 34 CC_AMT_PAYMENT_CURRENT_max 72120 non-null float64\n", " 35 CC_AMT_PAYMENT_CURRENT_min 72120 non-null float64\n", " 36 CC_AMT_PAYMENT_CURRENT_sum 103558 non-null float64\n", " 37 CC_AMT_PAYMENT_TOTAL_CURRENT_mean 103558 non-null float64\n", " 38 CC_AMT_PAYMENT_TOTAL_CURRENT_max 103558 non-null float64\n", " 39 CC_AMT_PAYMENT_TOTAL_CURRENT_min 103558 non-null float64\n", " 40 CC_AMT_PAYMENT_TOTAL_CURRENT_sum 103558 non-null float64\n", " 41 CC_AMT_RECEIVABLE_PRINCIPAL_mean 103558 non-null float64\n", " 42 CC_AMT_RECEIVABLE_PRINCIPAL_max 103558 non-null float64\n", " 43 CC_AMT_RECEIVABLE_PRINCIPAL_min 103558 non-null float64\n", " 44 CC_AMT_RECEIVABLE_PRINCIPAL_sum 103558 non-null float64\n", " 45 CC_AMT_RECIVABLE_mean 103558 non-null float64\n", " 46 CC_AMT_RECIVABLE_max 103558 non-null float64\n", " 47 CC_AMT_RECIVABLE_min 103558 non-null float64\n", " 48 CC_AMT_RECIVABLE_sum 103558 non-null float64\n", " 49 CC_AMT_TOTAL_RECEIVABLE_mean 103558 non-null float64\n", " 50 CC_AMT_TOTAL_RECEIVABLE_max 103558 non-null float64\n", " 51 CC_AMT_TOTAL_RECEIVABLE_min 103558 non-null float64\n", " 52 CC_AMT_TOTAL_RECEIVABLE_sum 103558 non-null float64\n", " 53 CC_CNT_DRAWINGS_ATM_CURRENT_mean 72194 non-null float64\n", " 54 CC_CNT_DRAWINGS_ATM_CURRENT_max 72194 non-null float64\n", " 55 CC_CNT_DRAWINGS_ATM_CURRENT_min 72194 non-null float64\n", " 56 CC_CNT_DRAWINGS_ATM_CURRENT_sum 103558 non-null float64\n", " 57 CC_CNT_DRAWINGS_CURRENT_mean 103558 non-null float64\n", " 58 CC_CNT_DRAWINGS_CURRENT_max 103558 non-null int64 \n", " 59 CC_CNT_DRAWINGS_CURRENT_min 103558 non-null int64 \n", " 60 CC_CNT_DRAWINGS_CURRENT_sum 103558 non-null int64 \n", " 61 CC_CNT_DRAWINGS_OTHER_CURRENT_mean 72194 non-null float64\n", " 62 CC_CNT_DRAWINGS_OTHER_CURRENT_max 72194 non-null float64\n", " 63 CC_CNT_DRAWINGS_OTHER_CURRENT_min 72194 non-null float64\n", " 64 CC_CNT_DRAWINGS_OTHER_CURRENT_sum 103558 non-null float64\n", " 65 CC_CNT_DRAWINGS_POS_CURRENT_mean 72194 non-null float64\n", " 66 CC_CNT_DRAWINGS_POS_CURRENT_max 72194 non-null float64\n", " 67 CC_CNT_DRAWINGS_POS_CURRENT_min 72194 non-null float64\n", " 68 CC_CNT_DRAWINGS_POS_CURRENT_sum 103558 non-null float64\n", " 69 CC_CNT_INSTALMENT_MATURE_CUM_mean 103558 non-null float64\n", " 70 CC_CNT_INSTALMENT_MATURE_CUM_max 103558 non-null float64\n", " 71 CC_CNT_INSTALMENT_MATURE_CUM_min 103558 non-null float64\n", " 72 CC_CNT_INSTALMENT_MATURE_CUM_sum 103558 non-null float64\n", " 73 CC_SK_DPD_mean 103558 non-null float64\n", " 74 CC_SK_DPD_max 103558 non-null int64 \n", " 75 CC_SK_DPD_min 103558 non-null int64 \n", " 76 CC_SK_DPD_sum 103558 non-null int64 \n", " 77 CC_SK_DPD_DEF_mean 103558 non-null float64\n", " 78 CC_SK_DPD_DEF_max 103558 non-null int64 \n", " 79 CC_SK_DPD_DEF_min 103558 non-null int64 \n", " 80 CC_SK_DPD_DEF_sum 103558 non-null int64 \n", " 81 CC_UTILIZATION_mean 102445 non-null float64\n", " 82 CC_UTILIZATION_max 102445 non-null float64\n", " 83 CC_UTILIZATION_min 102445 non-null float64\n", " 84 CC_UTILIZATION_sum 103558 non-null float64\n", " 85 CC_PAYMENT_RATIO_mean 70265 non-null float64\n", " 86 CC_PAYMENT_RATIO_max 70265 non-null float64\n", " 87 CC_PAYMENT_RATIO_min 70265 non-null float64\n", " 88 CC_PAYMENT_RATIO_sum 103558 non-null float64\n", " 89 CC_NAME_CONTRACT_STATUS_NUNIQUE 103558 non-null int64 \n", " 90 CC_NAME_CONTRACT_STATUS_Active 103558 non-null int64 \n", " 91 CC_NAME_CONTRACT_STATUS_Approved 103558 non-null int64 \n", " 92 CC_NAME_CONTRACT_STATUS_Completed 103558 non-null int64 \n", " 93 CC_NAME_CONTRACT_STATUS_Demand 103558 non-null int64 \n", " 94 CC_NAME_CONTRACT_STATUS_Refused 103558 non-null int64 \n", " 95 CC_NAME_CONTRACT_STATUS_Sent proposal 103558 non-null int64 \n", " 96 CC_NAME_CONTRACT_STATUS_Signed 103558 non-null int64 \n", "dtypes: float64(73), int64(24)\n", "memory usage: 76.6 MB\n" ] } ], "source": [ "# Application de la fonction d'aggrégation pour cash balance\n", "cc_balance_agg = agg_fichiers(credit_card_balance, prefix=\"CC\")\n", "cc_balance_agg.info()\n" ] }, { "cell_type": "code", "execution_count": 39, "id": "aaecbcd0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(307511, 576) (48744, 575)\n" ] } ], "source": [ "# Fusion avec train/test\n", "app_train_clean4 = app_train_clean3.merge(cc_balance_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "app_test_clean4 = app_test_clean3.merge(cc_balance_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", "print(app_train_clean4.shape, app_test_clean4.shape)" ] }, { "cell_type": "code", "execution_count": 40, "id": "a3c96594", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Vérification colonnes : test_cc vs train_cc ---\n", "Colonnes attendues : 576\n", "Colonnes trouvées : 575\n", "\n", "❌ Colonnes manquantes dans test_cc :\n", " - TARGET\n", "\n", "✔️ Aucune colonne supplémentaire\n", "\n", "--------------------------------------------\n", "\n" ] } ], "source": [ "# Check des colonnes entre train et test\n", "check_columns(app_train_clean4, app_test_clean4, name_ref=\"train_cc\", name_test=\"test_cc\")" ] }, { "cell_type": "code", "execution_count": 41, "id": "0a88d8d4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Début agg numériques\n", "Fin agg numériques\n", "Début nunique cat\n", "Fin nunique cat\n", "Début dummies + groupby cat\n", "Traitement 1/1 : NAME_CONTRACT_STATUS\n", "Fin catégoriel\n", "Fusion finale\n", "Agrégation terminée\n", "\n", "RangeIndex: 337252 entries, 0 to 337251\n", "Data columns (total 31 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 SK_ID_CURR 337252 non-null int64 \n", " 1 POS_MONTHS_BALANCE_mean 337252 non-null float64\n", " 2 POS_MONTHS_BALANCE_max 337252 non-null int64 \n", " 3 POS_MONTHS_BALANCE_min 337252 non-null int64 \n", " 4 POS_MONTHS_BALANCE_sum 337252 non-null int64 \n", " 5 POS_CNT_INSTALMENT_mean 337224 non-null float64\n", " 6 POS_CNT_INSTALMENT_max 337224 non-null float64\n", " 7 POS_CNT_INSTALMENT_min 337224 non-null float64\n", " 8 POS_CNT_INSTALMENT_sum 337252 non-null float64\n", " 9 POS_CNT_INSTALMENT_FUTURE_mean 337224 non-null float64\n", " 10 POS_CNT_INSTALMENT_FUTURE_max 337224 non-null float64\n", " 11 POS_CNT_INSTALMENT_FUTURE_min 337224 non-null float64\n", " 12 POS_CNT_INSTALMENT_FUTURE_sum 337252 non-null float64\n", " 13 POS_SK_DPD_mean 337252 non-null float64\n", " 14 POS_SK_DPD_max 337252 non-null int64 \n", " 15 POS_SK_DPD_min 337252 non-null int64 \n", " 16 POS_SK_DPD_sum 337252 non-null int64 \n", " 17 POS_SK_DPD_DEF_mean 337252 non-null float64\n", " 18 POS_SK_DPD_DEF_max 337252 non-null int64 \n", " 19 POS_SK_DPD_DEF_min 337252 non-null int64 \n", " 20 POS_SK_DPD_DEF_sum 337252 non-null int64 \n", " 21 POS_NAME_CONTRACT_STATUS_NUNIQUE 337252 non-null int64 \n", " 22 POS_NAME_CONTRACT_STATUS_Active 337252 non-null int64 \n", " 23 POS_NAME_CONTRACT_STATUS_Amortized debt 337252 non-null int64 \n", " 24 POS_NAME_CONTRACT_STATUS_Approved 337252 non-null int64 \n", " 25 POS_NAME_CONTRACT_STATUS_Canceled 337252 non-null int64 \n", " 26 POS_NAME_CONTRACT_STATUS_Completed 337252 non-null int64 \n", " 27 POS_NAME_CONTRACT_STATUS_Demand 337252 non-null int64 \n", " 28 POS_NAME_CONTRACT_STATUS_Returned to the store 337252 non-null int64 \n", " 29 POS_NAME_CONTRACT_STATUS_Signed 337252 non-null int64 \n", " 30 POS_NAME_CONTRACT_STATUS_XNA 337252 non-null int64 \n", "dtypes: float64(11), int64(20)\n", "memory usage: 79.8 MB\n" ] } ], "source": [ "# Application de la fonction d'aggrégation pour cash balance\n", "pos_cash_balance_agg = agg_fichiers(pos_cash_balance, prefix=\"POS\")\n", "pos_cash_balance_agg.info()\n" ] }, { "cell_type": "code", "execution_count": 42, "id": "cf4761e9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(307511, 606) (48744, 605)\n" ] } ], "source": [ "# Fusion avec train/test\n", "app_train_clean5 = app_train_clean4.merge(pos_cash_balance_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "app_test_clean5 = app_test_clean4.merge(pos_cash_balance_agg, on=\"SK_ID_CURR\", how=\"left\")\n", "\n", "print(app_train_clean5.shape, app_test_clean5.shape)" ] }, { "cell_type": "code", "execution_count": 43, "id": "fb4786f4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Vérification colonnes : test_pos vs train_pos ---\n", "Colonnes attendues : 606\n", "Colonnes trouvées : 605\n", "\n", "❌ Colonnes manquantes dans test_pos :\n", " - TARGET\n", "\n", "✔️ Aucune colonne supplémentaire\n", "\n", "--------------------------------------------\n", "\n" ] } ], "source": [ "# Check des colonnes entre train et test\n", "check_columns(app_train_clean5, app_test_clean5, name_ref=\"train_pos\", name_test=\"test_pos\")" ] }, { "cell_type": "code", "execution_count": 44, "id": "859a4a18", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Colonnes dans train mais pas dans test : set()\n", "Colonnes dans test mais pas dans train : set()\n" ] } ], "source": [ "# Vérifier que les colonnes entre train et test sont bien identiques mis à part \"TARGET\"\n", "train_cols = set(app_train_clean5.columns) - {\"TARGET\"}\n", "test_cols = set(app_test_clean5.columns)\n", "\n", "diff_train = train_cols - test_cols\n", "diff_test = test_cols - train_cols\n", "\n", "print(\"Colonnes dans train mais pas dans test :\", diff_train)\n", "print(\"Colonnes dans test mais pas dans train :\", diff_test)\n" ] }, { "cell_type": "code", "execution_count": 36, "id": "b97cbc6b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['data/app_test_clean5.joblib']" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "joblib.dump(app_train_clean5, \"data/app_train_clean5.joblib\")\n", "joblib.dump(app_test_clean5, \"data/app_test_clean5.joblib\")\n" ] } ], "metadata": { "kernelspec": { "display_name": "mlops", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 5 }