traning_zone/train_zone/entrainement_big_model.py

from traning_zone.traitement_data.feature_engeneering.data_clearning import *
from traning_zone.traitement_data.feature_engeneering.feature_Engineering import *
from traning_zone.modele_zone.modeles import *
from traning_zone.modele_zone.modeles_gridsearch import *
from traning_zone.modele_zone.model import *
import time
import yaml

from dotenv import load_dotenv
import os
load_dotenv()
project_id = os.getenv('PROJECT_ID')


data = pd.read_gbq("SELECT * FROM `c4-group-datagov-dev.classification_refbem.data_big_model`",project_id=project_id)

try :
     data.rename(columns={"ITEM_DESC" : "DESCRIPTION"}, inplace=True)
except :
     pass

start_time = time.time()

df = clearning_modele(data)
tv_xtrain, tv_xtest,  Ytrain, Ytest = engineering_modele(df)

end_time = time.time()
print(f"Temps d'exécution du pré-traitement est : {end_time - start_time} secondes")

for name in modeles.keys():
    try :
          start_time = time.time()
          trainer_modele(tv_xtrain, Ytrain, tv_xtest, Ytest, modeles[name], name)
          end_time = time.time()
          print(f"Temps d'exécution du d'apprentissage du modèle {name} est : {end_time - start_time} secondes")
    except :
          print(f"Erreur lors de l'apprentissage du modèle {name}")