add year matches ingestion endpoint

src/main.py

@@ -36,7 +36,11 @@ from src.entity.player import (
     PlayerApiDetail,
 )
 from src.repository.common import get_session
-from src.service.match import insert_new_match
+from src.service.match import (
+    insert_new_match,
+    fetch_raw_data,
+    get_cleaned_data,
+)
 
 from contextlib import asynccontextmanager
 from src.api_factory import create_forward_endpoint, get_remote_params
@@ -263,7 +267,8 @@ async def get_match(
 @app.post("/match/insert", tags=["match"], description="Insert a match into the database")
 async def insert_match(
     raw_match: RawMatch,
-    session: Session = Depends(provide_connection)
+    session: Session = Depends(provide_connection),
+    on_conflict_do_nothing: bool = False,
 ):
     """
     Insert a match into the database
@@ -271,7 +276,8 @@ async def insert_match(
     try:
         match = insert_new_match(
             db=session,
-            raw_match=raw_match.model_dump(exclude_unset=True)
+            raw_match=raw_match.model_dump(exclude_unset=True),
+            on_conflict_do_nothing=on_conflict_do_nothing,
         )
     except IntegrityError as e:
         logger.error(f"Error inserting match: {e}")
@@ -282,15 +288,42 @@ async def insert_match(
 
     output = {
         "status": "ok",
-        "match_id": match.id,
+        "match_id": match.id if match else None,
     }
 
     return JSONResponse(content=output, status_code=HTTP_200_OK)
 
+@app.post("/match/ingest_year", tags=["match"], description="Ingest matches from tennis-data.co.uk for a given year")
+async def ingest_matches(
+    year: Optional[int] = None,
+    session: Session = Depends(provide_connection)
+):
+    """
+    Ingest matches from tennis-data.co.uk for a given year
+    """
+    fetch_raw_data(db=session, year=year)
+    # Get the cleaned data
+    df = get_cleaned_data(year=year)
+    
+    # Send requests of 100 matches
+    for i in range(0, len(df), 100):
+        start = i
+        end = start + 99
+        df_small = df.loc[start:end]
+        
+        response = await insert_batch_match(
+            raw_matches=df_small.to_dict(orient='records'),
+            on_conflict_do_nothing=True,
+            session=session
+        )
+
+        if response.status_code != HTTP_200_OK:
+            logger.error(f"Batch insert failed: {response.status_code} - {response.text}")
 
 @app.post("/batch/match/insert", tags=["match"], description="Insert a batch of matches into the database")
 async def insert_batch_match(
     raw_matches: list[RawMatch],
+    on_conflict_do_nothing: bool = False,
     session: Session = Depends(provide_connection)
 ):
     """
@@ -302,7 +335,8 @@ async def insert_batch_match(
         try:
             match = insert_new_match(
                 db=session,
-                raw_match=raw_match.model_dump(exclude_unset=True)
+                raw_match=raw_match.model_dump(exclude_unset=True) if isinstance(raw_match, RawMatch) else raw_match,
+                on_conflict_do_nothing=on_conflict_do_nothing,
             )
             matches.append(match)
         except IntegrityError as e:

src/service/match.py

@@ -1,7 +1,13 @@
+import os
+import logging
+import requests
+import numpy as np
+import pandas as pd
 from typing import Dict, List, Tuple
+from datetime import datetime
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
-import logging
+
 from src.entity.match import Match
 from src.entity.odds import Odds
 from src.entity.player import Player
@@ -119,3 +125,52 @@ def _should_fetch_details(player: Player) -> bool:
     Check if player details should be fetched
     """
     return player.tennis_id is None or player.caracteristics is None
+
+def fetch_raw_data(db: Session, year: int) -> None:
+    """
+    Fetch data from tennis-data.co.uk for a given year and circuit (ATP or WTA) and save it to a file
+
+    Args:
+        year (int, optional): Year to retrieve. If None, fetch current year data.
+    """
+    current_year = datetime.now().year
+    
+    if not year:
+        year = current_year
+    
+    filename = f"{year}.xlsx"
+    file_path = f"./data/atp/{filename}"
+
+    # Check if the file already exists
+    if os.path.exists(file_path) and year != current_year:
+        logging.info(f"File {file_path} already exists. Skipping download.")
+        return
+
+    logging.info(f"Fetching data from tennis-data.co.uk for year {year}")
+
+    url = f"http://www.tennis-data.co.uk/{year}/{filename}"
+    
+    response = requests.get(url, stream=True)
+
+    # Check response status code
+    response.raise_for_status()
+
+    with open(file_path, "wb") as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+        file.flush()
+
+    logging.info(f"Data fetched from {url} 👍 and saved to {file_path}")
+
+def get_cleaned_data(year: int) -> pd.DataFrame:
+    df = pd.read_csv(f'./data/atp/{year}.csv')
+    # Remove rows where LRank or WRank is NaN
+    df = df.dropna(subset=['LRank', 'WRank'])
+    df['Lsets'] = df['Lsets'].fillna(0)
+    df['Wsets'] = df['Wsets'].fillna(0)
+    # Replace NaN values with None
+    df = df.replace({np.nan: None})
+    # Replace NaN values with None
+    df = df.where(pd.notnull(df), None)
+
+    return df