download_data.R

library(httr)

token    <- Sys.getenv("HF_TOKEN")
repo     <- "BU-Baseball/BU-Baseball-Data"
base_url <- paste0("https://huggingface.co/datasets/", repo, "/resolve/main/")

download_hf <- function(filename) {
  dest <- paste0("/app/", filename)
  if (file.exists(dest)) {
    message(filename, " already exists, skipping.")
    return(invisible(NULL))
  }
  message("Downloading ", filename, "...")
  resp <- GET(
    paste0(base_url, filename),
    add_headers(Authorization = paste("Bearer", token)),
    write_disk(dest, overwrite = TRUE),
    progress()
  )
  if (status_code(resp) != 200)
    stop("Failed to download ", filename, " — status ", status_code(resp))
  message(filename, " done.")
}

files <- c(
  "D1.parquet",
  "D2.parquet",
  "D3.parquet",
  "JUCO.parquet",
  "NAIA.parquet",
  "D1_scores.parquet",
  "D2_scores.parquet",
  "D3_scores.parquet",
  "JUCO_scores.parquet",
  "NAIA_scores.parquet",
  "transfer_portal.parquet",
  "hitter_portal.parquet",
  "D1_pitcher_leaderboard.parquet",
  "D2_pitcher_leaderboard.parquet",
  "D3_pitcher_leaderboard.parquet",
  "JUCO_pitcher_leaderboard.parquet",
  "NAIA_pitcher_leaderboard.parquet",
  "D1_hitter_leaderboard.parquet",
  "D2_hitter_leaderboard.parquet",
  "D3_hitter_leaderboard.parquet",
  "JUCO_hitter_leaderboard.parquet",
  "NAIA_hitter_leaderboard.parquet"
)

for (f in files) download_hf(f)
message("All files ready.")