Update app.R

app.R

@@ -32,59 +32,226 @@ hf_target_for <- function(source) {
 }
 
 # ============================================================
-# Model + run-value files (must be shipped in the Space)
+# READ IN PRIVATE DATASETS AND MODELS FOR CLEANING -- All in a private dataset on the main (HyannisHarborHawksCCBL) Huggingface
 # ============================================================
-stuffplus_recipe <- readRDS("stuffplus_recipe.rds")
-stuffplus_model  <- xgb.load("stuffplus_xgb.json")
-
-rv <- tryCatch(
-  read.csv("non_context_run_values.csv", stringsAsFactors = FALSE),
-  error = function(e) {
-    message("non_context_run_values.csv not found - run values will be unavailable")
-    NULL
+
+download_private_parquet <- function(repo_id, filename, max_retries = 3) {
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  api_key <- Sys.getenv("HF_READ_TOKEN")
+  if (api_key == "") stop("API key is not set.")
+  for (attempt in 1:max_retries) {
+    tryCatch({
+      response <- GET(url, add_headers(Authorization = paste("Bearer", api_key)), timeout(60))
+      if (status_code(response) == 200) {
+        tmp <- tempfile(fileext = ".parquet")
+        writeBin(content(response, "raw"), tmp)
+        return(read_parquet(tmp))
+      } else warning(paste("Attempt", attempt, "failed with status:", status_code(response)))
+    }, error = function(e) {
+      warning(paste("Attempt", attempt, "failed:", e$message))
+      if (attempt < max_retries) Sys.sleep(2)
+    })
   }
-)
+  stop(paste("Failed to download dataset after", max_retries, "attempts"))
+}
+
+download_private_csv <- function(repo_id, filename, max_retries = 3) {
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  api_key <- Sys.getenv("HF_READ_TOKEN")
+  if (api_key == "") stop("API key is not set.")
+  for (attempt in 1:max_retries) {
+    tryCatch({
+      response <- GET(url, add_headers(Authorization = paste("Bearer", api_key)), timeout(60))
+      if (status_code(response) == 200) {
+        tmp <- tempfile(fileext = ".csv")
+        writeBin(content(response, "raw"), tmp)
+        return(readr::read_csv(tmp, show_col_types = FALSE))
+      } else warning(paste("Attempt", attempt, "failed with status:", status_code(response)))
+    }, error = function(e) {
+      warning(paste("Attempt", attempt, "failed:", e$message))
+      if (attempt < max_retries) Sys.sleep(2)
+    })
+  }
+  stop(paste("Failed to download dataset after", max_retries, "attempts"))
+}
+
+download_private_xgb <- function(repo_id, filename, max_retries = 3) {
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  api_key <- Sys.getenv("HF_READ_TOKEN")
+  if (api_key == "") stop("API key is not set.")
+  for (attempt in 1:max_retries) {
+    tryCatch({
+      response <- GET(url, add_headers(Authorization = paste("Bearer", api_key)), timeout(60))
+      if (status_code(response) == 200) {
+        tmp <- tempfile(fileext = ".json")
+        writeBin(content(response, "raw"), tmp)
+        return(xgb.load(tmp))
+      } else warning(paste("Attempt", attempt, "failed with status:", status_code(response)))
+    }, error = function(e) {
+      warning(paste("Attempt", attempt, "failed:", e$message))
+      if (attempt < max_retries) Sys.sleep(2)
+    })
+  }
+  stop(paste("Failed to download dataset after", max_retries, "attempts"))
+}
+
+download_private_rds <- function(repo_id, filename, max_retries = 3) {
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  api_key <- Sys.getenv("HF_READ_TOKEN")
+  if (api_key == "") stop("API key is not set.")
+  for (attempt in 1:max_retries) {
+    tryCatch({
+      response <- GET(url, add_headers(Authorization = paste("Bearer", api_key)), timeout(60))
+      if (status_code(response) == 200) {
+        tmp <- tempfile(fileext = ".rds")
+        writeBin(content(response, "raw"), tmp)
+        return(readRDS(tmp))
+      } else warning(paste("Attempt", attempt, "failed:", e$message))
+    }, error = function(e) {
+      warning(paste("Attempt", attempt, "failed:", e$message))
+      if (attempt < max_retries) Sys.sleep(2)
+    })
+  }
+  stop(paste("Failed to download dataset after", max_retries, "attempts"))
+}
+
+#Stuff Model (XGBoost)
+stuffplus_recipe <- download_private_rds("HyannisHarborHawksCCBL/CleanDataObjects", "stuffplus_recipe.rds")
+stuffplus_model  <- download_private_xgb("HyannisHarborHawksCCBL/CleanDataObjects", "stuffplus_xgb.json")
+
+#xwOBA model (KNN)
+xwoba_model <- unbundle(download_private_rds("HyannisHarborHawksCCBL/CleanDataObjects", "bundle_college_xwoba_knn.rds"))
+
+#Data to join on (non-context run values, conferences, GUTS!, longer team names)
+rv <- download_private_csv("HyannisHarborHawksCCBL/CleanDataObjects", "non_context_run_values.csv")
+team_leagues <- download_private_csv("HyannisHarborHawksCCBL/CleanDataObjects", "college_team_leagues.csv")
+college_guts <- download_private_csv("HyannisHarborHawksCCBL/CleanDataObjects", "college_GUTS.csv")
+college_join <- download_private_csv("HyannisHarborHawksCCBL/CleanDataObjects", "college_join.csv")
+
+#Team logo
+logo_b64 <- base64enc::base64encode("HHLogo.png")
+logo_uri <- paste0("data:image/png;base64,", logo_b64)
 
 # ============================================================
-# Data pipeline (lifted verbatim from the report app)
+#Clean data 
 # ============================================================
-clean_college_data <- function(data, teams = NA){
 
+predict_stuffplus <- function(data) {
+  
+  predict_data <- data %>% 
+    mutate(RelSide = case_when(
+      PitcherThrows == "Right" ~ RelSide,
+      PitcherThrows == "Left" ~ -RelSide,
+      PitcherThrows %in% c("Both", "Undefined") & RelSide > 0 ~ RelSide,
+      PitcherThrows %in% c("Both", "Undefined") & RelSide < 0 ~ -RelSide),
+      ax0 = case_when(
+        PitcherThrows == "Right" ~ ax0,
+        PitcherThrows == "Left" ~ -ax0,
+        PitcherThrows %in% c("Both", "Undefined") & ax0 > 0 ~ ax0,
+        PitcherThrows %in% c("Both", "Undefined") & ax0 < 0 ~ -ax0),
+      ax0 = -ax0) %>%
+    group_by(Pitcher, GameID) %>%
+    mutate(
+      primary_pitch = case_when(
+        any(TaggedPitchType == "Fastball") ~ "Fastball",
+        any(TaggedPitchType == "Sinker") ~ "Sinker",
+        TRUE ~ names(sort(table(TaggedPitchType), decreasing = TRUE))[1]
+      )
+    ) %>%
+    ungroup() %>% 
+    group_by(Pitcher, GameID, primary_pitch) %>%
+    mutate(
+      primary_az0 = mean(az0[TaggedPitchType == primary_pitch], na.rm = TRUE),
+      primary_velo = mean(RelSpeed[TaggedPitchType == primary_pitch], na.rm = TRUE)
+    ) %>%
+    ungroup() %>% 
+    mutate(az0_diff = az0 - primary_az0, 
+           velo_diff = RelSpeed - primary_velo)
+  
+  
+  df_processed <- bake(stuffplus_recipe, new_data = predict_data)
+  
+  df_matrix <- as.matrix(df_processed)
+  
+  raw_stuff <- predict(stuffplus_model, df_matrix)
+  
+  data$raw_stuff <- raw_stuff
+  
+  
   data <- data %>%
-    mutate(PlayResult = ifelse(PlayResult %in% c("HomeRun", "homerun"), "Homerun", PlayResult),
-           Batter  = sub("(.*),\\s*(.*)", "\\2 \\1", Batter),
+    mutate(stuff_plus = ((raw_stuff - 0.004424894) / 0.01010482) * 10 + 100)
+  
+  return(data)
+  
+}
+
+#===================================================================================================
+#Clean College Data Function (includes predicting stuffplus)
+
+
+clean_college_data <- function(data, teams = NA) {
+  
+
+  data <- data %>% 
+    mutate(PlayResult = ifelse(PlayResult %in% c("HomeRun", "homerun"), "Homerun", PlayResult), 
+           Batter = sub("(.*),\\s*(.*)", "\\2 \\1", Batter),
            Pitcher = sub("(.*),\\s*(.*)", "\\2 \\1", Pitcher),
            Catcher = sub("(.*),\\s*(.*)", "\\2 \\1", Catcher))
-
+  
+  
   col <- colnames(data)
-  if ("Top/Bottom" %in% col){
-    data <- data %>% rename(`Top.Bottom` = `Top/Bottom`)
+  
+  if ("Top/Bottom" %in% col){ 
+    data <- data %>% 
+      rename(`Top.Bottom` = `Top/Bottom`)
   }
-
+  
+  #Fix column types
   numeric_columns <- c("PitchNo", "PAofInning", "PitchofPA", "PitcherId", "BatterId", "Inning", "Outs", "Balls",
-                       "Strikes", "OutsOnPlay", "RunsScored", "RelSpeed", "VertRelAngle", "HorzRelAngle", "SpinRate",
-                       "SpinAxis", "RelHeight", "RelSide", "Extension", "VertBreak", "InducedVertBreak", "HorzBreak",
-                       "PlateLocHeight", "PlateLocSide", "ZoneSpeed", "VertApprAngle", "HorzApprAngle", "ZoneTime",
+                       "Strikes", "OutsOnPlay", "RunsScored", "RelSpeed", "VertRelAngle", "HorzRelAngle", "SpinRate", 
+                       "SpinAxis", "RelHeight", "RelSide", "Extension", "VertBreak", "InducedVertBreak", "HorzBreak", 
+                       "PlateLocHeight", "PlateLocSide", "ZoneSpeed", "VertApprAngle", "HorzApprAngle", "ZoneTime", 
                        "ExitSpeed", "Angle", "Direction", "HitSpinRate", "Distance", "Bearing", "HangTime",
-                       "LastTrackedDistance", "pfxx", "pfxz", "x0", "y0", "z0", "vx0", "vz0", "vy0", "ax0", "ay0",
+                       "LastTrackedDistance", "pfxx", "pfxz", "x0", "y0", "z0", "vx0", "vz0", "vy0", "ax0", "ay0", 
                        "az0",  "EffectiveVelo", "MaxHeight", "SpeedDrop", "ContactPositionX", "ContactPositionY",
                        "ContactPositionZ", "HomeTeamForeignID", "AwayTeamForeignID", "CatcherId", "ThrowSpeed",
-                       "PopTime", "ExchangeTime", "TimeToBase")
-
-  data <- data %>%
-    mutate(across(any_of(numeric_columns), as.numeric),
-           PlateLocHeight = if ("PlateLocHeight" %in% names(.)) 12 * PlateLocHeight else PlateLocHeight,
-           PlateLocSide   = if ("PlateLocSide" %in% names(.)) 12 * PlateLocSide else PlateLocSide)
-
-  data <- data %>%
+                       "PopTime", "ExchangeTime", "TimeToBase", "OutsOnPlay") 
+  data <- data %>% 
+    mutate(across(any_of(numeric_columns), ~suppressWarnings(as.numeric(.x)))) 
+  
+  
+  #Correct for last, first
+  data <- data %>% 
+    mutate(
+      Batter = ifelse(str_detect(Batter, ","),
+                      str_trim(sub("^([^,]*),\\s*(.*)$", "\\2 \\1", Batter)),
+                      Batter),
+      Pitcher = ifelse(str_detect(Pitcher, ","),
+                       str_trim(sub("^([^,]*),\\s*(.*)$", "\\2 \\1", Pitcher)),
+                       Pitcher)
+    )
+  
+  #Fix plate locations where unit is feet and not inches 
+  data <- data %>% 
+    group_by(GameID) %>% 
+    mutate(
+      PlateLocHeight = PlateLocHeight * if (isTRUE(sd(PlateLocHeight, na.rm = TRUE) < 5)) 12 else 1,
+      PlateLocSide   = PlateLocSide   * if (isTRUE(sd(PlateLocSide, na.rm = TRUE) < 5)) 12 else 1
+    ) %>% 
+    ungroup()
+    
+  
+  #fix pitch tagging
+  data <- data %>% 
     mutate(TaggedPitchType = case_when(
-      TaggedPitchType == "FourSeamFastBall" ~ "Fastball",
-      TaggedPitchType %in% c("TwoSeamFastBall", "OneSeamFastBall") ~ "Sinker",
-      TaggedPitchType == "ChangeUp" ~ "Changeup",
-      TaggedPitchType == "Undefined" ~ "Other",
+      TaggedPitchType == "FourSeamFastBall" ~ "Fastball", 
+      TaggedPitchType %in% c("TwoSeamFastBall", "OneSeamFastBall") ~ "Sinker", 
+      TaggedPitchType == "ChangeUp" ~ "Changeup", 
+      TaggedPitchType == "Undefined" ~ "Other", 
       T ~ TaggedPitchType
     ))
-
+  
+  #Main aggregations
   data <- data %>%
     mutate(
       is_csw = case_when(
@@ -92,7 +259,7 @@ clean_college_data <- function(data, teams = NA){
         TRUE ~ 0
       ),
       is_swing = case_when(
-        PitchCall %in% c("StrikeSwinging", "FoulBallNotFieldable", "InPlay",
+        PitchCall %in% c("StrikeSwinging", "FoulBallNotFieldable", "InPlay", 
                          "FoulBallFieldable", "FoulBall") ~ 1,
         TRUE ~ 0
       ),
@@ -182,98 +349,116 @@ clean_college_data <- function(data, teams = NA){
       ),
       is_k = case_when(
         is_at_bat == 1 & KorBB == "Strikeout" ~ 1,
-        is_at_bat == 1 & KorBB != "Strikeout" ~ 0,
+        is_at_bat == 1 & KorBB != "Strikeout" ~ 0, 
         PlayResult %in% c("StrikeoutSwinging", "StrikeoutLooking") ~ 1,
         TRUE ~ NA_real_
-      ),
+      ), 
       is_put_away = case_when(
         Strikes == 2 & KorBB == "Strikeout" ~ 1,
         Strikes == 2 & KorBB != "Strikeout" ~ 0,
         Strikes == 2 & PlayResult %in% c("StrikeoutSwinging", "StrikeoutLooking") ~ 1,
         TRUE ~ NA_real_
-      ),
+      ), 
       OutsOnPlay = ifelse(KorBB == "Strikeout" | PlayResult %in% c("StrikeoutSwinging", "StrikeoutLooking"), OutsOnPlay + 1, OutsOnPlay)
     )
-
-  data <- data %>%
+  
+  data <- data %>% 
     mutate(event_type = case_when(
-      PitchCall %in% c("StrikeSwinging", "StrkeSwinging") ~ "Whiff",
-      PitchCall %in% c("StriekC", "StrikeCalled") ~ "Called Strike",
+      PitchCall %in% c("StrikeSwinging", "StrkeSwinging") ~ "Whiff", 
+      PitchCall %in% c("StriekC", "StrikeCalled") ~ "Called Strike", 
       PitchCall %in% c("FoulBallFieldable", "FoulBall", "FoulBallNotFieldable",
-                       "FouldBallNotFieldable") ~ "Foul Ball",
-      PitchCall %in% c("BallCalled", "BallinDirt", "BallIntentional", "BalIntentional") ~ "Ball",
-      PitchCall == "HitByPitch" ~ "HBP",
+                       "FouldBallNotFieldable") ~ "Foul Ball", 
+      PitchCall %in% c("BallCalled", "BallinDirt", "BallIntentional", "BalIntentional") ~ "Ball", 
+      PitchCall == "HitByPitch" ~ "HBP", 
       PitchCall == "InPlay" & PlayResult %in% c("Out", "FieldersChoice",
-                                                "Error", "error",
-                                                "Sacrifice") ~ "Field Out",
-      PitchCall == "InPlay" & PlayResult == "Single" ~ "Single",
+                                                "Error", "error", 
+                                                "Sacrifice") ~ "Field Out", 
+      PitchCall == "InPlay" & PlayResult == "Single" ~ "Single", 
       PitchCall == "InPlay" & PlayResult == "Double" ~ "Double",
       PitchCall == "InPlay" & PlayResult == "Triple" ~ "Triple",
-      PitchCall == "InPlay" & PlayResult == "Homerun" ~ "Home Run",
+      PitchCall == "InPlay" & PlayResult == "Homerun" ~ "Home Run", 
       T ~ NA
     ))
-
-  if (!is.null(rv)) data <- data %>% left_join(rv, by = "event_type")
-
+  
+  
+  if (!"mean_DRE" %in% names(data)) {
+    data <- data %>% 
+      left_join(rv, by = "event_type")
+  }
+  
   data <- data %>%
     dplyr::select(
-      -any_of(c("PitchLastMeasuredX", "PitchLastMeasuredY", "PitchLastMeasuredZ",
-                "HitSpinAxis",
-                "PitchReleaseConfidence", "PitchLocationConfidence", "PitchMovementConfidence",
-                "HitLaunchConfidence", "HitLandingConfidence",
-                "CatcherThrowCatchConfidence", "CatcherThrowReleaseConfidence", "CatcherThrowLocationConfidence",
-                "PositionAt110X", "PositionAt110Y", "PositionAt110Z")),
+      -any_of(c(
+        "PitchLastMeasuredX", "PitchLastMeasuredY", "PitchLastMeasuredZ",
+        "HitSpinAxis",
+        "PitchReleaseConfidence", "PitchLocationConfidence", "PitchMovementConfidence",
+        "HitLaunchConfidence", "HitLandingConfidence",
+        "CatcherThrowCatchConfidence", "CatcherThrowReleaseConfidence", "CatcherThrowLocationConfidence",
+        "PositionAt110X", "PositionAt110Y", "PositionAt110Z"
+      )),
       -starts_with("PitchTrajectory"),
       -starts_with("HitTrajectory")
     )
+  
+  data <- data %>% 
+    dplyr::select(-any_of(c("HomeLeague", "AwayLeague", "PitcherLeague", "BatterLeague"))) %>% 
+    left_join(team_leagues, by = "HomeTeam") %>% 
+    left_join(team_leagues %>% rename(AwayLeague = HomeLeague, AwayTeam = HomeTeam), by = "AwayTeam") %>% 
+    mutate(PitcherLeague = ifelse(`Top.Bottom` == "Top", HomeLeague, AwayLeague),
+           BatterLeague = ifelse(`Top.Bottom` == "Top", AwayLeague, HomeLeague),
+           HomeLeague = replace_na(HomeLeague, "Other"),
+           AwayLeague = replace_na(AwayLeague, "Other"),
+           PitcherLeague = replace_na(PitcherLeague, "Other"),
+           BatterLeague = replace_na(BatterLeague, "Other")
+    )
+  
 
-  return(data)
-}
-
-predict_stuffplus <- function(data) {
-
-  predict_data <- data %>%
-    mutate(RelSide = case_when(
-      PitcherThrows == "Right" ~ RelSide,
-      PitcherThrows == "Left" ~ -RelSide,
-      PitcherThrows %in% c("Both", "Undefined") & RelSide > 0 ~ RelSide,
-      PitcherThrows %in% c("Both", "Undefined") & RelSide < 0 ~ -RelSide),
-      ax0 = case_when(
-        PitcherThrows == "Right" ~ ax0,
-        PitcherThrows == "Left" ~ -ax0,
-        PitcherThrows %in% c("Both", "Undefined") & ax0 > 0 ~ ax0,
-        PitcherThrows %in% c("Both", "Undefined") & ax0 < 0 ~ -ax0),
-      PlateLocHeight = PlateLocHeight*12,
-      PlateLocSide = PlateLocSide*12,
-      ax0 = -ax0) %>%
-    group_by(Pitcher, GameID) %>%
-    mutate(
-      primary_pitch = case_when(
-        any(TaggedPitchType == "Fastball") ~ "Fastball",
-        any(TaggedPitchType == "Sinker") ~ "Sinker",
-        TRUE ~ names(sort(table(TaggedPitchType), decreasing = TRUE))[1]
-      )
-    ) %>%
-    group_by(Pitcher, GameID, primary_pitch) %>%
-    mutate(
-      primary_az0  = mean(az0[TaggedPitchType == primary_pitch], na.rm = TRUE),
-      primary_velo = mean(RelSpeed[TaggedPitchType == primary_pitch], na.rm = TRUE)
-    ) %>%
-    ungroup() %>%
-    mutate(az0_diff = az0 - primary_az0,
-           velo_diff = RelSpeed - primary_velo)
-
-  df_processed <- bake(stuffplus_recipe, new_data = predict_data)
-  df_matrix    <- as.matrix(df_processed)
-  raw_stuff    <- predict(stuffplus_model, df_matrix)
-
-  data$raw_stuff <- raw_stuff
+  
+  if (!"stuff_plus" %in% names(data)) {
+    data <- predict_stuffplus(data)
+  }
+  
+  data <- data %>% 
+    filter(!(Level %in% c("D2", "JUCO", "NAIA", "D3")))
+
+  #predict xwoba
+  data <- data %>% mutate(row_id = row_number())
+  
+  prob_cols <- c("prob_0", "prob_1", "prob_2", "prob_3", "prob_4")
+  
+  complete <- data %>%
+    filter(!is.na(ExitSpeed) & !is.na(Angle)) %>%
+    dplyr::select(-any_of(prob_cols))
+  
+  xwoba_preds <- predict(xwoba_model, complete, type = "prob")
+  colnames(xwoba_preds) <- prob_cols
+  
+  complete <- complete %>% bind_cols(xwoba_preds)
+  
   data <- data %>%
-    mutate(stuff_plus = ((raw_stuff - 0.004424894) / 0.01010482) * 10 + 100)
+    dplyr::select(-any_of(prob_cols)) %>% 
+    left_join(complete %>% dplyr::select(row_id, any_of(prob_cols)), by = "row_id") %>%
+    dplyr::select(-row_id)
+  
+  data <- data %>% 
+    dplyr::select(-any_of(c("w1B", "w2B", "w3B", "wHR"))) %>% 
+    cross_join(college_guts %>% dplyr::select(w1B, w2B, w3B, wHR)) %>% 
+    mutate(xwOBA = w1B*prob_1 + w2B*prob_2 + w3B*prob_3 + wHR*prob_4 + 0*prob_0,
+           xwOBA = case_when(
+             KorBB == "Strikeout" ~ 0,
+             KorBB == "Walk" ~ .82,
+             event_type == "HBP" ~ .85,
+             TRUE ~ xwOBA
+           ))
+  
+  message(nrow(data), " rows cleaned.")
 
   return(data)
+  
+  
 }
 
+
 # ============================================================
 # CSS
 # ============================================================