Update app.R

app.R

@@ -2026,71 +2026,87 @@ observe({
     }
   )
 
+
 observeEvent(input$upload_hf_btn, {
   req(scraped_data())
   
-  scrape_status_msg("Downloading existing dataset...")
-  
   hf_token <- Sys.getenv("HF_WRITE_TOKEN")
   repo_id <- "CoastalBaseball/2026MasterDataset"
-  filename <- paste0(input$scrape_source, "_2026_master.parquet")
   
-  # Try to download existing data
-  existing <- tryCatch({
-    resp <- httr::GET(
-      paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename),
-      httr::add_headers(Authorization = paste("Bearer", hf_token))
-    )
+  # Helper: download existing, merge, dedupe, upload
+  upload_to_hf <- function(new_data, filename, label) {
+    scrape_status_msg(paste0("Downloading existing ", label, "..."))
     
-    if (httr::status_code(resp) == 200) {
-      tmp_dl <- tempfile(fileext = ".parquet")
-      writeBin(httr::content(resp, as = "raw"), tmp_dl)
-      arrow::read_parquet(tmp_dl)
+    existing <- tryCatch({
+      resp <- httr::GET(
+        paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename),
+        httr::add_headers(Authorization = paste("Bearer", hf_token))
+      )
+      if (httr::status_code(resp) == 200) {
+        tmp_dl <- tempfile(fileext = ".parquet")
+        writeBin(httr::content(resp, as = "raw"), tmp_dl)
+        arrow::read_parquet(tmp_dl)
+      } else { NULL }
+    }, error = function(e) { NULL })
+    
+    if (!is.null(existing) && nrow(existing) > 0) {
+      scrape_status_msg(paste0("Merging ", label, "..."))
+      combined <- bind_rows(existing, new_data)
+      if ("PitchUID" %in% names(combined)) {
+        combined <- combined %>% distinct(PitchUID, .keep_all = TRUE)
+      } else {
+        combined <- combined %>% distinct()
+      }
     } else {
-      NULL
+      combined <- new_data
     }
-  }, error = function(e) { NULL })
-  
-  # Combine
-  if (!is.null(existing) && nrow(existing) > 0) {
-    scrape_status_msg("Merging with existing data...")
-    combined <- bind_rows(existing, scraped_data())
     
-    if ("PitchUID" %in% names(combined)) {
-      combined <- combined %>% distinct(PitchUID, .keep_all = TRUE)
-    } else {
-      combined <- combined %>% distinct()
-    }
-  } else {
-    combined <- scraped_data()
+    scrape_status_msg(paste0("Uploading ", label, " (", nrow(combined), " rows)..."))
+    
+    tmp <- tempfile(fileext = ".parquet")
+    arrow::write_parquet(combined, tmp)
+    
+    result <- tryCatch({
+      hf <- import("huggingface_hub")
+      api <- hf$HfApi()
+      api$upload_file(
+        path_or_fileobj = tmp,
+        path_in_repo = filename,
+        repo_id = repo_id,
+        repo_type = "dataset",
+        token = hf_token
+      )
+      paste0(label, ": ", nrow(combined), " rows uploaded.")
+    }, error = function(e) {
+      paste0(label, " upload error: ", e$message)
+    })
+    
+    file.remove(tmp)
+    gc()
+    return(result)
   }
   
-  # Upload via commit API
-  scrape_status_msg(paste0("Uploading ", nrow(combined), " total rows..."))
+  # 1. Upload main dataset
+  main_file <- paste0(input$scrape_source, "_2026_master.parquet")
+  msg1 <- upload_to_hf(scraped_data(), main_file, "Master Dataset")
   
-  tmp <- tempfile(fileext = ".parquet")
-  arrow::write_parquet(combined, tmp)
-  
-result <- tryCatch({
-    hf <- import("huggingface_hub")
-    api <- hf$HfApi()
-
+  # 2. If PBP, also upload Coastal filtered datasets
+  if (input$scrape_source == "pbp") {
+    coastal_pitchers <- scraped_data() %>% filter(PitcherTeam == "COA_CHA")
+    coastal_hitters  <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
     
-  api$upload_file(
-    path_or_fileobj = tmp,
-    path_in_repo = filename,
-    repo_id = repo_id,
-    repo_type = "dataset",
-    token = hf_token          
-  )
+    msg2 <- if (nrow(coastal_pitchers) > 0) {
+      upload_to_hf(coastal_pitchers, "CoastalPitchers2026.parquet", "Coastal Pitchers")
+    } else { "Coastal Pitchers: No matching rows" }
     
-    paste0("Done! ", nrow(combined), " total rows in dataset.")
-  }, error = function(e) {
-    paste("Upload error:", e$message)
-  })
-  
-  file.remove(tmp)
-  scrape_status_msg(result)
+    msg3 <- if (nrow(coastal_hitters) > 0) {
+      upload_to_hf(coastal_hitters, "CoastalHitters2026.parquet", "Coastal Hitters")
+    } else { "Coastal Hitters: No matching rows" }
+    
+    scrape_status_msg(paste(msg1, msg2, msg3, sep = "\n"))
+  } else {
+    scrape_status_msg(msg1)
+  }
 })
     
 }