Update app.R

app.R

@@ -2044,77 +2044,107 @@ observeEvent(input$upload_hf_btn, {
   
   hf_token <- Sys.getenv("HF_WRITE_TOKEN")
   repo_id <- "CoastalBaseball/2026MasterDataset"
+  timestamp <- format(Sys.time(), "%Y%m%d_%H%M%S")
   
-  tryCatch({
-    # Step 1: Get existing UIDs
-    scrape_status_msg("Step 1: Downloading UID index...")
-    tmp_idx <- tempfile(fileext = ".csv.gz")
-    resp <- httr::GET(
-      paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/pbp_uid_index.csv.gz"),
-      httr::add_headers(Authorization = paste("Bearer", hf_token)),
-      httr::write_disk(tmp_idx, overwrite = TRUE)
-    )
-    existing_uids <- read.csv(gzfile(tmp_idx), stringsAsFactors = FALSE)$PitchUID
-    file.remove(tmp_idx)
-    scrape_status_msg(paste0("Step 1 done: ", length(existing_uids), " existing UIDs"))
+  upload_to_hf <- function(new_data, folder, index_file, label) {
+    scrape_status_msg(paste0("Checking existing UIDs for ", label, "..."))
     
-    # Step 2: Filter to new rows
-    scrape_status_msg("Step 2: Filtering new rows...")
-    new_only <- scraped_data() %>% filter(!PitchUID %in% existing_uids)
-    scrape_status_msg(paste0("Step 2 done: ", nrow(new_only), " new rows"))
+    existing_uids <- tryCatch({
+      tmp_idx <- tempfile(fileext = ".csv.gz")
+      resp <- httr::GET(
+        paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", index_file),
+        httr::add_headers(Authorization = paste("Bearer", hf_token)),
+        httr::write_disk(tmp_idx, overwrite = TRUE)
+      )
+      if (httr::status_code(resp) == 200) {
+        d <- read.csv(gzfile(tmp_idx), stringsAsFactors = FALSE)
+        file.remove(tmp_idx)
+        d$PitchUID
+      } else {
+        file.remove(tmp_idx)
+        character(0)
+      }
+    }, error = function(e) { character(0) })
     
-    if (nrow(new_only) == 0) {
-      scrape_status_msg("No new rows to upload.")
-      return()
+    scraped_rows <- nrow(new_data)
+    
+    if (length(existing_uids) > 0 && "PitchUID" %in% names(new_data)) {
+      new_only <- new_data %>% filter(!PitchUID %in% existing_uids)
+    } else {
+      new_only <- new_data
     }
     
-    # Step 3: Write parquet
-    scrape_status_msg("Step 3: Writing parquet...")
-    tmp_data <- tempfile(fileext = ".parquet")
-    arrow::write_parquet(new_only, tmp_data)
-    scrape_status_msg(paste0("Step 3 done: ", round(file.size(tmp_data) / 1024^2, 1), " MB file"))
+    new_rows <- nrow(new_only)
+    total_after <- length(existing_uids) + new_rows
+    
+    if (new_rows == 0) {
+      return(paste0(label, ": ", scraped_rows, " rows scraped, 0 new rows added (", length(existing_uids), " total)"))
+    }
+    
+    scrape_status_msg(paste0("Uploading ", new_rows, " new rows for ", label, "..."))
     
-    # Step 4: Upload parquet
-    scrape_status_msg("Step 4: Uploading parquet...")
     hf <- reticulate::import("huggingface_hub")
     api <- hf$HfApi()
-    timestamp <- format(Sys.time(), "%Y%m%d_%H%M%S")
+    
+    tmp_data <- tempfile(fileext = ".parquet")
+    arrow::write_parquet(new_only, tmp_data)
     
     api$upload_file(
       path_or_fileobj = tmp_data,
-      path_in_repo = paste0("pbp/", timestamp, ".parquet"),
+      path_in_repo = paste0(folder, "/", timestamp, ".parquet"),
       repo_id = repo_id,
       repo_type = "dataset",
       token = hf_token
     )
     file.remove(tmp_data)
-    scrape_status_msg("Step 4 done: Parquet uploaded")
     
-    # Step 5: Update index
-    scrape_status_msg("Step 5: Updating UID index...")
-    all_uids <- data.frame(PitchUID = c(existing_uids, new_only$PitchUID))
-    rm(new_only, existing_uids); gc()
+    scrape_status_msg(paste0("Updating ", label, " index..."))
     
-    tmp_idx2 <- tempfile(fileext = ".csv.gz")
-    gz <- gzfile(tmp_idx2, "w")
+    all_uids <- data.frame(PitchUID = c(existing_uids, new_only$PitchUID))
+    tmp_idx <- tempfile(fileext = ".csv.gz")
+    gz <- gzfile(tmp_idx, "w")
     write.csv(all_uids, gz, row.names = FALSE)
     close(gz)
     
     api$upload_file(
-      path_or_fileobj = tmp_idx2,
-      path_in_repo = "pbp_uid_index.csv.gz",
+      path_or_fileobj = tmp_idx,
+      path_in_repo = index_file,
       repo_id = repo_id,
       repo_type = "dataset",
       token = hf_token
     )
-    file.remove(tmp_idx2)
-    rm(all_uids); gc()
+    file.remove(tmp_idx)
     
-    scrape_status_msg("Step 5 done: ALL COMPLETE")
+    rm(new_only, all_uids); gc()
+    paste0(label, ": ", scraped_rows, " rows scraped, ", new_rows, " new rows added (", total_after, " total)")
+  }
+  
+  if (input$scrape_source == "pbp") {
+    msg1 <- upload_to_hf(scraped_data(), "pbp", "pbp_uid_index.csv.gz", "Master Dataset")
+    gc()
     
-  }, error = function(e) {
-    scrape_status_msg(paste0("FAILED at: ", e$message))
-  })
+    cp <- scraped_data() %>% filter(PitcherTeam == "COA_CHA")
+    msg2 <- if (nrow(cp) > 0) {
+      upload_to_hf(cp, "coastal_pitchers", "coastal_pitchers_uid_index.csv.gz", "Coastal Pitchers")
+    } else { "Coastal Pitchers: No matching rows" }
+    rm(cp); gc()
+    
+    ch <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
+    msg3 <- if (nrow(ch) > 0) {
+      upload_to_hf(ch, "coastal_hitters", "coastal_hitters_uid_index.csv.gz", "Coastal Hitters")
+    } else { "Coastal Hitters: No matching rows" }
+    rm(ch); gc()
+    
+    scrape_status_msg(paste(msg1, msg2, msg3, sep = "\n"))
+    
+  } else if (input$scrape_source == "pos") {
+    msg1 <- upload_to_hf(scraped_data(), "pos", "pos_uid_index.csv.gz", "Positional Dataset")
+    scrape_status_msg(msg1)
+    
+  } else if (input$scrape_source == "ncaa") {
+    msg1 <- upload_to_hf(scraped_data(), "ncaa_pbp", "ncaa_pbp_uid_index.csv.gz", "NCAA PBP Dataset")
+    scrape_status_msg(msg1)
+  }
 })
     
 }