Update app.R

app.R

@@ -2044,95 +2044,104 @@ observeEvent(input$upload_hf_btn, {
   
   hf_token <- Sys.getenv("HF_WRITE_TOKEN")
   repo_id <- "CoastalBaseball/2026MasterDataset"
+  timestamp <- format(Sys.time(), "%Y%m%d_%H%M%S")
   
-  upload_to_hf <- function(new_data, filename, label) {
-    scrape_status_msg(paste0("Downloading existing ", label, "..."))
+  upload_to_hf <- function(new_data, folder, index_file, label) {
+    scrape_status_msg(paste0("Checking existing UIDs for ", label, "..."))
     
-    existing <- tryCatch({
+    # Download tiny UID index
+    existing_uids <- tryCatch({
+      tmp_idx <- tempfile(fileext = ".csv.gz")
       resp <- httr::GET(
-        paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename),
-        httr::add_headers(Authorization = paste("Bearer", hf_token))
+        paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", index_file),
+        httr::add_headers(Authorization = paste("Bearer", hf_token)),
+        httr::write_disk(tmp_idx, overwrite = TRUE)
       )
       if (httr::status_code(resp) == 200) {
-        tmp_dl <- tempfile(fileext = ".parquet")
-        writeBin(httr::content(resp, as = "raw"), tmp_dl)
-        d <- arrow::read_parquet(tmp_dl)
-        file.remove(tmp_dl)
-        d
-      } else { NULL }
-    }, error = function(e) { NULL })
-    
-    existing_rows <- if (!is.null(existing)) nrow(existing) else 0
-    scraped_rows <- nrow(new_data)
-    
-    if (existing_rows > 0) {
-      scrape_status_msg(paste0("Merging ", label, "..."))
-      combined <- bind_rows(existing, new_data)
-      rm(existing); gc()
-      
-      if ("PitchUID" %in% names(combined)) {
-        combined <- combined %>% distinct(PitchUID, .keep_all = TRUE)
+        d <- read.csv(gzfile(tmp_idx), stringsAsFactors = FALSE)
+        file.remove(tmp_idx)
+        d$PitchUID
       } else {
-        combined <- combined %>% distinct()
+        file.remove(tmp_idx)
+        character(0)
       }
+    }, error = function(e) { character(0) })
+    
+    # Filter to only new rows
+    scraped_rows <- nrow(new_data)
+    if (length(existing_uids) > 0 && "PitchUID" %in% names(new_data)) {
+      new_only <- new_data %>% filter(!PitchUID %in% existing_uids)
     } else {
-      combined <- new_data
-      rm(existing); gc()
+      new_only <- new_data
     }
     
-    new_rows <- nrow(combined) - existing_rows
+    if (nrow(new_only) == 0) {
+      return(paste0(label, ": 0 new rows (all ", scraped_rows, " already exist)"))
+    }
     
-    scrape_status_msg(paste0("Uploading ", label, " (", nrow(combined), " rows)..."))
+    scrape_status_msg(paste0("Uploading ", nrow(new_only), " new rows for ", label, "..."))
     
-    tmp <- tempfile(fileext = ".parquet")
-    arrow::write_parquet(combined, tmp)
-    rm(combined); gc()
+    hf <- reticulate::import("huggingface_hub")
+    api <- hf$HfApi()
     
-    result <- tryCatch({
-      hf <- reticulate::import("huggingface_hub")
-      api <- hf$HfApi()
-      api$upload_file(
-        path_or_fileobj = tmp,
-        path_in_repo = filename,
-        repo_id = repo_id,
-        repo_type = "dataset",
-        token = hf_token
-      )
-      paste0(label, ": ", scraped_rows, " rows scraped, ", new_rows, " new rows added (", nrow(new_data), " + ", existing_rows, " existing = deduped)")
-    }, error = function(e) {
-      paste0(label, " upload error: ", e$message)
-    })
+    # Upload new data as timestamped parquet into folder
+    tmp_data <- tempfile(fileext = ".parquet")
+    arrow::write_parquet(new_only, tmp_data)
     
-    file.remove(tmp)
-    gc()
-    return(result)
+    api$upload_file(
+      path_or_fileobj = tmp_data,
+      path_in_repo = paste0(folder, "/", timestamp, ".parquet"),
+      repo_id = repo_id,
+      repo_type = "dataset",
+      token = hf_token
+    )
+    file.remove(tmp_data)
+    
+    # Update UID index (append new UIDs)
+    all_uids <- data.frame(PitchUID = c(existing_uids, new_only$PitchUID))
+    tmp_idx <- tempfile(fileext = ".csv.gz")
+    gz <- gzfile(tmp_idx, "w")
+    write.csv(all_uids, gz, row.names = FALSE)
+    close(gz)
+    
+    api$upload_file(
+      path_or_fileobj = tmp_idx,
+      path_in_repo = index_file,
+      repo_id = repo_id,
+      repo_type = "dataset",
+      token = hf_token
+    )
+    file.remove(tmp_idx)
+    
+    rm(new_only, all_uids); gc()
+    paste0(label, ": ", scraped_rows, " scraped, ", nrow(new_only), " new rows added (", length(existing_uids) + nrow(new_only), " total)")
   }
   
-  # Upload main dataset first
-  main_file <- paste0(input$scrape_source, "_2026_master.parquet")
-  msg1 <- upload_to_hf(scraped_data(), main_file, "Master Dataset")
-  gc()
-  
+  # Determine folder based on scrape source
   if (input$scrape_source == "pbp") {
-    # Filter THEN upload one at a time to save memory
-    coastal_pitchers <- scraped_data() %>% filter(PitcherTeam == "COA_CHA")
+    msg1 <- upload_to_hf(scraped_data(), "pbp", "pbp_uid_index.csv.gz", "Master Dataset")
+    gc()
     
-    msg2 <- if (nrow(coastal_pitchers) > 0) {
-      upload_to_hf(coastal_pitchers, "CoastalPitchers2026.parquet", "Coastal Pitchers")
+    cp <- scraped_data() %>% filter(PitcherTeam == "COA_CHA")
+    msg2 <- if (nrow(cp) > 0) {
+      upload_to_hf(cp, "coastal_pitchers", "coastal_pitchers_uid_index.csv.gz", "Coastal Pitchers")
     } else { "Coastal Pitchers: No matching rows" }
+    rm(cp); gc()
     
-    rm(coastal_pitchers); gc()
-    
-    coastal_hitters <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
-    
-    msg3 <- if (nrow(coastal_hitters) > 0) {
-      upload_to_hf(coastal_hitters, "CoastalHitters2026.parquet", "Coastal Hitters")
+    ch <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
+    msg3 <- if (nrow(ch) > 0) {
+      upload_to_hf(ch, "coastal_hitters", "coastal_hitters_uid_index.csv.gz", "Coastal Hitters")
     } else { "Coastal Hitters: No matching rows" }
-    
-    rm(coastal_hitters); gc()
+    rm(ch); gc()
     
     scrape_status_msg(paste(msg1, msg2, msg3, sep = "\n"))
-  } else {
+    
+  } else if (input$scrape_source == "pos") {
+    msg1 <- upload_to_hf(scraped_data(), "pos", "pos_uid_index.csv.gz", "Positional Dataset")
+    scrape_status_msg(msg1)
+    
+  } else if (input$scrape_source == "ncaa") {
+    msg1 <- upload_to_hf(scraped_data(), "ncaa_pbp", "ncaa_pbp_uid_index.csv.gz", "NCAA PBP Dataset")
     scrape_status_msg(msg1)
   }
 })