Update app.R

app.R

@@ -2056,7 +2056,9 @@ observeEvent(input$upload_hf_btn, {
       if (httr::status_code(resp) == 200) {
         tmp_dl <- tempfile(fileext = ".parquet")
         writeBin(httr::content(resp, as = "raw"), tmp_dl)
-        arrow::read_parquet(tmp_dl)
+        d <- arrow::read_parquet(tmp_dl)
+        file.remove(tmp_dl)
+        d
       } else { NULL }
     }, error = function(e) { NULL })
     
@@ -2066,6 +2068,8 @@ observeEvent(input$upload_hf_btn, {
     if (existing_rows > 0) {
       scrape_status_msg(paste0("Merging ", label, "..."))
       combined <- bind_rows(existing, new_data)
+      rm(existing); gc()
+      
       if ("PitchUID" %in% names(combined)) {
         combined <- combined %>% distinct(PitchUID, .keep_all = TRUE)
       } else {
@@ -2073,6 +2077,7 @@ observeEvent(input$upload_hf_btn, {
       }
     } else {
       combined <- new_data
+      rm(existing); gc()
     }
     
     new_rows <- nrow(combined) - existing_rows
@@ -2081,9 +2086,10 @@ observeEvent(input$upload_hf_btn, {
     
     tmp <- tempfile(fileext = ".parquet")
     arrow::write_parquet(combined, tmp)
+    rm(combined); gc()
     
     result <- tryCatch({
-      hf <- import("huggingface_hub")
+      hf <- reticulate::import("huggingface_hub")
       api <- hf$HfApi()
       api$upload_file(
         path_or_fileobj = tmp,
@@ -2092,7 +2098,7 @@ observeEvent(input$upload_hf_btn, {
         repo_type = "dataset",
         token = hf_token
       )
-      paste0(label, ": ", scraped_rows, " rows scraped, ", new_rows, " new rows added (", nrow(combined), " total)")
+      paste0(label, ": ", scraped_rows, " rows scraped, ", new_rows, " new rows added (", nrow(new_data), " + ", existing_rows, " existing = deduped)")
     }, error = function(e) {
       paste0(label, " upload error: ", e$message)
     })
@@ -2102,21 +2108,29 @@ observeEvent(input$upload_hf_btn, {
     return(result)
   }
   
+  # Upload main dataset first
   main_file <- paste0(input$scrape_source, "_2026_master.parquet")
   msg1 <- upload_to_hf(scraped_data(), main_file, "Master Dataset")
+  gc()
   
   if (input$scrape_source == "pbp") {
+    # Filter THEN upload one at a time to save memory
     coastal_pitchers <- scraped_data() %>% filter(PitcherTeam == "COA_CHA")
-    coastal_hitters  <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
     
     msg2 <- if (nrow(coastal_pitchers) > 0) {
       upload_to_hf(coastal_pitchers, "CoastalPitchers2026.parquet", "Coastal Pitchers")
     } else { "Coastal Pitchers: No matching rows" }
     
+    rm(coastal_pitchers); gc()
+    
+    coastal_hitters <- scraped_data() %>% filter(BatterTeam == "COA_CHA")
+    
     msg3 <- if (nrow(coastal_hitters) > 0) {
       upload_to_hf(coastal_hitters, "CoastalHitters2026.parquet", "Coastal Hitters")
     } else { "Coastal Hitters: No matching rows" }
     
+    rm(coastal_hitters); gc()
+    
     scrape_status_msg(paste(msg1, msg2, msg3, sep = "\n"))
   } else {
     scrape_status_msg(msg1)