Update app.R

app.R

@@ -656,7 +656,6 @@ ui <- fluidPage(
                       
                       br(),
                       actionButton("scrape_btn", "Scrape Data", class = "btn-primary"),
-                      actionButton("fetch_btn", "Fetch Results", class = "btn-primary"),
                       br(), br(),
                       downloadButton("download_scrape", "Download CSV")
                     ),
@@ -703,6 +702,7 @@ server <- function(input, output, session) {
   bat_tracking_parsed <- reactiveVal(NULL)
   merge_result <- reactiveVal(NULL)
   scraped_data <- reactiveVal(NULL)
+  scrape_polling <- reactiveVal(FALSE)
 
   scrape_status_msg <- reactiveVal("Ready.")
   
@@ -1591,46 +1591,81 @@ observeEvent(input$scrape_btn, {
   if (is.null(result)) return()
   
   if (httr::status_code(result) == 204) {
-    scrape_status_msg("Scrape triggered! GitHub is running it now. Wait a few minutes then click 'Fetch Results'.")
+    scrape_status_msg("Scrape triggered! Waiting for GitHub to finish...")
+    scrape_polling(TRUE)
   } else {
     scrape_status_msg(paste("GitHub API error:", httr::status_code(result)))
   }
 })
 
-# Fetch results from GitHub
-observeEvent(input$fetch_btn, {
-  scrape_status_msg("Fetching results...")
+# Poll GitHub every 15 seconds to check if done
+observe({
+  req(scrape_polling())
+  
+  invalidateLater(15000, session)
   
   gh_token <- Sys.getenv("GITHUB_TOKEN")
   gh_repo <- Sys.getenv("GITHUB_REPO")
-  filename <- paste0(input$scrape_source, "_", input$start_date, "_to_", input$end_date, ".csv")
-  
-  url <- paste0("https://api.github.com/repos/", gh_repo, "/contents/data/", filename)
   
-  result <- tryCatch({
-    resp <- httr::GET(
-      url,
+  resp <- tryCatch({
+    httr::GET(
+      paste0("https://api.github.com/repos/", gh_repo, "/actions/runs?per_page=1"),
       httr::add_headers(
         Authorization = paste("Bearer", gh_token),
-        Accept = "application/vnd.github.v3.raw"
+        Accept = "application/vnd.github.v3+json"
       )
     )
+  }, error = function(e) { NULL })
+  
+  if (is.null(resp)) return()
+  
+  runs <- jsonlite::fromJSON(httr::content(resp, as = "text", encoding = "UTF-8"))
+  
+  if (length(runs$workflow_runs) == 0) return()
+  
+  latest <- runs$workflow_runs[1, ]
+  status <- latest$status
+  conclusion <- latest$conclusion
+  
+  if (status == "completed") {
+    scrape_polling(FALSE)
     
-    if (httr::status_code(resp) == 200) {
-      read.csv(text = httr::content(resp, as = "text", encoding = "UTF-8"))
+    if (conclusion == "success") {
+      scrape_status_msg("GitHub finished! Fetching data...")
+      
+      # Auto-fetch the CSV
+      filename <- paste0(input$scrape_source, "_", input$start_date, "_to_", input$end_date, ".csv")
+      url <- paste0("https://api.github.com/repos/", gh_repo, "/contents/data/", filename)
+      
+      data <- tryCatch({
+        file_resp <- httr::GET(
+          url,
+          httr::add_headers(
+            Authorization = paste("Bearer", gh_token),
+            Accept = "application/vnd.github.v3.raw"
+          )
+        )
+        
+        if (httr::status_code(file_resp) == 200) {
+          read.csv(text = httr::content(file_resp, as = "text", encoding = "UTF-8"))
+        } else {
+          NULL
+        }
+      }, error = function(e) { NULL })
+      
+      if (!is.null(data) && nrow(data) > 0) {
+        scraped_data(data)
+        scrape_status_msg(paste0("Done! ", nrow(data), " rows × ", ncol(data), " columns."))
+      } else {
+        scrape_status_msg("Scrape finished but couldn't fetch the file. Try 'Fetch Results' manually.")
+      }
+      
     } else {
-      NULL
+      scrape_status_msg(paste("GitHub Action failed:", conclusion))
     }
-  }, error = function(e) {
-    scrape_status_msg(paste("Error:", e$message))
-    NULL
-  })
-  
-  if (!is.null(result) && nrow(result) > 0) {
-    scraped_data(result)
-    scrape_status_msg(paste0("Done! ", nrow(result), " rows × ", ncol(result), " columns."))
+    
   } else {
-    scrape_status_msg("Not ready yet. Wait a minute and try again.")
+    scrape_status_msg(paste0("GitHub is running... (status: ", status, ")"))
   }
 })