Update app.R

app.R

@@ -7,6 +7,7 @@ library(httr)
 library(bslib)
 library(rtabulator)
 library(purrr)
+library(arrow)
 
 download_private_csv <- function(repo_id, filename) {
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
@@ -65,16 +66,83 @@ download_private_csv <- function(repo_id, filename) {
   }
 }
 
-MLB25 <- download_private_csv("TimStats/StatcastDataAll", "MLB25.csv")
+download_private_parquet <- function(repo_id, filename) {
+  library(httr)
+  library(arrow)
+  
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+  
+  if (status_code(response) == 200) {
+    # First check if it's an LFS pointer
+    content_raw <- content(response, "raw")
+    content_text <- rawToChar(content_raw)
+    
+    # Check if this is an LFS pointer
+    if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
+      # Extract the oid (hash) from the pointer
+      oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
+      oid <- gsub("oid sha256:", "", oid_line)
+      oid <- trimws(oid)
+      
+      # Construct the LFS content URL
+      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
+                        substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
+      
+      # Get the actual content from LFS storage
+      lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+      
+      if (status_code(lfs_response) == 200) {
+        content_raw <- content(lfs_response, "raw")
+      } else {
+        # Alternative LFS URL format
+        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
+        lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+        
+        if (status_code(lfs_response) == 200) {
+          content_raw <- content(lfs_response, "raw")
+        } else {
+          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+        }
+      }
+    }
+    
+    # Write the raw content to a temporary file
+    temp_file <- tempfile(fileext = ".parquet")
+    writeBin(content_raw, temp_file)
+    
+    # Read the parquet file using arrow
+    tryCatch({
+      data <- read_parquet(temp_file)
+      
+      # Clean up - remove the temporary file
+      if (file.exists(temp_file)) {
+        file.remove(temp_file)
+      }
+      
+      return(data)
+    }, error = function(e) {
+      # Clean up in case of error
+      if (file.exists(temp_file)) {
+        file.remove(temp_file)
+      }
+      stop(paste("Error reading parquet file:", e$message))
+    })
+  } else {
+    stop(paste("Failed to download dataset. Status code:", status_code(response)))
+  }
+}
+
+MLB25 <- download_private_parquet("TimStats/StatcastDataAll", "MLB25.parquet")
 MLB25$level <- "MLB"
-AAA25 <- download_private_csv("TimStats/StatcastDataAll", "AAA25.csv")
+AAA25 <- download_private_parquet("TimStats/StatcastDataAll", "AAA25.parquet")
 AAA25$level <- "AAA"
 #ST <- read.csv("SpringT25.csv", header = TRUE, check.names = FALSE, fileEncoding = "UTF-8")
 
 #names(ST)
-MLB <- download_private_csv("TimStats/StatcastDataAll", "MLB.csv")
-AAA <- download_private_csv("TimStats/StatcastDataAll", "AAA.csv")
-FSL <- download_private_csv("TimStats/StatcastDataAll", "FSL.csv")
+MLB <- download_private_parquet("TimStats/StatcastDataAll", "MLB.parquet")
+AAA <- download_private_parquet("TimStats/StatcastDataAll", "AAA.parquet")
+FSL <- download_private_parquet("TimStats/StatcastDataAll", "FSL.parquet")
 MLB <- rbind(MLB,MLB25)
 AAA <- rbind(AAA,AAA25)