Update app.R

app.R

@@ -188,6 +188,166 @@ download_private_parquet <- function(repo_id, filename) {
   }
 }
 
+download_private_parquet <- function(repo_id, filename) {
+  library(httr)
+  
+  # Main URL for direct file download
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+  
+  # Download with retry for reliability
+  temp_file <- tempfile(fileext = paste0(".", tools::file_ext(filename)))
+  
+  cat("Downloading from URL:", url, "\n")
+  
+  response <- RETRY(
+    "GET",
+    url,
+    add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+    write_disk(temp_file, overwrite = TRUE),
+    times = 3
+  )
+  
+  if (status_code(response) == 200) {
+    # Check file size
+    file_info <- file.info(temp_file)
+    cat("File downloaded, size:", file_info$size, "bytes\n")
+    
+    if (file_info$size == 0) {
+      file.remove(temp_file)
+      stop("Downloaded file is empty")
+    }
+    
+    # Check if this is an LFS pointer (small text file)
+    if (file_info$size < 1000) {
+      tryCatch({
+        content_text <- readLines(temp_file, n = 1, warn = FALSE)
+        
+        if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
+          # It's an LFS pointer - read all lines
+          all_lines <- readLines(temp_file, warn = FALSE)
+          
+          # Extract oid
+          oid_line <- grep("oid sha256:", all_lines, value = TRUE)
+          oid <- gsub("oid sha256:", "", oid_line)
+          oid <- trimws(oid)
+          
+          # Try the Hugging Face LFS URL format
+          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename, "?download=true")
+          
+          cat("Detected LFS pointer. Downloading from:", lfs_url, "\n")
+          
+          # Download actual content
+          lfs_response <- RETRY(
+            "GET",
+            lfs_url,
+            add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+            write_disk(temp_file, overwrite = TRUE),
+            times = 3
+          )
+          
+          if (status_code(lfs_response) != 200) {
+            file.remove(temp_file)
+            stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+          }
+          
+          file_info <- file.info(temp_file)
+          cat("LFS file downloaded, size:", file_info$size, "bytes\n")
+        }
+      }, error = function(e) {
+        cat("Not an LFS pointer or error reading it:", e$message, "\n")
+      })
+    }
+    
+    # Try to detect file type by reading first few bytes
+    con <- file(temp_file, "rb")
+    header <- readBin(con, "raw", n = 8)
+    close(con)
+    
+    # Convert to character for inspection
+    header_str <- rawToChar(header)
+    
+    # Create a download path in the working directory for reference
+    download_path <- file.path(getwd(), basename(filename))
+    file.copy(temp_file, download_path, overwrite = TRUE)
+    cat("File copied to:", download_path, "\n")
+    
+    # Try different file formats based on header or extension
+    if (grepl("PAR1", header_str)) {
+      # It's a parquet file
+      cat("Detected parquet format\n")
+      library(arrow)
+      tryCatch({
+        data <- read_parquet(temp_file)
+        file.remove(temp_file)
+        return(data)
+      }, error = function(e) {
+        cat("Error reading as parquet:", e$message, "\n")
+        cat("Try reading manually with arrow::read_parquet('", download_path, "')\n", sep = "")
+        stop(paste("Error reading parquet file:", e$message))
+      })
+    } else if (grepl("^PK", header_str)) {
+      # It's a zip/xlsx file
+      cat("Detected zip/xlsx format\n")
+      if (grepl("\\.xlsx$", filename, ignore.case = TRUE)) {
+        library(readxl)
+        tryCatch({
+          data <- read_excel(temp_file)
+          file.remove(temp_file)
+          return(data)
+        }, error = function(e) {
+          cat("Error reading as Excel:", e$message, "\n")
+          cat("Try reading manually with readxl::read_excel('", download_path, "')\n", sep = "")
+          stop(paste("Error reading Excel file:", e$message))
+        })
+      } else {
+        cat("Zip file detected. Cannot automatically process.\n")
+        cat("File is available at:", download_path, "\n")
+        stop("Downloaded file appears to be a zip file. Please process manually.")
+      }
+    } else {
+      # Try as CSV
+      cat("Trying as CSV/text format\n")
+      
+      # Check if it looks like text
+      can_be_text <- tryCatch({
+        readLines(temp_file, n = 1, warn = FALSE)
+        TRUE
+      }, error = function(e) {
+        FALSE
+      })
+      
+      if (can_be_text) {
+        tryCatch({
+          data <- read.csv(temp_file, header = TRUE, stringsAsFactors = FALSE, fileEncoding = "UTF-8")
+          file.remove(temp_file)
+          return(data)
+        }, error = function(e) {
+          cat("Error reading as CSV:", e$message, "\n")
+          
+          # Try reading as TSV
+          tryCatch({
+            data <- read.delim(temp_file, header = TRUE, stringsAsFactors = FALSE, fileEncoding = "UTF-8")
+            file.remove(temp_file)
+            return(data)
+          }, error = function(e2) {
+            cat("Error reading as TSV:", e2$message, "\n")
+            cat("File is available at:", download_path, "\n")
+            cat("Try examining the file manually to determine format\n")
+            stop("Could not automatically determine file format")
+          })
+        })
+      } else {
+        cat("File does not appear to be text-based\n")
+        cat("File is available at:", download_path, "\n")
+        stop("Could not automatically determine file format. Please check the downloaded file.")
+      }
+    }
+  } else {
+    file.remove(temp_file)
+    stop(paste("Failed to download dataset. Status code:", status_code(response)))
+  }
+}
+
 MLB25 <- download_private_parquet("TimStats/StatcastDataAll", "MLB25.parquet")
 MLB25$level <- "MLB"
 AAA25 <- download_private_parquet("TimStats/StatcastDataAll", "AAA25.parquet")