Update app.R

app.R

@@ -73,61 +73,114 @@ download_private_parquet <- function(repo_id, filename) {
   # Main URL for direct file download
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
   
-  # Create a temporary file to save the download
+  # Download with curl for more control and debugging
   temp_file <- tempfile(fileext = ".parquet")
   
-  # Try direct download first
-  response <- GET(
+  # Use httr's RETRY for more reliable downloads
+  response <- RETRY(
+    "GET",
     url,
     add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-    write_disk(temp_file, overwrite = TRUE)
+    write_disk(temp_file, overwrite = TRUE),
+    times = 3
   )
   
   if (status_code(response) == 200) {
-    # Try to read the downloaded file as parquet
+    # Check file size
+    file_info <- file.info(temp_file)
+    
+    if (file_info$size == 0) {
+      file.remove(temp_file)
+      stop("Downloaded file is empty")
+    }
+    
+    # Check if this looks like an LFS pointer (small file)
+    if (file_info$size < 1000) {  # LFS pointers are typically small
+      tryCatch({
+        # Read the first line to see if it's an LFS pointer
+        content_text <- readLines(temp_file, n = 1, warn = FALSE)
+        
+        if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
+          # It's an LFS pointer - read all lines
+          all_lines <- readLines(temp_file, warn = FALSE)
+          
+          # Extract oid
+          oid_line <- grep("oid sha256:", all_lines, value = TRUE)
+          if (length(oid_line) == 0) {
+            file.remove(temp_file)
+            stop("Could not find oid in LFS pointer")
+          }
+          
+          oid <- gsub("oid sha256:", "", oid_line)
+          oid <- trimws(oid)
+          
+          # Try the more reliable Hugging Face LFS URL format
+          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename, "?download=true")
+          
+          cat("Detected LFS pointer. Downloading from:", lfs_url, "\n")
+          
+          # Download actual content
+          lfs_response <- RETRY(
+            "GET",
+            lfs_url,
+            add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+            write_disk(temp_file, overwrite = TRUE),
+            times = 3
+          )
+          
+          if (status_code(lfs_response) != 200) {
+            file.remove(temp_file)
+            stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+          }
+        }
+      }, error = function(e) {
+        # Not an LFS pointer or error reading it
+        cat("Not an LFS pointer or error:", e$message, "\n")
+      })
+    }
+    
+    # Try to read as parquet, with detailed error reporting
     tryCatch({
+      file_info <- file.info(temp_file)
+      cat("File size before reading:", file_info$size, "bytes\n")
+      
+      # Read first few bytes to check for parquet signature
+      con <- file(temp_file, "rb")
+      header <- readBin(con, "raw", n = 4)
+      close(con)
+      
+      if (rawToChar(header) != "PAR1") {
+        cat("WARNING: File does not start with PAR1 magic bytes\n")
+      }
+      
+      # Try to read as parquet
       data <- read_parquet(temp_file)
       file.remove(temp_file)
       return(data)
     }, error = function(e) {
-      # Check if this might be an LFS pointer
-      content_text <- readLines(temp_file, warn = FALSE)
+      # If read_parquet fails, try using httr to download the file directly to a specific location
+      # Sometimes that can help with file corruption issues
+      download_path <- file.path(getwd(), basename(filename))
       
-      # Check if this is an LFS pointer
-      if (any(grepl("^version https://git-lfs.github.com/spec/", content_text))) {
-        # Extract the oid (hash) from the pointer
-        oid_line <- grep("oid sha256:", content_text, value = TRUE)
-        oid <- gsub("oid sha256:", "", oid_line)
-        oid <- trimws(oid)
-        
-        # Alternative LFS URL format (this is more reliable for Hugging Face)
-        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
-        
-        # Download LFS content to the temp file
-        lfs_response <- GET(
-          lfs_url,
-          add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-          write_disk(temp_file, overwrite = TRUE)
-        )
-        
-        if (status_code(lfs_response) == 200) {
-          # Try to read the LFS file as parquet
-          tryCatch({
-            data <- read_parquet(temp_file)
-            file.remove(temp_file)
-            return(data)
-          }, error = function(e2) {
-            file.remove(temp_file)
-            stop(paste("Downloaded LFS content is not a valid parquet file:", e2$message))
-          })
-        } else {
-          file.remove(temp_file)
-          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
-        }
-      } else {
-        file.remove(temp_file)
-        stop(paste("Downloaded content is not a valid parquet file:", e$message))
+      cat("Failed to read as parquet. Downloading to:", download_path, "\n")
+      cat("Error was:", e$message, "\n")
+      
+      download_response <- RETRY(
+        "GET",
+        url,
+        add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+        write_disk(download_path, overwrite = TRUE),
+        times = 3
+      )
+      
+      if (status_code(download_response) == 200) {
+        cat("Downloaded file to:", download_path, "\n")
+        cat("Please try reading this file directly with read_parquet(\"", download_path, "\")\n", sep = "")
       }
+      
+      file.remove(temp_file)
+      stop(paste("Could not read file as parquet:", e$message, 
+                "\nFile downloaded to:", download_path))
     })
   } else {
     file.remove(temp_file)