Update app.R

app.R

@@ -189,78 +189,55 @@ download_private_parquet <- function(repo_id, filename) {
 }
 
 download_private_parquet <- function(repo_id, filename) {
-  library(httr)
-  library(arrow)
-  
-  # Main URL for direct file download
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
-  
-  cat("Downloading from URL:", url, "\n")
-  
-  # Download the file content directly into memory
-  response <- RETRY(
-    "GET",
-    url,
-    add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-    times = 3
-  )
+  response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
   
   if (status_code(response) == 200) {
-    # Get the raw binary content
-    content_raw <- content(response, "raw")
-    cat("Content downloaded, size:", length(content_raw), "bytes\n")
-    
-    # Write to a temporary file in a directory we should have access to
-    temp_dir <- tempdir()
-    temp_file <- file.path(temp_dir, basename(filename))
-    
-    cat("Writing to temporary file:", temp_file, "\n")
-    writeBin(content_raw, temp_file)
+    # Get content as text first to check if it's an LFS pointer
+    content_text <- content(response, "text", encoding = "UTF-8")
     
-    # Try to read the parquet file
-    tryCatch({
-      cat("Reading parquet file...\n")
-      data <- read_parquet(temp_file)
+    # Check if this is an LFS pointer (LFS files start with "version https://git-lfs.github.com/spec/")
+    if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
+      # This is an LFS file - extract the oid (hash) from the pointer
+      oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
+      oid <- gsub("oid sha256:", "", oid_line)
+      oid <- trimws(oid)
       
-      # Clean up
-      unlink(temp_file)
+      # Construct the LFS content URL
+      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
+                        substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
       
-      cat("Successfully read parquet file\n")
-      return(data)
-    }, error = function(e) {
-      cat("Error reading parquet file:", e$message, "\n")
+      # Get the actual content from LFS storage
+      lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
       
-      # Try an alternative approach using the arrow C++ library directly
-      cat("Trying alternative method...\n")
-      tryCatch({
-        # Create a buffer from raw content
-        buffer <- arrow::py_buffer(content_raw)
-        
-        # Read from the buffer
-        data <- arrow::read_parquet(buffer)
-        
-        unlink(temp_file)
-        cat("Successfully read parquet using buffer method\n")
-        return(data)
-      }, error = function(e2) {
-        cat("Alternative method failed:", e2$message, "\n")
+      if (status_code(lfs_response) == 200) {
+        content_text <- content(lfs_response, "text", encoding = "UTF-8")
+      } else {
+        # Alternative LFS URL format
+        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
+        lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
         
-        # Try with different options
-        tryCatch({
-          cat("Trying with options...\n")
-          data <- read_parquet(temp_file, mmap = FALSE)
-          
-          unlink(temp_file)
-          cat("Successfully read parquet with mmap=FALSE\n")
-          return(data)
-        }, error = function(e3) {
-          cat("All methods failed to read the parquet file\n")
-          stop(paste("Failed to read parquet file:", e3$message))
-        })
-      })
+        if (status_code(lfs_response) == 200) {
+          content_text <- content(lfs_response, "text", encoding = "UTF-8")
+        } else {
+          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+        }
+      }
+    }
+    
+    # Process the content (whether it was LFS or regular)
+    con <- textConnection(content_text)
+    tryCatch({
+      data <- read_parquet(con)
+      return(data)
+    }, error = function(e) {
+      close(con)
+      stop(paste("Error parsing CSV:", e$message))
+    }, finally = {
+      close(con)
     })
   } else {
-    stop(paste("Failed to download file. Status code:", status_code(response)))
+    stop(paste("Failed to download dataset. Status code:", status_code(response)))
   }
 }