Update app.R

app.R

@@ -70,118 +70,67 @@ download_private_parquet <- function(repo_id, filename) {
   library(httr)
   library(arrow)
   
+  # Main URL for direct file download
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
-  response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+  
+  # Create a temporary file to save the download
+  temp_file <- tempfile(fileext = ".parquet")
+  
+  # Try direct download first
+  response <- GET(
+    url,
+    add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+    write_disk(temp_file, overwrite = TRUE)
+  )
   
   if (status_code(response) == 200) {
-    content_raw <- content(response, "raw")
-    
-    # Check first few bytes to see if it's a parquet file
-    # Parquet files start with "PAR1"
-    if (length(content_raw) >= 4 && 
-        rawToChar(content_raw[1:4]) == "PAR1") {
-      # It's a parquet file, process it directly
-      temp_file <- tempfile(fileext = ".parquet")
-      writeBin(content_raw, temp_file)
+    # Try to read the downloaded file as parquet
+    tryCatch({
+      data <- read_parquet(temp_file)
+      file.remove(temp_file)
+      return(data)
+    }, error = function(e) {
+      # Check if this might be an LFS pointer
+      content_text <- readLines(temp_file, warn = FALSE)
       
-      tryCatch({
-        data <- read_parquet(temp_file)
+      # Check if this is an LFS pointer
+      if (any(grepl("^version https://git-lfs.github.com/spec/", content_text))) {
+        # Extract the oid (hash) from the pointer
+        oid_line <- grep("oid sha256:", content_text, value = TRUE)
+        oid <- gsub("oid sha256:", "", oid_line)
+        oid <- trimws(oid)
         
-        # Clean up
-        if (file.exists(temp_file)) {
-          file.remove(temp_file)
-        }
+        # Alternative LFS URL format (this is more reliable for Hugging Face)
+        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
         
-        return(data)
-      }, error = function(e) {
-        if (file.exists(temp_file)) {
-          file.remove(temp_file)
-        }
-        stop(paste("Error reading parquet file:", e$message))
-      })
-    } else {
-      # It might be an LFS pointer, try to safely read it as text
-      tryCatch({
-        # Try to convert only the start of the content to check for LFS pointer
-        # This avoids issues with embedded nulls
-        first_100_chars <- rawToChar(content_raw[1:min(100, length(content_raw))])
+        # Download LFS content to the temp file
+        lfs_response <- GET(
+          lfs_url,
+          add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+          write_disk(temp_file, overwrite = TRUE)
+        )
         
-        if (grepl("^version https://git-lfs.github.com/spec/", first_100_chars)) {
-          # Convert the whole content to text, assuming it's small and valid text
-          content_text <- rawToChar(content_raw)
-          
-          # Extract the oid (hash) from the pointer
-          oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
-          oid <- gsub("oid sha256:", "", oid_line)
-          oid <- trimws(oid)
-          
-          # Construct the LFS content URL
-          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
-                          substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
-          
-          # Get the actual content from LFS storage
-          lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
-          
-          if (status_code(lfs_response) == 200) {
-            content_raw <- content(lfs_response, "raw")
-          } else {
-            # Alternative LFS URL format
-            lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
-            lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
-            
-            if (status_code(lfs_response) == 200) {
-              content_raw <- content(lfs_response, "raw")
-            } else {
-              stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
-            }
-          }
-          
-          # Write to temp file and read with arrow
-          temp_file <- tempfile(fileext = ".parquet")
-          writeBin(content_raw, temp_file)
-          
+        if (status_code(lfs_response) == 200) {
+          # Try to read the LFS file as parquet
           tryCatch({
             data <- read_parquet(temp_file)
-            
-            # Clean up
-            if (file.exists(temp_file)) {
-              file.remove(temp_file)
-            }
-            
+            file.remove(temp_file)
             return(data)
-          }, error = function(e) {
-            if (file.exists(temp_file)) {
-              file.remove(temp_file)
-            }
-            stop(paste("Error reading parquet file:", e$message))
+          }, error = function(e2) {
+            file.remove(temp_file)
+            stop(paste("Downloaded LFS content is not a valid parquet file:", e2$message))
           })
         } else {
-          # Not an LFS pointer, but also not a valid parquet file
-          stop("Downloaded content is neither a valid parquet file nor an LFS pointer")
+          file.remove(temp_file)
+          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
         }
-      }, error = function(e) {
-        # If we can't even read the first few bytes as text, assume it's a binary parquet file
-        temp_file <- tempfile(fileext = ".parquet")
-        writeBin(content_raw, temp_file)
-        
-        tryCatch({
-          data <- read_parquet(temp_file)
-          
-          # Clean up
-          if (file.exists(temp_file)) {
-            file.remove(temp_file)
-          }
-          
-          return(data)
-        }, error = function(e2) {
-          if (file.exists(temp_file)) {
-            file.remove(temp_file)
-          }
-          stop(paste("Error reading content as parquet:", e2$message))
-        })
-      })
-    }
+      } else {
+        file.remove(temp_file)
+        stop(paste("Downloaded content is not a valid parquet file:", e$message))
+      }
+    })
   } else {
+    file.remove(temp_file)
     stop(paste("Failed to download dataset. Status code:", status_code(response)))
   }
 }