Update app.R

app.R

@@ -67,63 +67,54 @@ download_private_csv <- function(repo_id, filename) {
 }
 
 download_private_parquet <- function(repo_id, filename) {
-  library(httr)
-  library(arrow)
   
-  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
-  response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+  # Create a temporary file
+  temp_file <- tempfile(fileext = ".parquet")
+  
+  # Alternative LFS URL format (more direct download)
+  url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
+  
+  # Download directly to file
+  response <- GET(
+    url,
+    add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+    write_disk(temp_file, overwrite = TRUE)
+  )
   
+  # Check if download was successful
   if (status_code(response) == 200) {
-    # Get raw content
-    content_raw <- content(response, "raw")
-    
-    # Check if this is an LFS pointer by looking at the first few bytes
-    first_chunk <- rawToChar(head(content_raw, 100))
-    
-    if (grepl("^version https://git-lfs.github.com/spec/", first_chunk)) {
-      # This is an LFS file - extract the oid (hash) from the pointer
-      content_text <- rawToChar(content_raw)
-      oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
-      oid <- gsub("oid sha256:", "", oid_line)
-      oid <- trimws(oid)
-      
-      # Try the alternative LFS URL format first (more reliable for Hugging Face)
-      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
-      lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
-      
-      if (status_code(lfs_response) == 200) {
-        content_raw <- content(lfs_response, "raw")
-      } else {
-        # Try original LFS URL format as fallback
-        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
-                          substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
-        lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
-        
-        if (status_code(lfs_response) == 200) {
-          content_raw <- content(lfs_response, "raw")
-        } else {
-          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
-        }
-      }
-    }
-    
-    # Write to a temporary file and read with arrow
-    temp_file <- tempfile(fileext = ".parquet")
-    writeBin(content_raw, temp_file)
-    
     tryCatch({
+      # Read the parquet file
       data <- read_parquet(temp_file)
+      file.remove(temp_file)
       return(data)
     }, error = function(e) {
-      stop(paste("Error parsing parquet file:", e$message))
-    }, finally {
-      # Clean up
-      if (file.exists(temp_file)) {
-        file.remove(temp_file)
-      }
+      file.remove(temp_file)
+      stop(paste("Error reading parquet file:", e$message))
     })
   } else {
-    stop(paste("Failed to download dataset. Status code:", status_code(response)))
+    # Try the standard URL if LFS URL failed
+    url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
+    
+    response <- GET(
+      url,
+      add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
+      write_disk(temp_file, overwrite = TRUE)
+    )
+    
+    if (status_code(response) == 200) {
+      tryCatch({
+        data <- read_parquet(temp_file)
+        file.remove(temp_file)
+        return(data)
+      }, error = function(e) {
+        file.remove(temp_file)
+        stop(paste("Error reading parquet file:", e$message))
+      })
+    } else {
+      file.remove(temp_file)
+      stop(paste("Failed to download file. Status code:", status_code(response)))
+    }
   }
 }