Update app.R

app.R

@@ -70,171 +70,57 @@ download_private_parquet <- function(repo_id, filename) {
   library(httr)
   library(arrow)
   
-  # Main URL for direct file download
-  url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
-  
-  # Download with curl for more control and debugging
-  temp_file <- tempfile(fileext = ".parquet")
-  
-  # Use httr's RETRY for more reliable downloads
-  response <- RETRY(
-    "GET",
-    url,
-    add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-    write_disk(temp_file, overwrite = TRUE),
-    times = 3
-  )
-  
-  if (status_code(response) == 200) {
-    # Check file size
-    file_info <- file.info(temp_file)
-    
-    if (file_info$size == 0) {
-      file.remove(temp_file)
-      stop("Downloaded file is empty")
-    }
-    
-    # Check if this looks like an LFS pointer (small file)
-    if (file_info$size < 1000) {  # LFS pointers are typically small
-      tryCatch({
-        # Read the first line to see if it's an LFS pointer
-        content_text <- readLines(temp_file, n = 1, warn = FALSE)
-        
-        if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
-          # It's an LFS pointer - read all lines
-          all_lines <- readLines(temp_file, warn = FALSE)
-          
-          # Extract oid
-          oid_line <- grep("oid sha256:", all_lines, value = TRUE)
-          if (length(oid_line) == 0) {
-            file.remove(temp_file)
-            stop("Could not find oid in LFS pointer")
-          }
-          
-          oid <- gsub("oid sha256:", "", oid_line)
-          oid <- trimws(oid)
-          
-          # Try the more reliable Hugging Face LFS URL format
-          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename, "?download=true")
-          
-          cat("Detected LFS pointer. Downloading from:", lfs_url, "\n")
-          
-          # Download actual content
-          lfs_response <- RETRY(
-            "GET",
-            lfs_url,
-            add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-            write_disk(temp_file, overwrite = TRUE),
-            times = 3
-          )
-          
-          if (status_code(lfs_response) != 200) {
-            file.remove(temp_file)
-            stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
-          }
-        }
-      }, error = function(e) {
-        # Not an LFS pointer or error reading it
-        cat("Not an LFS pointer or error:", e$message, "\n")
-      })
-    }
-    
-    # Try to read as parquet, with detailed error reporting
-    tryCatch({
-      file_info <- file.info(temp_file)
-      cat("File size before reading:", file_info$size, "bytes\n")
-      
-      # Read first few bytes to check for parquet signature
-      con <- file(temp_file, "rb")
-      header <- readBin(con, "raw", n = 4)
-      close(con)
-      
-      if (rawToChar(header) != "PAR1") {
-        cat("WARNING: File does not start with PAR1 magic bytes\n")
-      }
-      
-      # Try to read as parquet
-      data <- read_parquet(temp_file)
-      file.remove(temp_file)
-      return(data)
-    }, error = function(e) {
-      # If read_parquet fails, try using httr to download the file directly to a specific location
-      # Sometimes that can help with file corruption issues
-      download_path <- file.path(getwd(), basename(filename))
-      
-      cat("Failed to read as parquet. Downloading to:", download_path, "\n")
-      cat("Error was:", e$message, "\n")
-      
-      download_response <- RETRY(
-        "GET",
-        url,
-        add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-        write_disk(download_path, overwrite = TRUE),
-        times = 3
-      )
-      
-      if (status_code(download_response) == 200) {
-        cat("Downloaded file to:", download_path, "\n")
-        cat("Please try reading this file directly with read_parquet(\"", download_path, "\")\n", sep = "")
-      }
-      
-      file.remove(temp_file)
-      stop(paste("Could not read file as parquet:", e$message, 
-                "\nFile downloaded to:", download_path))
-    })
-  } else {
-    file.remove(temp_file)
-    stop(paste("Failed to download dataset. Status code:", status_code(response)))
-  }
-}
-
-download_private_parquet <- function(repo_id, filename) {
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
   response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
   
   if (status_code(response) == 200) {
-    # Get content as text first to check if it's an LFS pointer
-    content_text <- content(response, "text", encoding = "UTF-8")
+    # Get raw content
+    content_raw <- content(response, "raw")
     
-    # Check if this is an LFS pointer (LFS files start with "version https://git-lfs.github.com/spec/")
-    if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
+    # Check if this is an LFS pointer by looking at the first few bytes
+    first_chunk <- rawToChar(head(content_raw, 100))
+    
+    if (grepl("^version https://git-lfs.github.com/spec/", first_chunk)) {
       # This is an LFS file - extract the oid (hash) from the pointer
+      content_text <- rawToChar(content_raw)
       oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
       oid <- gsub("oid sha256:", "", oid_line)
       oid <- trimws(oid)
       
-      # Construct the LFS content URL
-      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
-                        substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
-      
-      # Get the actual content from LFS storage
+      # Try the alternative LFS URL format first (more reliable for Hugging Face)
+      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
       lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
       
       if (status_code(lfs_response) == 200) {
-        content_text <- content(lfs_response, "text", encoding = "UTF-8")
+        content_raw <- content(lfs_response, "raw")
       } else {
-        # Alternative LFS URL format
-        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
+        # Try original LFS URL format as fallback
+        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
+                          substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
         lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
         
         if (status_code(lfs_response) == 200) {
-          content_text <- content(lfs_response, "text", encoding = "UTF-8")
+          content_raw <- content(lfs_response, "raw")
         } else {
           stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
         }
       }
     }
     
-    # Process the content (whether it was LFS or regular)
-    con <- textConnection(content_text)
+    # Write to a temporary file and read with arrow
+    temp_file <- tempfile(fileext = ".parquet")
+    writeBin(content_raw, temp_file)
+    
     tryCatch({
-      data <- read_parquet(con)
+      data <- read_parquet(temp_file)
       return(data)
     }, error = function(e) {
-      close(con)
-      stop(paste("Error parsing CSV:", e$message))
-    }, finally = {
-      close(con)
+      stop(paste("Error parsing parquet file:", e$message))
+    }, finally {
+      # Clean up
+      if (file.exists(temp_file)) {
+        file.remove(temp_file)
+      }
     })
   } else {
     stop(paste("Failed to download dataset. Status code:", status_code(response)))