Update app.R

app.R

@@ -74,60 +74,113 @@ download_private_parquet <- function(repo_id, filename) {
   response <- GET(url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
   
   if (status_code(response) == 200) {
-    # First check if it's an LFS pointer
     content_raw <- content(response, "raw")
-    content_text <- rawToChar(content_raw)
     
-    # Check if this is an LFS pointer
-    if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
-      # Extract the oid (hash) from the pointer
-      oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
-      oid <- gsub("oid sha256:", "", oid_line)
-      oid <- trimws(oid)
-      
-      # Construct the LFS content URL
-      lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
-                        substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
-      
-      # Get the actual content from LFS storage
-      lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+    # Check first few bytes to see if it's a parquet file
+    # Parquet files start with "PAR1"
+    if (length(content_raw) >= 4 && 
+        rawToChar(content_raw[1:4]) == "PAR1") {
+      # It's a parquet file, process it directly
+      temp_file <- tempfile(fileext = ".parquet")
+      writeBin(content_raw, temp_file)
       
-      if (status_code(lfs_response) == 200) {
-        content_raw <- content(lfs_response, "raw")
-      } else {
-        # Alternative LFS URL format
-        lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
-        lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+      tryCatch({
+        data <- read_parquet(temp_file)
         
-        if (status_code(lfs_response) == 200) {
-          content_raw <- content(lfs_response, "raw")
+        # Clean up
+        if (file.exists(temp_file)) {
+          file.remove(temp_file)
+        }
+        
+        return(data)
+      }, error = function(e) {
+        if (file.exists(temp_file)) {
+          file.remove(temp_file)
+        }
+        stop(paste("Error reading parquet file:", e$message))
+      })
+    } else {
+      # It might be an LFS pointer, try to safely read it as text
+      tryCatch({
+        # Try to convert only the start of the content to check for LFS pointer
+        # This avoids issues with embedded nulls
+        first_100_chars <- rawToChar(content_raw[1:min(100, length(content_raw))])
+        
+        if (grepl("^version https://git-lfs.github.com/spec/", first_100_chars)) {
+          # Convert the whole content to text, assuming it's small and valid text
+          content_text <- rawToChar(content_raw)
+          
+          # Extract the oid (hash) from the pointer
+          oid_line <- grep("oid sha256:", strsplit(content_text, "\n")[[1]], value = TRUE)
+          oid <- gsub("oid sha256:", "", oid_line)
+          oid <- trimws(oid)
+          
+          # Construct the LFS content URL
+          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/.git/lfs/objects/", 
+                          substr(oid, 1, 2), "/", substr(oid, 3, 4), "/", oid)
+          
+          # Get the actual content from LFS storage
+          lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+          
+          if (status_code(lfs_response) == 200) {
+            content_raw <- content(lfs_response, "raw")
+          } else {
+            # Alternative LFS URL format
+            lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/lfs/resolve/main/", filename, "?download=true")
+            lfs_response <- GET(lfs_url, add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))))
+            
+            if (status_code(lfs_response) == 200) {
+              content_raw <- content(lfs_response, "raw")
+            } else {
+              stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+            }
+          }
+          
+          # Write to temp file and read with arrow
+          temp_file <- tempfile(fileext = ".parquet")
+          writeBin(content_raw, temp_file)
+          
+          tryCatch({
+            data <- read_parquet(temp_file)
+            
+            # Clean up
+            if (file.exists(temp_file)) {
+              file.remove(temp_file)
+            }
+            
+            return(data)
+          }, error = function(e) {
+            if (file.exists(temp_file)) {
+              file.remove(temp_file)
+            }
+            stop(paste("Error reading parquet file:", e$message))
+          })
         } else {
-          stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
+          # Not an LFS pointer, but also not a valid parquet file
+          stop("Downloaded content is neither a valid parquet file nor an LFS pointer")
         }
-      }
+      }, error = function(e) {
+        # If we can't even read the first few bytes as text, assume it's a binary parquet file
+        temp_file <- tempfile(fileext = ".parquet")
+        writeBin(content_raw, temp_file)
+        
+        tryCatch({
+          data <- read_parquet(temp_file)
+          
+          # Clean up
+          if (file.exists(temp_file)) {
+            file.remove(temp_file)
+          }
+          
+          return(data)
+        }, error = function(e2) {
+          if (file.exists(temp_file)) {
+            file.remove(temp_file)
+          }
+          stop(paste("Error reading content as parquet:", e2$message))
+        })
+      })
     }
-    
-    # Write the raw content to a temporary file
-    temp_file <- tempfile(fileext = ".parquet")
-    writeBin(content_raw, temp_file)
-    
-    # Read the parquet file using arrow
-    tryCatch({
-      data <- read_parquet(temp_file)
-      
-      # Clean up - remove the temporary file
-      if (file.exists(temp_file)) {
-        file.remove(temp_file)
-      }
-      
-      return(data)
-    }, error = function(e) {
-      # Clean up in case of error
-      if (file.exists(temp_file)) {
-        file.remove(temp_file)
-      }
-      stop(paste("Error reading parquet file:", e$message))
-    })
   } else {
     stop(paste("Failed to download dataset. Status code:", status_code(response)))
   }