Update app.R

app.R

@@ -190,161 +190,77 @@ download_private_parquet <- function(repo_id, filename) {
 
 download_private_parquet <- function(repo_id, filename) {
   library(httr)
+  library(arrow)
   
   # Main URL for direct file download
   url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename)
   
-  # Download with retry for reliability
-  temp_file <- tempfile(fileext = paste0(".", tools::file_ext(filename)))
-  
   cat("Downloading from URL:", url, "\n")
   
+  # Download the file content directly into memory
   response <- RETRY(
     "GET",
     url,
     add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-    write_disk(temp_file, overwrite = TRUE),
     times = 3
   )
   
   if (status_code(response) == 200) {
-    # Check file size
-    file_info <- file.info(temp_file)
-    cat("File downloaded, size:", file_info$size, "bytes\n")
+    # Get the raw binary content
+    content_raw <- content(response, "raw")
+    cat("Content downloaded, size:", length(content_raw), "bytes\n")
     
-    if (file_info$size == 0) {
-      file.remove(temp_file)
-      stop("Downloaded file is empty")
-    }
+    # Write to a temporary file in a directory we should have access to
+    temp_dir <- tempdir()
+    temp_file <- file.path(temp_dir, basename(filename))
     
-    # Check if this is an LFS pointer (small text file)
-    if (file_info$size < 1000) {
-      tryCatch({
-        content_text <- readLines(temp_file, n = 1, warn = FALSE)
-        
-        if (grepl("^version https://git-lfs.github.com/spec/", content_text)) {
-          # It's an LFS pointer - read all lines
-          all_lines <- readLines(temp_file, warn = FALSE)
-          
-          # Extract oid
-          oid_line <- grep("oid sha256:", all_lines, value = TRUE)
-          oid <- gsub("oid sha256:", "", oid_line)
-          oid <- trimws(oid)
-          
-          # Try the Hugging Face LFS URL format
-          lfs_url <- paste0("https://huggingface.co/datasets/", repo_id, "/resolve/main/", filename, "?download=true")
-          
-          cat("Detected LFS pointer. Downloading from:", lfs_url, "\n")
-          
-          # Download actual content
-          lfs_response <- RETRY(
-            "GET",
-            lfs_url,
-            add_headers(Authorization = paste("Bearer", Sys.getenv("GETCSV"))),
-            write_disk(temp_file, overwrite = TRUE),
-            times = 3
-          )
-          
-          if (status_code(lfs_response) != 200) {
-            file.remove(temp_file)
-            stop(paste("Failed to download LFS content. Status code:", status_code(lfs_response)))
-          }
-          
-          file_info <- file.info(temp_file)
-          cat("LFS file downloaded, size:", file_info$size, "bytes\n")
-        }
-      }, error = function(e) {
-        cat("Not an LFS pointer or error reading it:", e$message, "\n")
-      })
-    }
+    cat("Writing to temporary file:", temp_file, "\n")
+    writeBin(content_raw, temp_file)
     
-    # Try to detect file type by reading first few bytes
-    con <- file(temp_file, "rb")
-    header <- readBin(con, "raw", n = 8)
-    close(con)
-    
-    # Convert to character for inspection
-    header_str <- rawToChar(header)
-    
-    # Create a download path in the working directory for reference
-    download_path <- file.path(getwd(), basename(filename))
-    file.copy(temp_file, download_path, overwrite = TRUE)
-    cat("File copied to:", download_path, "\n")
-    
-    # Try different file formats based on header or extension
-    if (grepl("PAR1", header_str)) {
-      # It's a parquet file
-      cat("Detected parquet format\n")
-      library(arrow)
+    # Try to read the parquet file
+    tryCatch({
+      cat("Reading parquet file...\n")
+      data <- read_parquet(temp_file)
+      
+      # Clean up
+      unlink(temp_file)
+      
+      cat("Successfully read parquet file\n")
+      return(data)
+    }, error = function(e) {
+      cat("Error reading parquet file:", e$message, "\n")
+      
+      # Try an alternative approach using the arrow C++ library directly
+      cat("Trying alternative method...\n")
       tryCatch({
-        data <- read_parquet(temp_file)
-        file.remove(temp_file)
+        # Create a buffer from raw content
+        buffer <- arrow::py_buffer(content_raw)
+        
+        # Read from the buffer
+        data <- arrow::read_parquet(buffer)
+        
+        unlink(temp_file)
+        cat("Successfully read parquet using buffer method\n")
         return(data)
-      }, error = function(e) {
-        cat("Error reading as parquet:", e$message, "\n")
-        cat("Try reading manually with arrow::read_parquet('", download_path, "')\n", sep = "")
-        stop(paste("Error reading parquet file:", e$message))
-      })
-    } else if (grepl("^PK", header_str)) {
-      # It's a zip/xlsx file
-      cat("Detected zip/xlsx format\n")
-      if (grepl("\\.xlsx$", filename, ignore.case = TRUE)) {
-        library(readxl)
+      }, error = function(e2) {
+        cat("Alternative method failed:", e2$message, "\n")
+        
+        # Try with different options
         tryCatch({
-          data <- read_excel(temp_file)
-          file.remove(temp_file)
+          cat("Trying with options...\n")
+          data <- read_parquet(temp_file, mmap = FALSE)
+          
+          unlink(temp_file)
+          cat("Successfully read parquet with mmap=FALSE\n")
           return(data)
-        }, error = function(e) {
-          cat("Error reading as Excel:", e$message, "\n")
-          cat("Try reading manually with readxl::read_excel('", download_path, "')\n", sep = "")
-          stop(paste("Error reading Excel file:", e$message))
+        }, error = function(e3) {
+          cat("All methods failed to read the parquet file\n")
+          stop(paste("Failed to read parquet file:", e3$message))
         })
-      } else {
-        cat("Zip file detected. Cannot automatically process.\n")
-        cat("File is available at:", download_path, "\n")
-        stop("Downloaded file appears to be a zip file. Please process manually.")
-      }
-    } else {
-      # Try as CSV
-      cat("Trying as CSV/text format\n")
-      
-      # Check if it looks like text
-      can_be_text <- tryCatch({
-        readLines(temp_file, n = 1, warn = FALSE)
-        TRUE
-      }, error = function(e) {
-        FALSE
       })
-      
-      if (can_be_text) {
-        tryCatch({
-          data <- read.csv(temp_file, header = TRUE, stringsAsFactors = FALSE, fileEncoding = "UTF-8")
-          file.remove(temp_file)
-          return(data)
-        }, error = function(e) {
-          cat("Error reading as CSV:", e$message, "\n")
-          
-          # Try reading as TSV
-          tryCatch({
-            data <- read.delim(temp_file, header = TRUE, stringsAsFactors = FALSE, fileEncoding = "UTF-8")
-            file.remove(temp_file)
-            return(data)
-          }, error = function(e2) {
-            cat("Error reading as TSV:", e2$message, "\n")
-            cat("File is available at:", download_path, "\n")
-            cat("Try examining the file manually to determine format\n")
-            stop("Could not automatically determine file format")
-          })
-        })
-      } else {
-        cat("File does not appear to be text-based\n")
-        cat("File is available at:", download_path, "\n")
-        stop("Could not automatically determine file format. Please check the downloaded file.")
-      }
-    }
+    })
   } else {
-    file.remove(temp_file)
-    stop(paste("Failed to download dataset. Status code:", status_code(response)))
+    stop(paste("Failed to download file. Status code:", status_code(response)))
   }
 }