Lenson commited on
Commit
15ac309
·
1 Parent(s): 0c1bc80

fix: use curl instead of huggingface-cli for config download

Browse files
Files changed (3) hide show
  1. .hf/setup.sh +18 -6
  2. Dockerfile +1 -3
  3. Dockerfile.hfspaces +1 -3
.hf/setup.sh CHANGED
@@ -22,13 +22,25 @@ fi
22
  if [ -n "$HF_TOKEN" ]; then
23
  echo "Downloading config from HF Dataset: $HF_DATASET_NAME"
24
 
25
- # Download data directory
26
- huggingface-cli download "$HF_DATASET_NAME" data/ --repo-type dataset --local-dir /app/data --local-dir-use-symlinks False 2>/dev/null || \
27
- echo "No data directory in dataset, starting fresh"
 
 
 
 
 
 
28
 
29
- # Download config directory
30
- huggingface-cli download "$HF_DATASET_NAME" config/ --repo-type dataset --local-dir /app/config --local-dir-use-symlinks False 2>/dev/null || \
31
- echo "No config directory in dataset, using defaults"
 
 
 
 
 
 
32
 
33
  echo "Config downloaded successfully"
34
  else
 
22
  if [ -n "$HF_TOKEN" ]; then
23
  echo "Downloading config from HF Dataset: $HF_DATASET_NAME"
24
 
25
+ # Download data files
26
+ curl -s -H "Authorization: Bearer $HF_TOKEN" \
27
+ "https://huggingface.co/api/datasets/$HF_DATASET_NAME/tree/main/data" | \
28
+ grep -o '"path":"[^"]*"' | sed 's/"path":"//g' | sed 's/"//g' | while read file; do
29
+ mkdir -p "/app/data/$(dirname $file)"
30
+ curl -s -H "Authorization: Bearer $HF_TOKEN" \
31
+ "https://huggingface.co/datasets/$HF_DATASET_NAME/resolve/main/$file" \
32
+ -o "/app/data/$file" || true
33
+ done
34
 
35
+ # Download config files
36
+ curl -s -H "Authorization: Bearer $HF_TOKEN" \
37
+ "https://huggingface.co/api/datasets/$HF_DATASET_NAME/tree/main/config" | \
38
+ grep -o '"path":"[^"]*"' | sed 's/"path":"//g' | sed 's/"//g' | while read file; do
39
+ mkdir -p "/app/config/$(dirname $file)"
40
+ curl -s -H "Authorization: Bearer $HF_TOKEN" \
41
+ "https://huggingface.co/datasets/$HF_DATASET_NAME/resolve/main/$file" \
42
+ -o "/app/config/$file" || true
43
+ done
44
 
45
  echo "Config downloaded successfully"
46
  else
Dockerfile CHANGED
@@ -3,10 +3,8 @@ FROM node:20-slim
3
  # curl: needed by setup-curl.ts and full-update.ts
4
  # unzip: needed for full-update.ts to extract Codex.app
5
  # gosu: needed by entrypoint to drop from root to node user
6
- # python3 + pip: needed for huggingface-cli
7
  RUN apt-get update && \
8
- apt-get install -y --no-install-recommends curl unzip ca-certificates gosu python3 python3-pip && \
9
- pip3 install --no-cache-dir huggingface-hub && \
10
  rm -rf /var/lib/apt/lists/*
11
 
12
  WORKDIR /app
 
3
  # curl: needed by setup-curl.ts and full-update.ts
4
  # unzip: needed for full-update.ts to extract Codex.app
5
  # gosu: needed by entrypoint to drop from root to node user
 
6
  RUN apt-get update && \
7
+ apt-get install -y --no-install-recommends curl unzip ca-certificates gosu && \
 
8
  rm -rf /var/lib/apt/lists/*
9
 
10
  WORKDIR /app
Dockerfile.hfspaces CHANGED
@@ -3,10 +3,8 @@ FROM node:20-slim
3
  # curl: needed by setup-curl.ts and full-update.ts
4
  # unzip: needed for full-update.ts to extract Codex.app
5
  # gosu: needed by entrypoint to drop from root to node user
6
- # python3 + pip: needed for huggingface-cli
7
  RUN apt-get update && \
8
- apt-get install -y --no-install-recommends curl unzip ca-certificates gosu python3 python3-pip && \
9
- pip3 install --no-cache-dir huggingface-hub && \
10
  rm -rf /var/lib/apt/lists/*
11
 
12
  WORKDIR /app
 
3
  # curl: needed by setup-curl.ts and full-update.ts
4
  # unzip: needed for full-update.ts to extract Codex.app
5
  # gosu: needed by entrypoint to drop from root to node user
 
6
  RUN apt-get update && \
7
+ apt-get install -y --no-install-recommends curl unzip ca-certificates gosu && \
 
8
  rm -rf /var/lib/apt/lists/*
9
 
10
  WORKDIR /app