Piyusharanjan Pradhan commited on
Commit
31ab0c0
·
1 Parent(s): dedec0d
Files changed (1) hide show
  1. startup.sh +60 -17
startup.sh CHANGED
@@ -10,9 +10,9 @@ if [ -z "$HF_TOKEN" ] || [ -z "$HF_DATASET" ]; then
10
  exit 0
11
  fi
12
 
13
- # Login to Hugging Face
14
  echo "Logging in to Hugging Face..."
15
- huggingface-cli login --token "$HF_TOKEN" --add-to-git-credential
16
 
17
  # Directories to persist
18
  CONFIG_DIR="/home/coder/.config/code-server"
@@ -27,18 +27,35 @@ mkdir -p "$BACKUP_DIR"
27
  download_from_hf() {
28
  echo "Downloading persisted data from Hugging Face dataset: $HF_DATASET"
29
 
30
- # Try to download the backup archive
31
- if huggingface-cli download "$HF_DATASET" backup.tar.gz --repo-type dataset --local-dir "$BACKUP_DIR" 2>/dev/null; then
32
- echo "Found existing backup, restoring..."
33
-
34
- # Extract the backup
35
- cd /home/coder
36
- tar -xzf "$BACKUP_DIR/backup.tar.gz" 2>/dev/null || echo "No previous backup found or extraction failed"
37
-
38
- echo "Restoration complete!"
39
- else
40
- echo "No existing backup found. Starting fresh."
41
- fi
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  }
43
 
44
  # Function to upload to HF dataset
@@ -57,9 +74,35 @@ upload_to_hf() {
57
  .local/share/code-server \
58
  workspace 2>/dev/null || true
59
 
60
- # Upload to HF dataset
61
- huggingface-cli upload "$HF_DATASET" "$BACKUP_DIR/backup.tar.gz" backup.tar.gz --repo-type dataset --create || \
62
- echo "Upload failed, continuing anyway..."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  }
64
 
65
  # Setup trap to save on exit
 
10
  exit 0
11
  fi
12
 
13
+ # Login to Hugging Face using Python
14
  echo "Logging in to Hugging Face..."
15
+ python3 -c "from huggingface_hub import login; login(token='$HF_TOKEN', add_to_git_credential=True)"
16
 
17
  # Directories to persist
18
  CONFIG_DIR="/home/coder/.config/code-server"
 
27
  download_from_hf() {
28
  echo "Downloading persisted data from Hugging Face dataset: $HF_DATASET"
29
 
30
+ # Try to download the backup archive using Python
31
+ python3 << 'PYTHON_SCRIPT'
32
+ import os
33
+ from huggingface_hub import hf_hub_download
34
+ import tarfile
35
+
36
+ try:
37
+ backup_dir = "/tmp/hf_backup"
38
+ os.makedirs(backup_dir, exist_ok=True)
39
+
40
+ # Download backup file
41
+ file_path = hf_hub_download(
42
+ repo_id=os.environ["HF_DATASET"],
43
+ filename="backup.tar.gz",
44
+ repo_type="dataset",
45
+ local_dir=backup_dir
46
+ )
47
+
48
+ print(f"Found existing backup at {file_path}, restoring...")
49
+
50
+ # Extract the backup
51
+ with tarfile.open(file_path, "r:gz") as tar:
52
+ tar.extractall("/home/coder")
53
+
54
+ print("Restoration complete!")
55
+ except Exception as e:
56
+ print(f"No existing backup found or restoration failed: {e}")
57
+ print("Starting fresh.")
58
+ PYTHON_SCRIPT
59
  }
60
 
61
  # Function to upload to HF dataset
 
74
  .local/share/code-server \
75
  workspace 2>/dev/null || true
76
 
77
+ # Upload to HF dataset using Python
78
+ python3 << 'PYTHON_SCRIPT'
79
+ import os
80
+ from huggingface_hub import HfApi
81
+
82
+ try:
83
+ api = HfApi()
84
+ backup_file = "/tmp/hf_backup/backup.tar.gz"
85
+
86
+ # Create repo if it doesn't exist and upload
87
+ api.create_repo(
88
+ repo_id=os.environ["HF_DATASET"],
89
+ repo_type="dataset",
90
+ private=True,
91
+ exist_ok=True
92
+ )
93
+
94
+ api.upload_file(
95
+ path_or_fileobj=backup_file,
96
+ path_in_repo="backup.tar.gz",
97
+ repo_id=os.environ["HF_DATASET"],
98
+ repo_type="dataset"
99
+ )
100
+
101
+ print("Upload successful!")
102
+ except Exception as e:
103
+ print(f"Upload failed: {e}")
104
+ print("Continuing anyway...")
105
+ PYTHON_SCRIPT
106
  }
107
 
108
  # Setup trap to save on exit