Commit
·
c654a00
1
Parent(s):
4719d13
Enhance DVC pull with retry logic and cache cleanup for improved reliability
Browse files- .github/workflows/ci.yml +27 -1
.github/workflows/ci.yml
CHANGED
|
@@ -86,7 +86,33 @@ jobs:
|
|
| 86 |
|
| 87 |
- name: Pull Models with DVC
|
| 88 |
run: |
|
| 89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
- name: Build Docker Image
|
| 92 |
run: |
|
|
|
|
| 86 |
|
| 87 |
- name: Pull Models with DVC
|
| 88 |
run: |
|
| 89 |
+
# Clean any potentially corrupted cache
|
| 90 |
+
rm -rf .dvc/cache/files/md5/e1 || true
|
| 91 |
+
|
| 92 |
+
# Retry logic for DVC pull to handle intermittent server errors
|
| 93 |
+
max_attempts=3
|
| 94 |
+
attempt=0
|
| 95 |
+
until [ $attempt -ge $max_attempts ]
|
| 96 |
+
do
|
| 97 |
+
dvc pull models/random_forest_embedding_gridsearch.pkl models/label_names.pkl && break
|
| 98 |
+
attempt=$((attempt+1))
|
| 99 |
+
echo "DVC pull attempt $attempt failed. Retrying in 10 seconds..."
|
| 100 |
+
# Clean cache on retry
|
| 101 |
+
rm -rf .dvc/cache || true
|
| 102 |
+
sleep 10
|
| 103 |
+
done
|
| 104 |
+
if [ $attempt -ge $max_attempts ]; then
|
| 105 |
+
echo "DVC pull failed after $max_attempts attempts"
|
| 106 |
+
exit 1
|
| 107 |
+
fi
|
| 108 |
+
|
| 109 |
+
- name: Verify Models Downloaded
|
| 110 |
+
run: |
|
| 111 |
+
if [ ! -f "models/random_forest_embedding_gridsearch.pkl" ] || [ ! -f "models/label_names.pkl" ]; then
|
| 112 |
+
echo "ERROR: Required model files not found after DVC pull"
|
| 113 |
+
exit 1
|
| 114 |
+
fi
|
| 115 |
+
echo "All required model files present"
|
| 116 |
|
| 117 |
- name: Build Docker Image
|
| 118 |
run: |
|