File size: 2,178 Bytes
f53b3ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
CACHE_DIR="data/cache"
OUTPUT_DIR="data/archives"
BATCH_SIZE=60000 # Smaller for frequent updates
mkdir -p "$OUTPUT_DIR"
echo "========================================================"
echo "Archiving '$CACHE_DIR' into multiple zip files..."
echo "Batch Size: $BATCH_SIZE files per archive"
echo "========================================================"
echo "Scanning for .pt files..."
find "$CACHE_DIR" -maxdepth 1 -name "sample_*.pt" > all_files_temp.txt
TOTAL_FILES=$(wc -l < all_files_temp.txt)
echo "Found $TOTAL_FILES .pt files."
if [ "$TOTAL_FILES" -eq 0 ]; then
echo "No files found to archive."
rm all_files_temp.txt
exit 0
fi
# Split list into temporary chunk files
split -l "$BATCH_SIZE" -d -a 3 all_files_temp.txt file_list_part_
echo "Starting sequential archiving..."
for LIST_FILE in file_list_part_*; do
PART_NUM=${LIST_FILE##*_}
ZIP_NAME="$OUTPUT_DIR/cache_batch_$PART_NUM.zip"
echo "[$(date +%T)] Starting batch $PART_NUM ($BATCH_SIZE files) -> $ZIP_NAME"
# Process sequentially:
# -1: Fast compression
# -m: Move files (delete after successful zip) -- keeping requested behavior (?)
# No, user "give up dont upload metadata". Script previously had -m.
# User might want to KEEP source files if upload fails?
# Usually archiving cache implies "pack it up".
# I'll stick to -m (move) to save space as we go, unless previously requested otherwise?
# User didn't specify "keep". Defaulting to -m clears disk space.
# BUT wait, user said "no way to see if is zipping".
# If I use -m, files disappear.
# Let's use -m to clean up.
cat "$LIST_FILE" | zip -1 -mq -j "$ZIP_NAME" -@
# Verify zip created
if [ -f "$ZIP_NAME" ]; then
SIZE=$(du -h "$ZIP_NAME" | cut -f1)
echo "[$(date +%T)] Finished batch $PART_NUM (Size: $SIZE)"
else
echo "ERROR: Failed to create $ZIP_NAME"
exit 1
fi
rm "$LIST_FILE"
done
# Cleanup
rm all_files_temp.txt
echo "========================================================"
echo "Done! Archives are in $OUTPUT_DIR"
echo "========================================================"
|