Spaces:
Runtime error
Runtime error
Yixin Liu
commited on
Commit
·
8963c43
1
Parent(s):
0fa848f
upload
Browse files
config.py
CHANGED
|
@@ -93,7 +93,7 @@ function update_device_idx {
|
|
| 93 |
useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
| 94 |
utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
| 95 |
|
| 96 |
-
if [ $useage -ge $((
|
| 97 |
echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
|
| 98 |
sleep 3
|
| 99 |
continue
|
|
|
|
| 93 |
useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
| 94 |
utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
| 95 |
|
| 96 |
+
if [ $useage -ge $((total_gpu_memory-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
|
| 97 |
echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
|
| 98 |
sleep 3
|
| 99 |
continue
|