upload

config.py

@@ -0,0 +1,107 @@
+
+update_device_command = "update_device_idx;\n"
+
+update_device_func = """
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}
+"""

gpu_utility.sh

@@ -1,16 +1,16 @@
 
-username_mine=root
+username=yila22
 max_gpu_utilization=90
-total_aviable=24564
+total_gpu_memory=24564
 max_gpu_memory_gap=5000
-available_devices=( 0 1 2 3 4 5 6 7 8 9 )
+available_devices=( 0 1 2 3 4 )
 current_device_idx=-1
-sleeptime=30
+sleeptime=10
 cpu_mean_max=77
 memory_rate_max=80
-constrain_total=true
-constrain_mine=false
-constrain_rate=2
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
 
 
 

main.py

@@ -4,144 +4,161 @@ import streamlit as st
 import pandas as pd
 import numpy as np
 import time
-# import matplotlib.pyplot as plt
-# import seaborn as sns
-# import plotly.figure_factory as ff
-# import altair as alt
-# from PIL import Image
-# import base64
-# import tarfile
-# import os
-# import requests
-
+from config import  update_device_func
+from parse_code import parse_base_code
 
 
 # title
 st.title("Exp Command Generator")
 
 # experiment mode
-exp_mode = st.sidebar.selectbox("Select Experiment Mode", ["OneExpOnecard", "MultipleExpOnecard"],key="MultipleExpOnecard")
+# exp_mode = st.sidebar.selectbox("Select Experiment Mode", ["OneExpOnecard", "MultipleExpOnecard"],key="MultipleExpOnecard")
 
 ## 检查框
-debug = st.sidebar.checkbox("Debug:选择则会串行地执行命令", value=True)
+debug = st.sidebar.checkbox("Debug: 选择则会串行地执行命令", value=True)
 # st.sidebar.write(f"checkbox的值是{res}")
 
-setup = st.sidebar.text_area("Some setup of env at beginning.", """cd $(dirname $(dirname $0))
-source activate xai
-export PYTHONPATH=${PYTHONPATH}:/Users/apple/Desktop/workspace/research_project/attention:/mnt/yixin/:/home/yila22/prj""")
-
-exp_hyper = st.sidebar.text_area("Hyperparameters", """exp_name="debug-adv-training-emotion"
-dataset=emotion
-n_epoch=3
-K=3
-encoder=bert
-lambda_1=1
-lambda_2=1
-x_pgd_radius=0.01
-pgd_radius=0.001
-seed=2
-bsize=8
-lr=5e-5""")
+# setup = st.sidebar.text_area("Some setup of env at beginning.", """cd $(dirname $(dirname $0))
+# source activate xai
+# export PYTHONPATH=${PYTHONPATH}:/Users/apple/Desktop/workspace/research_project/attention:/mnt/yixin/:/home/yila22/prj""")
+
+# exp_hyper = st.sidebar.text_area("Hyperparameters", """exp_name="debug-adv-training-emotion"
+# dataset=emotion
+# n_epoch=3
+# K=3
+# encoder=bert
+# lambda_1=1
+# lambda_2=1
+# x_pgd_radius=0.01
+# pgd_radius=0.001
+# seed=2
+# bsize=8
+# lr=5e-5""")
 
 ## gpu 相关参数
-gpu_list = st.sidebar.multiselect("multi select", range(10), [5, 6, 7, 8, 9])
+gpu_list = st.sidebar.multiselect("multi select", range(10), [0,1,2,3,4,])
 # print(gpu_list)
-if exp_mode == "OneExpOnecard":
-    allow_gpu_memory_threshold_default = 20000
-    gpu_threshold_default = 1
-elif exp_mode == "MultipleExpOnecard":
-    allow_gpu_memory_threshold_default = 3000
-    gpu_threshold_default = 70
-allow_gpu_memory_threshold = st.sidebar.number_input("最小单卡剩余容量", value=allow_gpu_memory_threshold_default, min_value=0, max_value=30000, step=1000)
-gpu_threshold = st.sidebar.number_input("最大单卡利用率", value=gpu_threshold_default, min_value=0, max_value=100, step=10)
-sleep_time_after_loading_task= st.sidebar.number_input("加载任务后等待秒数", value=20, min_value=0,step=5)
-all_full_sleep_time = st.sidebar.number_input("全满之后等待秒数", value=20, min_value=0,step=5)
-
-gpu_list_str = ' '.join([str(i) for i in gpu_list])
-gpu_hyper = f"gpu=({gpu_list_str})\n"
-gpu_hyper+=f"allow_gpu_memory_threshold={allow_gpu_memory_threshold}\n"
-gpu_hyper+=f"gpu_threshold={gpu_threshold}\n"
-gpu_hyper+=f"sleep_time_after_loading_task={sleep_time_after_loading_task}s\n"
-gpu_hyper+=f"all_full_sleep_time={all_full_sleep_time}s\n"
-gpu_hyper+="gpunum=${#gpu[@]}\n"
-gpu_hyper+="i=0\n"
-
-main_loop = st.text_area("Main loop", """for lambda_1 in 1 3;do
-  for lambda_2 in 1 10;do
-    for n_epoch in 3;do
-      for x_pgd_radius in 0.005 0.01;do
-        for pgd_radius in 0.0005 0.001 0.002;do
-          python train.py --dataset $dataset --data_dir . --output_dir ./outputs/ --attention tanh \
-              --encoder $encoder \
-                --exp_name $exp_name --lambda_1 $lambda_1 --lambda_2 $lambda_2 --pgd_radius $pgd_radius --x_pgd_radius $x_pgd_radius \
-                --K $K  --seed $seed --train_mode adv_train --bsize $bsize --n_epoch $n_epoch --lr $lr \
-                --eval_baseline
-done;done;done;done;done;""")
-if 'python' in main_loop:
-    hyper_loop = main_loop.split("python")[0]
-    python_cmd = main_loop[main_loop.index('python'):].split('done;')[0]
-elif 'bash' in main_loop:
-    hyper_loop = main_loop.split("bash")[0]
-    python_cmd = main_loop[main_loop.index('bash'):].split('done;')[0]
-print(hyper_loop)
-print(python_cmd)
-end_loop = "done;"*hyper_loop.count("for")
-print(end_loop)
+allow_gpu_memory_threshold_default=5000
+gpu_threshold_default=90
+total_gpu_memory = st.sidebar.number_input("单卡总容量", value=24564, min_value=0, max_value=30000, step=1000)
+max_gpu_memory_gap = st.sidebar.number_input("最小单卡剩余容量", value=allow_gpu_memory_threshold_default, min_value=0, max_value=total_gpu_memory, step=500)
+max_gpu_utilization = st.sidebar.number_input("最大单卡利用率", value=gpu_threshold_default, min_value=0, max_value=100, step=10)
+sleep_time_after_loading_task= st.sidebar.number_input("加载任务后等待秒数", value=10, min_value=0,step=5)
+# all_full_sleep_time = st.sidebar.number_input("全满之后等待秒数", value=20, min_value=0,step=5)
+username = st.sidebar.text_input("用户名", value="yila22")
+cpu_max_utility = st.sidebar.number_input("cpu最大利用率", value=77, min_value=0, max_value=100, step=1)
+memory_max_utility = st.sidebar.number_input("内存最大利用率", value=80, min_value=0, max_value=100, step=1)
+constrain_total = st.sidebar.checkbox("限制总资源", value=True)
+constrain_mine = st.sidebar.checkbox("限制我的资源", value=False)
+constrain_rate = st.sidebar.number_input("限制率", value=2, min_value=1, max_value=10, step=1)
+
+# username_mine=root
+# max_gpu_utilization=90
+# total_gpu_memory=24564
+# max_gpu_memory_gap=5000
+# available_devices=( 0 1 2 3 4 5 6 7 8 9 )
+# current_device_idx=-1
+# sleeptime=30
+# cpu_mean_max=77
+# memory_rate_max=80
+# constrain_total=true
+# constrain_mine=false
+# constrain_rate=2
+gpu_list = " ".join([str(i) for i in gpu_list])
+setup_for_gpu_utility = f"""
+username={username}
+max_gpu_utilization={max_gpu_utilization}
+total_gpu_memory={total_gpu_memory}
+max_gpu_memory_gap={max_gpu_memory_gap}
+available_devices=( {gpu_list} )
+current_device_idx=-1
+sleeptime={sleep_time_after_loading_task}
+cpu_mean_max={cpu_max_utility}
+memory_rate_max={memory_max_utility}
+constrain_total={constrain_total}
+constrain_mine={constrain_mine}
+constrain_rate={constrain_rate}
+"""
+
+
+base_code = st.text_area("Base Code", """##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+##### main
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0 \
+   --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name \
+    --log_wb
+#####
+
+#####
+done;done;
+#####""", height=400)
+
+
 
 g = st.button("Generate")
 if g:
-    s = ""
-    s += setup + "\n\n"
-    s += exp_hyper + "\n\n"
-    s += gpu_hyper + "\n\n"
-    s += hyper_loop + "\n\n"
-    s += """
-while true; do
-    gpu_id=${gpu[$i]}
-#    nvidia-smi --query-gpu=utilization.gpu  --format=csv -i 2 | grep -Eo "[0-9]+"
-    gpu_u=$(nvidia-smi --query-gpu=utilization.gpu  --format=csv -i $gpu_id | grep -Eo "[0-9]+")
-    free_mem=$(nvidia-smi --query-gpu=memory.free --format=csv -i $gpu_id | grep -Eo "[0-9]+")
-    if [[ $free_mem -lt $allow_gpu_memory_threshold || $gpu_u -ge ${gpu_threshold} ]]; then
-        i=`expr $i + 1`
-        i=`expr $i % $gpunum`
-        echo "gpu id ${gpu[$i]} is full loaded, skip"
-        if [ "$i" == "0" ]; then
-            sleep ${all_full_sleep_time}
-            echo "all the gpus are full, sleep 1m"
-        fi
-    else
-        break
-    fi
-done
-
-gpu_id=${gpu[$i]}
-# search from the next gpu
-i=`expr $i + 1`
-i=`expr $i % $gpunum`
-
-free_mem=$(nvidia-smi --query-gpu=memory.free --format=csv -i $gpu_id | grep -Eo "[0-9]+")
-gpu_u=$(nvidia-smi --query-gpu=utilization.gpu  --format=csv -i $gpu_id | grep -Eo "[0-9]+")
-export CUDA_VISIBLE_DEVICES=$gpu_id
-echo "use gpu id is ${gpu[$i]}, free memory is $free_mem, it utilization is ${gpu_u}%"
-"""
-    s += f"""com="{python_cmd}"\n"""
-    s += "echo $com\n"
-    s += "echo ==========================================================================================\n"
-    if debug:
-        s += "$com\n"
-        s += "# mkdir -p ./logs/\n"
-        s += "# nohup $com > ./logs/$exp_name-$RANDOM.log 2>&1 &\n"
-    else:
-        s += "# $com\n"
-        s += "mkdir -p ./logs/\n"
-        s += "nohup $com > ./logs/$exp_name-$RANDOM.log 2>&1 &\n"
-    s += """echo "sleep for $sleep_time_after_loading_task to wait the task loaded"
-    sleep  $sleep_time_after_loading_task\n"""
-    s += end_loop
     st.success("Finished")
-    st.code(s, language="shell")
-  
-
-  
+    contents = base_code
+    gpu_utility = ""
+    gpu_utility = setup_for_gpu_utility + "\n\n" + update_device_func 
+    
+    
+    new_code = parse_base_code(contents, debug=debug)
+    
+    # create file for download 
+    timestr = time.strftime("%Y%m%d-%Hh%Mm%Ss")
+    import os 
+    os.makedirs(f"./res/{timestr}", exist_ok=True)
+    filename_script = f"./res/{timestr}/script.sh"
+    with open(filename_script, "w") as f:
+        f.write(new_code)
+    filename_config = f"./res/{timestr}/gpu_utility.sh"
+    with open(filename_config, "w") as f:
+        f.write(gpu_utility)
+    
+    # zip them into one file
+    # import shutil
+    # shutil.make_archive(f"./res/{timestr}", 'zip', f"./res/{timestr}")
+    # st.download_button(
+    #     label="Download zip",
+    #     data=f"./res/{timestr}.zip",
+    #     file_name=f"{timestr}.zip",
+    #     mime="application/zip",
+    # )
+    
+    
+    st.download_button(
+        label="Download script",
+        data=new_code,
+        file_name=filename_script,
+        mime="text/plain",
+    )
+    # after clicking i don't want the website to refresh
+    st.download_button(
+        label="Download gpu_utility.sh",
+        data=gpu_utility,
+        file_name=filename_config,
+        mime="text/plain",
+    )
+    
+    # st.markdown(f"### [Download script](./{filename_script})")
+    # st.markdown(f"### [Download gpu_utility.sh](P{filename_config})")
+    st.markdown("## script.sh")
+    st.code(new_code, language="shell")
+    
+    
+    st.markdown("## gpu_utility.sh")
+    st.code(gpu_utility, language="shell")
+    
+    
 
 

parse_code.py

@@ -0,0 +1,61 @@
+from config import update_device_command
+
+def parse_base_code(contents, debug = False):
+    import re
+    indexes = [m.start() for m in re.finditer('#####', contents)]
+    
+    assert len(indexes) % 2 == 0
+    
+    # split to span
+    spans = []
+    # spans.append(contents[:indexes[0]])
+    for i in range(len(indexes)):
+        if i != len(indexes) - 1:
+            spans.append(contents[indexes[i]:indexes[i+1]])
+    # spans.append(contents[indexes[-1]:])
+        
+    spans_with_type = [
+        
+    ]
+    for span in spans:
+        if "setup" in span:
+            spans_with_type.append((span, "setup"))
+        elif "loop" in span:
+            spans_with_type.append((span, "loop"))
+        elif "main" in span:
+            spans_with_type.append((span, "command"))
+        else:
+            spans_with_type.append((span, "other"))
+    
+    spans_with_type_added_device_control = []
+
+    for span, type_ in spans_with_type:
+        if type_ == "setup":
+            spans_with_type_added_device_control.append((
+                """cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh\n\n"""
+                , "device_control"))
+            spans_with_type_added_device_control.append((span, type_))
+            # spans_with_type_added_device_control.append((gpu_env, "device_control"))
+            # spans_with_type_added_device_control.append((update_device_func, "device_control"))
+        elif type_ == "loop":
+            spans_with_type_added_device_control.append((span, type_))
+        elif type_ == "command":
+            spans_with_type_added_device_control.append((update_device_command, "device_control"))
+            span_remove_the_first_part = span[span.index("\n"):]
+            if not debug:
+                spans_with_type_added_device_control.append((f"\n\ncommand=\"\"\"{span_remove_the_first_part}\"\"\"\n", type_))
+                run_command = "eval $command"
+                run_command += " &"
+                run_command += "\n\n\n"
+                spans_with_type_added_device_control.append((run_command, type_))
+            else: 
+                spans_with_type_added_device_control.append(
+                    (f"{span_remove_the_first_part}\n", type_)
+                )
+            sleep_command = "sleep $sleeptime\n\n"
+            spans_with_type_added_device_control.append((sleep_command, type_))
+        else:
+            spans_with_type_added_device_control.append((span, type_))
+    spans_without_type = [span for span, type_ in spans_with_type_added_device_control]
+    spans_without_type_str = "".join(spans_without_type)
+    return spans_without_type_str

res/20230615-17h44m58s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h44m58s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h45m38s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h45m38s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h46m42s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h46m42s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h48m58s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h48m58s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h49m08s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h49m08s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h49m45s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h49m45s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h50m13s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h50m13s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h50m22s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h50m22s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;

res/20230615-17h50m57s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h50m57s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h51m33s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h51m33s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h51m43s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h51m43s/script.sh

@@ -0,0 +1,29 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+command="""
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+"""
+eval $command &
+
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h53m28s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h53m28s/script.sh

@@ -0,0 +1,25 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h53m44s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h53m44s/script.sh

@@ -0,0 +1,24 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+
+update_device_idx;
+
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h55m17s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=True
+constrain_rate=True
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h55m17s/script.sh

@@ -0,0 +1,23 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+update_device_idx;
+
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;

res/20230615-17h56m22s/gpu_utility.sh

@@ -0,0 +1,118 @@
+
+username=yila22
+max_gpu_utilization=90
+total_gpu_memory=24564
+max_gpu_memory_gap=5000
+available_devices=( 0 1 2 3 4 )
+current_device_idx=-1
+sleeptime=10
+cpu_mean_max=77
+memory_rate_max=80
+constrain_total=True
+constrain_mine=False
+constrain_rate=2
+
+
+
+function update_device_idx {
+    if [ $constrain_total = true ]; then
+    # check total cpu usage
+    while true; do
+        cpu_mean_1=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_2=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        sleep 1
+        cpu_mean_3=$(mpstat -P ALL 1 1 | awk '/Average:/ && $2 ~ /[0-9]/ { cpu_usage=100-$NF; total+=cpu_usage; count++ } END { print total/count }')
+        cpu_mean=$(echo "scale=2; ($cpu_mean_1+$cpu_mean_2+$cpu_mean_3)/3" | bc)
+
+        # if currently cpu usage is less than the threshold, then break
+        if [ $(echo "$cpu_mean < $cpu_mean_max" | bc) -eq 1 ]; then
+            echo "total cpu mean: $cpu_mean is less than $cpu_mean_max, continue to check total memory usage"
+            break
+        else
+            echo "total cpu mean: $cpu_mean is greater than $cpu_mean_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+
+    # check total memory usage
+    while true; do
+        # get memory usage of whole system
+        mem_used_1=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_2=$(free -m | awk '/Mem:/ {print $3}')
+        sleep 1
+        mem_used_3=$(free -m | awk '/Mem:/ {print $3}')
+        mem_used=$(echo "scale=2; ($mem_used_1+$mem_used_2+$mem_used_3)/3" | bc)
+        
+        # echo $mem_used
+        # get rate of memory usage
+        mem_rate=$(echo "scale=2; $mem_used/$(free -m | awk '/Mem:/ {print $2}')*100" | bc)
+        # echo $mem_rate
+        if [ $(echo "$mem_rate < $memory_rate_max" | bc) -eq 1 ]; then
+            echo "total memory rate: $mem_rate is less than $memory_rate_max, continue to check my own cpu and memory usage"
+            break
+        else
+            echo "total memory rate: $mem_rate is greater than $memory_rate_max, sleep 10 seconds"
+            sleep 10
+        fi
+    done;
+    fi;
+
+    # if constrain_mine
+    if [ $constrain_mine = true ]; then
+
+        # check my own cpu and memory usage, it should be less than 1/$constrain_rate of the given cpu_mean_max / memory_rate_max
+        while true; do
+            username=$username_mine
+            cpu_usage_user_sum=$(ps -u $username -o %cpu | awk '{sum+=$1} END {print sum}')
+            # echo $cpu_usage_user_sum
+            total_aviable_cpu=$(nproc)
+            total_aviable_cpu=$(echo "$total_aviable_cpu*100" | bc)
+            # echo $total_aviable_cpu
+            cpu_usage_user_ratio=$(echo "scale=2; $cpu_usage_user_sum/$total_aviable_cpu*100" | bc)
+            # echo $cpu_usage_user_ratio
+
+            memory_usage_user_sum=$(ps -u $username -o rss | awk '{sum+=$1} END {print sum/1024}')
+            # echo $memory_usage_user_sum
+            memory_usage_total=$(free -m | awk '/Mem:/ {print $2}')
+            # echo $memory_usage_total
+            memory_usage_user_ratio=$(echo "scale=2; $memory_usage_user_sum/$memory_usage_total*100" | bc)
+            # echo $memory_usage_user_ratio
+
+            # so my ratio should be less than 1/$constrain_rate of the given threshold
+            cpu_mean_max_mine=$(echo "$cpu_mean_max/$constrain_rate" | bc)
+            memory_rate_max_mine=$(echo "$memory_rate_max/$constrain_rate" | bc)
+            if [ $(echo "$cpu_usage_user_ratio < $cpu_mean_max_mine" | bc) -eq 1 ] && [ $(echo "$memory_usage_user_ratio < $memory_rate_max_mine" | bc) -eq 1 ]; then
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is less than half of the given threshold for cpu: $cpu_mean_max_mine and memory: $memory_rate_max_mine, ready to take off"
+                break
+            else
+                echo "my cpu usage: $cpu_usage_user_ratio, memory usage: $memory_usage_user_ratio is greater than half of the given threshold, sleep 10 seconds"
+                sleep 10
+            fi
+        done;
+    fi;
+
+    # so all the conditions are satisfied, we can update the device idx and run the next experiment
+    while true; do
+        current_device_idx=$((current_device_idx+1))
+        if [ $current_device_idx -ge ${#available_devices[@]} ]; then
+            # reset 
+            current_device_idx=0
+        fi
+        # check whether this device is fully booked using nvidia-smi
+        # get the gpu current memory usage 
+        useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
+        
+        if [ $useage -ge $((total_aviable-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
+            echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
+            sleep 3
+            continue
+        else
+            break
+        fi
+    done
+    echo "current device: ${available_devices[$current_device_idx]}"
+    device=${available_devices[$current_device_idx]}
+}

res/20230615-17h56m22s/script.sh

@@ -0,0 +1,23 @@
+cd $(cd "$(dirname "$0")";pwd); source gpu_utility.sh
+
+##### setup
+export CUDA_VISIBLE_DEVICES=2
+source activate /data/yixin/anaconda/mib
+exp_name="single_user"
+#####
+
+##### loop
+for poison_method in char_basic word_basic sent_basic; do
+for dataset_idx in 0 1 2; do
+#####
+
+update_device_idx;
+
+  python single_user.py --dataset_idx $dataset_idx --trigger_size 1 --target 0    --loc 0 --batch_size 16 --num_epochs 2 --poison_method $poison_method --lr 5e-5 --pattern 0 --exp_name $exp_name     --log_wb
+
+sleep $sleeptime
+
+#####
+
+#####
+done;done;