Fix eval

README.md

@@ -58,14 +58,14 @@ You can also create a custom experiment by creating a new experiment file in `cf
 At the end of training, the train script will automatically evaluate the test set using the best checkpoint (by validation loss). If epoch 0 is not finished, it will throw an error. To evaluate a specific checkpoint, run
 
 ```
-python scripts/test.py +exp={experiment_name} +ckpt_path={path/to/checkpoint} render_files=False
+python scripts/test.py +exp={experiment_name} +ckpt_path="{path/to/checkpoint}" render_files=False
 ```
 
 The checkpoints will be saved in `./logs/ckpts/{timestamp}`
 Metrics and hyperparams will be logged in `./lightning_logs/{timestamp}`
 
 By default, the dataset needed for the experiment is generated before training.
-If you have generated the dataset separately (see Generate datasets used in the paper), be sure to set `render_files=False` in the config or command-line, and set `render_root={path_to_dataset}` if it is in a custom location.
+If you have generated the dataset separately (see Generate datasets used in the paper), be sure to set `render_files=False` in the config or command-line, and set `render_root={path/to/dataset}` if it is in a custom location.
 
 Also note that the training assumes you have a GPU. To train on CPU, set `accelerator=null` in the config or command-line.
 
@@ -119,7 +119,7 @@ python scripts/generate_dataset.py +exp=chorus_aug
 See the Misc. section below for a description of the parameters.
 By default, files are rendered to `{render_root} / processed / {string_of_effects} / {train|val|test}`.
 
-If training, this process will be done automatically at the start of training. To disable this, set `render_files=False` in the config or command-line, and set `render_root={path_to_dataset}` if it is in a custom location.
+If training, this process will be done automatically at the start of training. To disable this, set `render_files=False` in the config or command-line, and set `render_root={path/to/dataset}` if it is in a custom location.
 
 # Misc.
 ## Experimental parameters

eval.sh

@@ -5,47 +5,44 @@
 # ./eval.sh distortion_aug 0-0 -ckpt logs/ckpts/2023-01-21-12-21-44
 # First 2 arguments are required, third argument is optional
 
-# Check if first argument is empty
-if [ -z "$1" ]
-then
-  echo "No experiment name supplied"
-  exit 1
+# Default value for the optional parameter
+ckpt_path=""
+
+# Function to display script usage
+function display_usage {
+    echo "Usage: $0 <experiment> <dataset> [-ckpt {ckpt_path}]"
+}
+
+# Check if the number of arguments is less than 2 (minimum required)
+if [ "$#" -lt 2 ]; then
+    display_usage
+    exit 1
 fi
 
-# Check if second argument is empty
-if [ -z "$2" ]
-then
-  echo "No dataset name supplied"
-  exit 1
-fi
 dataset_name=$2
 
-# Check if ckpt flag is set using getopts
-ckpt_flag=0
-while getopts ":ckpt:" opt; do
-  case $opt in
-    ckpt)
-      ckpt_flag=1
-      ckpt_path=$OPTARG
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&3
-      ;;
-  esac
-done
-
-# If checkpoint flag is empty, run chain inference
-if [ $ckpt_flag -eq 0 ]
-then
-  # Running chain inference
-  echo "Running chain inference"
-  python scripts/chain_inference.py +exp=$1 datamodule.train_dataset=None datamodule.val_dataset=None datamodule.test_dataset.render_root=./RemFX_eval_datasets/ render_files=False num_removed_effects=[${dataset_name:0:1},${dataset_name:2:1}]
-  exit 1
+# Parse optional parameter if provided
+if [ "$3" == "-ckpt" ]; then
+    # Check if the ckpt_path is provided
+    if [ -z "$4" ]; then
+        echo "Error: -ckpt flag requires a path argument."
+        display_usage
+        exit 1
+    fi
+    ckpt_path="$4"
+fi
+
+# If ckpt_path is empty, run chain inference
+if [ -z "$ckpt_path" ]; then
+    echo "Running chain inference"
+    python scripts/chain_inference.py +exp=$1 datamodule.train_dataset=None datamodule.val_dataset=None datamodule.test_dataset.render_root=./RemFX_eval_datasets/ render_files=False num_removed_effects=[${dataset_name:0:1},${dataset_name:2:1}]
+    exit 1
 fi
 
+
 # Otherwise run inference on the specified checkpoint
 echo "Running monolithic inference on checkpoint $3"
-python scripts/test.py +exp=$1 datamodule.train_dataset=None datamodule.val_dataset=None datamodule.test_dataset.render_root=./RemFX_eval_datasets/ datamodule.test_dataset.num_kept_effects="[0,0]" num_removed_effects=[${dataset_name:0:1},${dataset_name:2:1}] effects_to_keep=[] effects_to_remove="[compressor,reverb,chorus,delay,distortion]" render_files=False +ckpt_path=$2
+python scripts/test.py +exp=$1 datamodule.train_dataset=None datamodule.val_dataset=None datamodule.test_dataset.render_root=./RemFX_eval_datasets/ datamodule.test_dataset.num_kept_effects="[0,0]" num_removed_effects=[${dataset_name:0:1},${dataset_name:2:1}] effects_to_keep=[] effects_to_remove="[distortion, compressor,reverb,chorus,delay]" render_files=False +ckpt_path=$ckpt_path