File size: 2,334 Bytes
bff2f94 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | #!/bin/bash
#
# BioRLHF Model Evaluation Script
# ================================
#
# Evaluates the ecosystem-improved model on:
# - Calibration (uncertainty expression)
# - Adversarial resistance
# - Protocol completeness
# - Fact recall
#
# Usage on HPC:
# srun -p scu-gpu --gres=gpu:a100:1 --mem=48G -c 8 --time=1:00:00 --pty bash
# conda activate biorlhf
# ./scripts/run_evaluation.sh
#
echo "============================================================"
echo "BioRLHF Ecosystem Model Evaluation"
echo "============================================================"
echo "Start time: $(date)"
echo "Host: $(hostname)"
echo ""
# Set working directory
cd "$(dirname "$0")/.." || exit 1
echo "Working directory: $(pwd)"
# Check GPU
echo ""
echo "GPU Information:"
nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv 2>/dev/null || echo "No GPU detected"
echo ""
# Configuration
MODEL_PATH="./ecosystem_improved_model"
TEST_DATA="data/ecosystem_failures_training.json"
OUTPUT="ecosystem_eval_results_$(date +%Y%m%d_%H%M%S).json"
echo "============================================================"
echo "Configuration:"
echo "============================================================"
echo "Model: $MODEL_PATH"
echo "Test data: $TEST_DATA"
echo "Output: $OUTPUT"
echo ""
# Check files exist
if [ ! -d "$MODEL_PATH" ]; then
echo "ERROR: Model not found at $MODEL_PATH"
exit 1
fi
if [ ! -f "$TEST_DATA" ]; then
echo "ERROR: Test data not found at $TEST_DATA"
exit 1
fi
# Run evaluation
echo "============================================================"
echo "Starting Evaluation..."
echo "============================================================"
python3 scripts/evaluate_ecosystem_model.py \
--model "$MODEL_PATH" \
--test-data "$TEST_DATA" \
--output "$OUTPUT"
# Check exit status
if [ $? -eq 0 ]; then
echo ""
echo "============================================================"
echo "Evaluation Complete!"
echo "============================================================"
echo "Results saved to: $OUTPUT"
echo "End time: $(date)"
else
echo ""
echo "============================================================"
echo "Evaluation Failed!"
echo "============================================================"
exit 1
fi
|