Spaces:
Running
Running
File size: 5,239 Bytes
0646b18 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
#!/bin/bash
# Digital Sales Task Profiler Runner
# This script runs the digital sales task profiler with different configurations
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROFILING_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PROJECT_ROOT="$(cd "$PROFILING_ROOT/../../.." && pwd)"
# Load environment variables from .env file if it exists
if [ -f "$PROJECT_ROOT/.env" ]; then
export $(cat "$PROJECT_ROOT/.env" | grep -v '^#' | xargs)
fi
# Default values
CONFIGS="settings.openai.toml,settings.azure.toml,settings.watsonx.toml"
MODES="fast,balanced,accurate"
TASKS="test_get_top_account_by_revenue_stream,test_list_my_accounts,test_find_vp_sales_active_high_value_accounts"
RUNS=1
OUTPUT="$PROFILING_ROOT/reports/profiling_report_$(date +%Y%m%d_%H%M%S).json"
CONFIG_FILE=""
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
--config-file)
CONFIG_FILE="$2"
shift 2
;;
--configs)
CONFIGS="$2"
shift 2
;;
--modes)
MODES="$2"
shift 2
;;
--tasks)
TASKS="$2"
shift 2
;;
--runs)
RUNS="$2"
shift 2
;;
--output)
OUTPUT="$2"
shift 2
;;
--test-id)
TEST_ID="$2"
shift 2
;;
--list-tests)
LIST_TESTS=true
shift
;;
--help)
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " --config-file FILE YAML configuration file to use"
echo " --configs CONFIGS Comma-separated list of configs (default: $CONFIGS)"
echo " --modes MODES Comma-separated list of modes (default: $MODES)"
echo " --tasks TASKS Comma-separated list of tasks (default: $TASKS)"
echo " --runs RUNS Number of runs per configuration (default: $RUNS)"
echo " --output OUTPUT Output file for the report (default: $OUTPUT)"
echo " --test-id TEST_ID Run only a specific test by ID (format: config:mode:task)"
echo " --list-tests List all available test IDs and exit"
echo " --help Show this help message"
echo ""
echo "Environment Variables Required:"
echo " LANGFUSE_PUBLIC_KEY Your Langfuse public key"
echo " LANGFUSE_SECRET_KEY Your Langfuse secret key"
echo " LANGFUSE_HOST Langfuse host URL (optional, default: https://cloud.langfuse.com)"
echo ""
echo "Examples:"
echo " $0 --config-file default_experiment.yaml"
echo " $0 --configs settings.openai.toml,settings.azure.toml --modes fast,balanced --runs 3"
echo " $0 --test-id settings.openai.toml:fast:test_get_top_account_by_revenue_stream --runs 5"
exit 0
;;
*)
echo "Unknown option: $1"
echo "Use --help for usage information"
exit 1
;;
esac
done
# Check for required environment variables
if [ -z "$LANGFUSE_PUBLIC_KEY" ] || [ -z "$LANGFUSE_SECRET_KEY" ]; then
echo "Error: LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY environment variables are required"
echo "Please set them in your environment or .env file"
exit 1
fi
# Kill any existing processes on the ports we'll use
echo "Cleaning up existing processes..."
lsof -ti:8000,8001,7860 | xargs kill -9 2>/dev/null || true
echo "Starting Digital Sales Task Profiler..."
echo "Profiling Root: $PROFILING_ROOT"
# Handle list-tests option
if [ "$LIST_TESTS" = true ]; then
echo "Listing available test IDs..."
cd "$PROJECT_ROOT"
uv run python "$SCRIPT_DIR/profile_digital_sales_tasks.py" --list-tests
exit 0
fi
# Build command based on whether config file is provided
cd "$PROJECT_ROOT"
if [ -n "$CONFIG_FILE" ]; then
echo "Using configuration file: $CONFIG_FILE"
CMD_ARGS="--config-file $CONFIG_FILE"
# Allow CLI overrides
[ -n "$TEST_ID" ] && CMD_ARGS="$CMD_ARGS --test-id $TEST_ID"
[ "$RUNS" != "1" ] && CMD_ARGS="$CMD_ARGS --runs $RUNS"
[ -n "$OUTPUT" ] && [ "$OUTPUT" != "$PROFILING_ROOT/reports/profiling_report_$(date +%Y%m%d_%H%M%S).json" ] && CMD_ARGS="$CMD_ARGS --output $OUTPUT"
else
echo "Configurations: $CONFIGS"
echo "Modes: $MODES"
echo "Tasks: $TASKS"
echo "Runs per config: $RUNS"
echo "Output file: $OUTPUT"
echo ""
CMD_ARGS=""
[ -n "$TEST_ID" ] && CMD_ARGS="--test-id $TEST_ID" || CMD_ARGS="--configs $CONFIGS --modes $MODES --tasks $TASKS"
CMD_ARGS="$CMD_ARGS --runs $RUNS --output $OUTPUT"
fi
# Display environment variables if MODEL_NAME is set
echo "[DEBUG run_profiling.sh] MODEL_NAME in environment: ${MODEL_NAME:-NOT SET}"
if [ -n "$MODEL_NAME" ]; then
echo "Environment: MODEL_NAME=$MODEL_NAME"
fi
# Run the profiler
echo "Running: uv run python $SCRIPT_DIR/profile_digital_sales_tasks.py $CMD_ARGS"
uv run python "$SCRIPT_DIR/profile_digital_sales_tasks.py" $CMD_ARGS
echo ""
echo "Profiling completed!"
|