File size: 3,208 Bytes
4024ed7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | #!/bin/bash
# 分析accept length的脚本
echo "=========================================="
echo "Accept Length Analysis"
echo "=========================================="
echo ""
# 检查results目录
if [ ! -d "results" ]; then
echo "Error: results directory not found"
exit 1
fi
# 查找所有结果文件
result_files=$(ls results/*.jsonl 2>/dev/null)
if [ -z "$result_files" ]; then
echo "No result files found in results/ directory"
echo ""
echo "Please run the benchmark first:"
echo " python benchmarks/bench_eagle3.py ..."
exit 1
fi
echo "Found result files:"
ls -lh results/*.jsonl
echo ""
echo "=========================================="
echo ""
# 分析每个结果文件
for file in $result_files; do
filename=$(basename "$file")
echo "File: $filename"
echo "----------------------------------------"
# 检查文件是否包含mtbench结果
if grep -q "mtbench" "$file"; then
# 提取accept_length
echo "Accept lengths:"
cat "$file" | jq -r '.mtbench[0].metrics[] | " Sample \(.sample_id): accept_length=\(.accept_length // "N/A"), output_tokens=\(.output_tokens // "N/A")"' 2>/dev/null
echo ""
echo "Statistics:"
# 计算平均值
avg_accept=$(cat "$file" | jq -r '.mtbench[0].metrics[] | .accept_length' 2>/dev/null | awk '{sum+=$1; count++} END {if(count>0) printf " Average accept_length: %.4f\n", sum/count; else print " No data"}')
echo "$avg_accept"
# 计算最小值和最大值
min_accept=$(cat "$file" | jq -r '.mtbench[0].metrics[] | .accept_length' 2>/dev/null | sort -n | head -1)
max_accept=$(cat "$file" | jq -r '.mtbench[0].metrics[] | .accept_length' 2>/dev/null | sort -n | tail -1)
echo " Min accept_length: $min_accept"
echo " Max accept_length: $max_accept"
# 样本数量
sample_count=$(cat "$file" | jq -r '.mtbench[0].metrics | length' 2>/dev/null)
echo " Total samples: $sample_count"
else
echo " No mtbench results found in this file"
fi
echo ""
echo "=========================================="
echo ""
done
# 如果有baseline和trained的结果，进行对比
baseline_file=$(ls results/baseline*.jsonl 2>/dev/null | head -1)
trained_file=$(ls results/trained*.jsonl 2>/dev/null | head -1)
if [ -n "$baseline_file" ] && [ -n "$trained_file" ]; then
echo "Comparison: Baseline vs Trained"
echo "----------------------------------------"
baseline_avg=$(cat "$baseline_file" | jq -r '.mtbench[0].metrics[] | .accept_length' 2>/dev/null | awk '{sum+=$1; count++} END {if(count>0) print sum/count}')
trained_avg=$(cat "$trained_file" | jq -r '.mtbench[0].metrics[] | .accept_length' 2>/dev/null | awk '{sum+=$1; count++} END {if(count>0) print sum/count}')
if [ -n "$baseline_avg" ] && [ -n "$trained_avg" ]; then
echo "Baseline average: $baseline_avg"
echo "Trained average: $trained_avg"
# 计算提升百分比
improvement=$(echo "$baseline_avg $trained_avg" | awk '{printf "%.2f%%", ($2-$1)/$1*100}')
echo "Improvement: $improvement"
fi
echo ""
fi
echo "Done!"
|