File size: 4,636 Bytes
cc036ff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | #!/usr/bin/env python3
"""
Detect flaky tests from pytest JSON report.
Usage:
python detect_flaky_tests.py <pytest_report.json> [options]
Options:
--threshold FLOAT Pass rate threshold (default: 0.8)
--min-runs INT Minimum runs before considering (default: 3)
--last-n INT Only consider last N runs (default: all)
--output FILE Output file for flaky test report (default: stdout)
Exit codes:
0: No flaky tests found
1: Flaky tests detected
2: Error occurred
"""
import argparse
import json
import sys
from pathlib import Path
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))
from scripts.flaky_test_tracker import FlakyTestTracker
def parse_args():
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description="Detect flaky tests from pytest JSON report"
)
parser.add_argument(
"report_file",
help="Path to pytest JSON report file"
)
parser.add_argument(
"--threshold",
type=float,
default=0.8,
help="Pass rate threshold (default: 0.8)"
)
parser.add_argument(
"--min-runs",
type=int,
default=3,
help="Minimum runs before considering test (default: 3)"
)
parser.add_argument(
"--last-n",
type=int,
default=None,
help="Only consider last N runs (default: all)"
)
parser.add_argument(
"--output",
type=str,
default=None,
help="Output file for flaky test report (default: stdout)"
)
parser.add_argument(
"--data-file",
type=str,
default="backend/tests/e2e_ui/data/flaky_tests.json",
help="Flaky test tracker data file"
)
return parser.parse_args()
def main():
"""Main entry point."""
args = parse_args()
# Verify report file exists
if not Path(args.report_file).exists():
print(f"Error: Report file '{args.report_file}' not found", file=sys.stderr)
return 2
# Update tracker with current report
tracker = FlakyTestTracker(data_file=args.data_file)
try:
tracker.update_from_pytest_report(args.report_file)
except Exception as e:
print(f"Error updating tracker: {e}", file=sys.stderr)
return 2
# Get flaky tests
flaky_tests = tracker.get_flaky_tests(
pass_threshold=args.threshold,
min_runs=args.min_runs,
last_n_runs=args.last_n
)
# Generate report
report_lines = []
report_lines.append("=" * 60)
report_lines.append("FLAKY TEST DETECTION REPORT")
report_lines.append("=" * 60)
report_lines.append(f"Threshold: {args.threshold:.0%} pass rate")
report_lines.append(f"Minimum runs: {args.min_runs}")
report_lines.append("")
if flaky_tests:
report_lines.append(f"Found {len(flaky_tests)} FLAKY TESTS:")
report_lines.append("")
for i, test in enumerate(flaky_tests, 1):
report_lines.append(f"{i}. {test['test']}")
report_lines.append(f" Pass rate: {test['pass_rate']:.1%} ({test['passed']}/{test['total_runs']} runs)")
report_lines.append(f" Last outcome: {test['last_outcome']}")
report_lines.append("")
# Summary
summary = tracker.get_summary()
report_lines.append("-" * 60)
report_lines.append(f"Total tests tracked: {summary['total_tests']}")
report_lines.append(f"Total CI runs: {summary['total_runs']}")
report_lines.append(f"Flaky tests: {len(flaky_tests)}")
report_lines.append("=" * 60)
report_text = "\n".join(report_lines)
if args.output:
with open(args.output, 'w') as f:
f.write(report_text)
print(f"Report written to {args.output}")
else:
print(report_text)
return 1 # Exit code 1 = flaky tests found
else:
report_lines.append("No flaky tests detected!")
report_lines.append("")
summary = tracker.get_summary()
report_lines.append(f"Total tests tracked: {summary['total_tests']}")
report_lines.append(f"Total CI runs: {summary['total_runs']}")
report_lines.append("=" * 60)
report_text = "\n".join(report_lines)
if args.output:
with open(args.output, 'w') as f:
f.write(report_text)
print(f"Report written to {args.output}")
else:
print(report_text)
return 0 # Exit code 0 = no flaky tests
if __name__ == "__main__":
sys.exit(main())
|