"""Tests for check_issue_readiness.py — the ready-for-dev gate logic."""
import sys
from pathlib import Path
# Make the sibling script importable.
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from check_issue_readiness import (
evaluate_readiness,
extract_sections,
has_screenshot_or_video,
references_run_method,
has_checklist_item,
visible_text,
main,
BUG_LABEL,
ENHANCEMENT_LABEL,
)
# ---------------------------------------------------------------------------
# Helper builders
# ---------------------------------------------------------------------------
BUG_BODY_READY = """### Steps to Reproduce
Run `npm run dev` and click the button.
### Actual Behavior
The button was misaligned.
### Acceptance Criteria
- [ ] Button is centered
"""
BUG_BODY_NO_RUN_METHOD = """### Steps to Reproduce
Click the button.
### Actual Behavior
The button was misaligned.
"""
BUG_BODY_NO_SCREENSHOT = """### Steps to Reproduce
I ran `npm run dev` and clicked the button.
### Actual Behavior
The button was misaligned.
"""
BUG_BODY_NO_ACCEPTANCE = """### Steps to Reproduce
I ran `npm run dev` and clicked the button.
### Actual Behavior
The button was misaligned.
"""
BUG_BODY_EMPTY_ACTUAL = """### Steps to Reproduce
I ran `npm run dev` and clicked the button.
### Actual Behavior
_No response_
### Acceptance Criteria
- [ ] Button is centered
"""
BUG_BODY_AGENT_CANVAS = """### Steps to Reproduce
I used agent-canvas to reproduce this.
### Actual Behavior
The button was misaligned.
### Acceptance Criteria
- [ ] Fixed
"""
BUG_BODY_HOSTED_URL = """### Steps to Reproduce
Reproduced on app.all-hands.dev/canvas.
### Actual Behavior
The button was misaligned.
### Acceptance Criteria
- [ ] Fixed
"""
BUG_BODY_MISSING_REPRODUCTION = """### Actual Behavior
The button was misaligned.
### Acceptance Criteria
- [ ] Button is centered
"""
ENHANCEMENT_BODY_READY = """### Desired Behavior
The button should animate on hover.
### Acceptance Criteria
- [ ] Hover animation works
- [ ] No perf regression
"""
ENHANCEMENT_BODY_NO_DESIRED = """### Acceptance Criteria
- [ ] Something
"""
ENHANCEMENT_BODY_NO_ACCEPTANCE = """### Desired Behavior
The button should animate on hover.
"""
ENHANCEMENT_BODY_PROSE_ACCEPTANCE = """### Desired Behavior
The button should animate on hover.
### Acceptance Criteria
Make it look nice.
"""
# ---------------------------------------------------------------------------
# Bug readiness
# ---------------------------------------------------------------------------
def test_bug_ready_npm_run_screenshot():
result = evaluate_readiness(BUG_BODY_READY, [BUG_LABEL])
assert result.ready, result.reasons
def test_bug_ready_agent_canvas():
result = evaluate_readiness(BUG_BODY_AGENT_CANVAS, [BUG_LABEL])
assert result.ready, result.reasons
def test_bug_ready_hosted_url():
result = evaluate_readiness(BUG_BODY_HOSTED_URL, [BUG_LABEL])
assert result.ready, result.reasons
def test_bug_not_ready_no_run_method():
result = evaluate_readiness(BUG_BODY_NO_RUN_METHOD, [BUG_LABEL])
assert not result.ready
assert any("run method" in r for r in result.reasons)
def test_bug_not_ready_no_screenshot():
result = evaluate_readiness(BUG_BODY_NO_SCREENSHOT, [BUG_LABEL])
assert not result.ready
assert any("screenshot" in r for r in result.reasons)
def test_bug_not_ready_no_acceptance():
result = evaluate_readiness(BUG_BODY_NO_ACCEPTANCE, [BUG_LABEL])
assert not result.ready
assert any("Acceptance Criteria" in r for r in result.reasons)
def test_bug_not_ready_empty_actual():
result = evaluate_readiness(BUG_BODY_EMPTY_ACTUAL, [BUG_LABEL])
assert not result.ready
assert any("Actual Behavior" in r for r in result.reasons)
def test_bug_not_ready_missing_reproduction():
result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [BUG_LABEL])
assert not result.ready
assert any("Steps to Reproduce" in r for r in result.reasons)
# ---------------------------------------------------------------------------
# Enhancement readiness
# ---------------------------------------------------------------------------
def test_enhancement_ready():
result = evaluate_readiness(ENHANCEMENT_BODY_READY, [ENHANCEMENT_LABEL])
assert result.ready, result.reasons
def test_enhancement_not_ready_no_desired():
result = evaluate_readiness(ENHANCEMENT_BODY_NO_DESIRED, [ENHANCEMENT_LABEL])
assert not result.ready
assert any("Desired Behavior" in r for r in result.reasons)
def test_enhancement_not_ready_no_acceptance():
result = evaluate_readiness(ENHANCEMENT_BODY_NO_ACCEPTANCE, [ENHANCEMENT_LABEL])
assert not result.ready
assert any("Acceptance Criteria" in r for r in result.reasons)
def test_enhancement_not_ready_prose_acceptance():
result = evaluate_readiness(ENHANCEMENT_BODY_PROSE_ACCEPTANCE, [ENHANCEMENT_LABEL])
assert not result.ready
assert any("checklist" in r for r in result.reasons)
# ---------------------------------------------------------------------------
# No type section
# ---------------------------------------------------------------------------
def test_no_type_section_not_ready():
result = evaluate_readiness("### Something\nSome text", [])
assert not result.ready
assert result.issue_type is None
assert any("neither" in r.lower() for r in result.reasons)
def test_type_inferred_from_body_ignores_labels():
# A bug-shaped body is treated as a bug even without a `bug` label.
result = evaluate_readiness(BUG_BODY_READY, [])
assert result.ready, result.reasons
assert result.issue_type == BUG_LABEL
# A feature-shaped body is treated as an enhancement even with a `bug` label.
result = evaluate_readiness(ENHANCEMENT_BODY_READY, ["bug"])
assert result.ready, result.reasons
assert result.issue_type == ENHANCEMENT_LABEL
# A body with only an `### Actual Behavior` section (no Steps to Reproduce)
# is still recognized as a bug report.
result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [])
assert not result.ready
assert any("Steps to Reproduce" in r for r in result.reasons)
def test_empty_type_section_is_still_classified():
result = evaluate_readiness("### Actual Behavior\n### Acceptance Criteria\n", [])
assert not result.ready
assert result.issue_type == BUG_LABEL
assert any("Steps to Reproduce" in r for r in result.reasons)
def test_ambiguous_body_does_not_infer_type():
body = f"{BUG_BODY_READY}\n{ENHANCEMENT_BODY_READY}"
result = evaluate_readiness(body, [])
assert not result.ready
assert result.issue_type is None
assert any("both" in reason.lower() for reason in result.reasons)
# ---------------------------------------------------------------------------
# Unit-level helpers
# ---------------------------------------------------------------------------
def test_has_screenshot_markdown_image():
assert has_screenshot_or_video("")
def test_has_screenshot_github_attachment():
assert has_screenshot_or_video("https://github.com/user-attachments/assets/abc123")
def test_has_screenshot_html_video():
assert has_screenshot_or_video('')
def test_has_screenshot_youtube():
assert has_screenshot_or_video("https://youtube.com/watch?v=abc123")
def test_has_screenshot_none():
assert not has_screenshot_or_video("Just text, no media")
def test_references_run_method_npm():
assert references_run_method("I ran npm run dev")
def test_references_run_method_agent_canvas():
assert references_run_method("Used agent-canvas to test")
def test_references_run_method_hosted():
assert references_run_method("Reproduced on app.all-hands.dev/canvas")
def test_references_run_method_none():
assert not references_run_method("I clicked the button")
def test_has_checklist_item():
assert has_checklist_item("- [ ] Do something")
assert has_checklist_item("- [x] Done")
assert has_checklist_item(" * [ ] Indented")
def test_has_checklist_item_none():
assert not has_checklist_item("Just prose, no checklist")
def test_visible_text_strips_html_comments():
assert visible_text("visible text") == "visible text"
def test_visible_text_no_response():
assert visible_text("_No response_") == ""
def test_extract_sections():
sections = extract_sections("### Title One\nText 1\n### Title Two\nText 2")
assert "title one" in sections
assert "title two" in sections
assert "Text 1" in sections["title one"]
assert "Text 2" in sections["title two"]
def test_bug_ready_with_h2_sections():
body = BUG_BODY_READY.replace("### ", "## ")
result = evaluate_readiness(body, [])
assert result.ready, result.reasons
def test_enhancement_ready_with_mixed_h2_h3_sections():
body = ENHANCEMENT_BODY_READY.replace(
"### Desired Behavior", "## Desired Behavior"
)
result = evaluate_readiness(body, [ENHANCEMENT_LABEL])
assert result.ready, result.reasons
def test_extract_sections_excludes_h1_and_h4_headings():
body = "# Document Title\nintro\n#### Nested Detail\ntext"
assert extract_sections(body) == {}
def test_nested_h3_stays_inside_h2_readiness_section():
body = """## Steps to Reproduce
Run `npm run dev`.
## Actual Behavior
The page is broken.
### Screenshot
## Acceptance Criteria
- [ ] The page works
"""
result = evaluate_readiness(body, [BUG_LABEL])
assert result.ready, result.reasons
assert "### Screenshot" in extract_sections(body)["actual behavior"]
def test_extract_sections_ignores_h2_heading_inside_fence():
body = """## Notes
The template says:
```markdown
## Acceptance Criteria
- [ ] Add criteria here
```
"""
sections = extract_sections(body)
assert set(sections) == {"notes"}
def test_extract_sections_ignores_heading_inside_fence():
body = """### Notes
The template says:
```markdown
### Acceptance Criteria
- [ ] Add criteria here
```
"""
sections = extract_sections(body)
assert set(sections) == {"notes"}
def test_fenced_heading_does_not_truncate_actual_behavior():
body = """### Steps to Reproduce
Run `npm run dev` and reproduce the error.
### Actual Behavior
I ran `npm run dev` and saw:
~~~text
### Error detail
something went wrong
~~~
### Acceptance Criteria
- [ ] The bug is fixed
"""
sections = extract_sections(body)
assert "error detail" not in sections
assert "user-attachments" in sections["actual behavior"]
assert evaluate_readiness(body, [BUG_LABEL]).ready
def test_unclosed_fence_does_not_swallow_later_sections():
"""One stray marker in a log paste must not reject an otherwise-ready report."""
body = """### Steps to Reproduce
Run `npm run dev` and reproduce the crash.
### Relevant Logs
```shell
Traceback (most recent call last):
the paste was cut off before the closing fence
### Actual Behavior
I ran `npm run dev` and saw the crash above.
### Acceptance Criteria
- [ ] The bug is fixed
"""
sections = extract_sections(body)
assert {"relevant logs", "actual behavior", "acceptance criteria"} <= set(sections)
assert evaluate_readiness(body, [BUG_LABEL]).ready
def test_main_json_not_ready(tmp_path, capsys, monkeypatch):
import json
body_file = tmp_path / "issue.md"
body_file.write_text(BUG_BODY_NO_SCREENSHOT)
monkeypatch.setattr(
"sys.argv",
[
"check_issue_readiness.py",
"--body-file",
str(body_file),
"--labels",
"bug",
"--json",
],
)
exit_code = main()
assert exit_code == 1
captured = capsys.readouterr()
data = json.loads(captured.out)
assert data["ready"] is False
assert data["issue_type"] == BUG_LABEL
assert len(data["reasons"]) > 0
def test_main_json_ready(tmp_path, capsys, monkeypatch):
import json
body_file = tmp_path / "issue.md"
body_file.write_text(BUG_BODY_READY)
monkeypatch.setattr(
"sys.argv",
[
"check_issue_readiness.py",
"--body-file",
str(body_file),
"--labels",
"bug",
"--json",
],
)
exit_code = main()
assert exit_code == 0
captured = capsys.readouterr()
data = json.loads(captured.out)
assert data["ready"] is True
assert data["issue_type"] == BUG_LABEL
assert len(data["reasons"]) == 0
def test_main_text_ready(tmp_path, capsys, monkeypatch):
body_file = tmp_path / "issue.md"
body_file.write_text(BUG_BODY_READY)
monkeypatch.setattr(
"sys.argv",
[
"check_issue_readiness.py",
"--body-file",
str(body_file),
"--labels",
"bug",
],
)
exit_code = main()
assert exit_code == 0
captured = capsys.readouterr()
assert "Issue meets ready-for-dev criteria." in captured.out
def test_main_text_not_ready(tmp_path, capsys, monkeypatch):
body_file = tmp_path / "issue.md"
body_file.write_text(BUG_BODY_NO_SCREENSHOT)
monkeypatch.setattr(
"sys.argv",
[
"check_issue_readiness.py",
"--body-file",
str(body_file),
"--labels",
"bug",
],
)
exit_code = main()
assert exit_code == 1
captured = capsys.readouterr()
assert "Issue does not meet ready-for-dev criteria:" in captured.out
def test_main_event_path_json_ready(tmp_path, capsys, monkeypatch):
import json
event_file = tmp_path / "event.json"
event_file.write_text(
json.dumps(
{
"issue": {
"body": BUG_BODY_READY,
"labels": [{"name": "bug"}],
}
}
)
)
monkeypatch.setattr(
"sys.argv",
[
"check_issue_readiness.py",
"--event-path",
str(event_file),
"--json",
],
)
exit_code = main()
assert exit_code == 0
captured = capsys.readouterr()
data = json.loads(captured.out)
assert data["ready"] is True
assert data["issue_type"] == BUG_LABEL
assert len(data["reasons"]) == 0