"""Tests for VerilogEvaluator.""" class TestCompilation: """Test Verilog compilation.""" def test_compile_valid_module(self, evaluator, requires_iverilog): verilog = "module foo (input a, output b); assign b = a; endmodule" result = evaluator.compile(verilog) assert result.success is True assert result.stderr == "" or "warning" not in result.stderr.lower() def test_compile_syntax_error(self, evaluator, requires_iverilog): verilog = "module foo (input a, output b) assign b = a; endmodule" # missing ; result = evaluator.compile(verilog) assert result.success is False assert len(result.stderr) > 0 def test_compile_empty_module(self, evaluator, requires_iverilog): verilog = "module empty; endmodule" result = evaluator.compile(verilog) assert result.success is True class TestMACUnit: """Test MAC unit compilation, simulation, and grading.""" def test_mac_reference_compiles( self, evaluator, mac_reference_verilog, requires_iverilog ): result = evaluator.compile(mac_reference_verilog) assert result.success is True def test_mac_reference_grading( self, evaluator, mac_reference_verilog, environment, requires_eda_tools ): task = environment.tasks["mac_unit"] result = evaluator.grade(mac_reference_verilog, task_id="mac_unit", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) # Verify grading pipeline works: compile → simulate → grade assert result.compilation.success is True assert result.simulation is not None assert result.simulation.tests_total > 0 # Final score should be computed assert result.final_score is not None assert 0.01 <= result.final_score <= 0.99 def test_mac_empty_submission(self, evaluator, environment): task = environment.tasks["mac_unit"] result = evaluator.grade("", task_id="mac_unit", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.final_score == 0.01 assert result.compilation.success is False def test_mac_broken_module(self, evaluator, environment): broken = "module mac_unit (input a, output b); endmodule" # wrong interface task = environment.tasks["mac_unit"] result = evaluator.grade(broken, task_id="mac_unit", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) # Broken module fails simulation, gets very low score assert result.final_score < 0.2 assert result.simulation is not None assert result.simulation.tests_passed == 0 class TestAXIFIFO: """Test AXI FIFO compilation and grading.""" def test_axi_reference_compiles( self, evaluator, axi_reference_verilog, requires_iverilog ): result = evaluator.compile(axi_reference_verilog) assert result.success is True def test_axi_reference_grading( self, evaluator, axi_reference_verilog, environment, requires_eda_tools ): task = environment.tasks["axi_fifo"] result = evaluator.grade(axi_reference_verilog, task_id="axi_fifo", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.final_score > 0.2 # Should score decently assert result.compilation.success is True assert result.simulation is not None assert result.simulation.tests_total > 0 class TestSystolicArray: """Test systolic array compilation and grading.""" def test_systolic_reference_compiles( self, evaluator, systolic_reference_verilog, requires_iverilog ): result = evaluator.compile(systolic_reference_verilog) assert result.success is True def test_systolic_reference_grading( self, evaluator, systolic_reference_verilog, environment, requires_eda_tools ): task = environment.tasks["systolic_array"] result = evaluator.grade(systolic_reference_verilog, task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.final_score > 0.0 assert result.compilation.success is True assert result.simulation is not None assert result.simulation.tests_total > 0 # Reference should pass some tests assert result.simulation.tests_passed > 0 def test_systolic_reference_timing( self, evaluator, systolic_reference_verilog, environment, requires_eda_tools ): """Timing dimension is captured and scored within [0, 1].""" task = environment.tasks["systolic_array"] result = evaluator.grade(systolic_reference_verilog, task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) # Score breakdown always present and timing score is in valid range assert result.score_breakdown is not None timing_score = result.score_breakdown.get("timing", -1.0) assert 0.01 <= timing_score <= 0.99 def test_systolic_empty_submission( self, evaluator, environment, requires_eda_tools ): """Empty submission must score the minimum clamped value.""" task = environment.tasks["systolic_array"] result = evaluator.grade("", task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.final_score == 0.01 assert result.compilation.success is False def test_systolic_broken_module(self, evaluator, environment, requires_eda_tools): """A module that compiles but has wrong logic should score only compile credit.""" # Use valid interface but output all 1s (0xFFFF) so it fails EVERY test case broken = """ module systolic_array ( input wire clk, input wire rst, input wire load_weights, input wire [63:0] weights_flat, input wire start, input wire [127:0] activations_flat, output wire [255:0] outputs_flat, output wire done ); assign outputs_flat = {256{1'b1}}; assign done = 1'b0; endmodule """ task = environment.tasks["systolic_array"] result = evaluator.grade(broken, task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.compilation.success is True # Sim should fail 100% of checks — compile credit (5%) plus minimum floor (0.01) # for each unevaluated component: 0.05×0.99 + (0.50+0.30+0.15)×0.01 = 0.059 assert result.final_score <= 0.059 + 1e-6 if result.simulation: assert result.simulation.tests_passed == 0 def test_systolic_area_gated_on_sim( self, evaluator, environment, requires_eda_tools ): """Area score must be 0 when no tests pass, even for a compact module.""" broken = """ module systolic_array ( input wire clk, input wire rst, input wire load_weights, input wire [63:0] weights_flat, input wire start, input wire [127:0] activations_flat, output wire [255:0] outputs_flat, output wire done ); assign outputs_flat = {256{1'b1}}; assign done = 1'b0; endmodule """ task = environment.tasks["systolic_array"] result = evaluator.grade(broken, task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) if result.score_breakdown: # Area must not exceed minimum when no tests pass (gated on sim.tests_passed > 0) assert result.score_breakdown.get("area", 0.01) <= 0.01 def test_systolic_score_breakdown_keys( self, evaluator, environment, requires_eda_tools ): """Score breakdown must contain compile, sim, timing, area keys for systolic_array.""" broken = """ module systolic_array ( input wire clk, input wire rst, input wire load_weights, input wire [63:0] weights_flat, input wire start, input wire [127:0] activations_flat, output wire [255:0] outputs_flat, output wire done ); assign outputs_flat = {256{1'b1}}; assign done = 1'b0; endmodule """ task = environment.tasks["systolic_array"] result = evaluator.grade(broken, task_id="systolic_array", testbench_path=task.testbench_path, reference_cells=task.reference_cells, ) assert result.score_breakdown is not None assert set(result.score_breakdown.keys()) == { "compile", "sim", "timing", "area", } for v in result.score_breakdown.values(): assert 0.01 <= v <= 0.99 class TestSynthesis: """Test synthesis (yosys).""" def test_simple_synthesis(self, evaluator, requires_yosys): verilog = "module foo (input a, b, output c); assign c = a & b; endmodule" result = evaluator.synthesize(verilog) # Synthesis may succeed or fail depending on yosys version, but should run assert result.stdout is not None or result.stderr is not None def test_mac_synthesis(self, evaluator, mac_reference_verilog, requires_yosys): result = evaluator.synthesize(mac_reference_verilog) # Synthesis should produce some output assert result.stdout is not None or result.stderr is not None