src/verify.rs

//! Verifier. The metric (EVALUATION.md): execution correctness, pass@1. A
//! candidate program is correct iff, on the held-out test inputs, it reproduces
//! the reference outputs. This is the same function used to label data and to
//! score the model at eval time.

use crate::sandbox::Sandbox;
use crate::spec::IoPair;

#[derive(Debug, Clone, PartialEq)]
pub enum Verdict {
    Pass,
    /// Candidate loaded and ran but disagreed on some input.
    Wrong,
    /// Candidate failed to load.
    LoadError(String),
    /// Candidate raised/timed out on some input.
    RunError(String),
}

/// Verify a candidate `function f` against held-out tests. Pass requires every
/// test input to reproduce the expected output exactly.
pub fn verify(sb: &Sandbox, candidate_source: &str, tests: &[IoPair], budget: i64) -> Verdict {
    if let Err(e) = sb.load_program(candidate_source, budget) {
        return Verdict::LoadError(format!("{e:?}"));
    }
    for t in tests {
        match sb.call_f(&t.input, budget) {
            Ok(out) => {
                if out != t.output {
                    return Verdict::Wrong;
                }
            }
            Err(e) => return Verdict::RunError(format!("{e:?}")),
        }
    }
    Verdict::Pass
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::value::LValue;

    fn io(i: i64, o: i64) -> IoPair {
        IoPair { input: vec![LValue::Int(i)], output: LValue::Int(o) }
    }

    #[test]
    fn correct_candidate_passes() {
        let sb = Sandbox::new().unwrap();
        let tests = vec![io(1, 3), io(2, 5), io(10, 21)]; // 2x+1
        assert_eq!(verify(&sb, "function f(x) return 2*x + 1 end", &tests, 100_000), Verdict::Pass);
    }

    #[test]
    fn wrong_candidate_fails() {
        let sb = Sandbox::new().unwrap();
        let tests = vec![io(1, 3), io(2, 5)];
        assert_eq!(verify(&sb, "function f(x) return 2*x end", &tests, 100_000), Verdict::Wrong);
    }

    #[test]
    fn broken_candidate_load_error() {
        let sb = Sandbox::new().unwrap();
        let tests = vec![io(1, 3)];
        assert!(matches!(
            verify(&sb, "function f(x) return ", &tests, 100_000),
            Verdict::LoadError(_)
        ));
    }
}