#!/usr/bin/env python3 """ Checker for LLM geometric reasoning benchmark responses. Given: 1. A target triangulation (from the benchmark) 2. LLM's proposed answer (either point set or "None") Verify: - If LLM said "None": Check that triangulation is indeed non-realizable - If LLM gave points: Check that Delaunay triangulation is isomorphic to target Uses pynauty for robust graph isomorphism checking. """ import numpy as np import json import sys from pathlib import Path from scipy.spatial import Delaunay from typing import Optional, Dict, List, Tuple sys.path.insert(0, str(Path(__file__).parent.parent)) try: import pynauty except ImportError: print("Error: pynauty not installed. Install with: pip install pynauty") sys.exit(1) def compute_canonical_hash(triangles: List[Tuple[int, int, int]]) -> str: """ Compute canonical hash of triangulation using pynauty. This gives us a way to check if two triangulations are isomorphic, regardless of vertex labeling. Args: triangles: List of triangles as (v0, v1, v2) tuples Returns: Canonical hash string """ # Get all vertices vertices = set() for tri in triangles: vertices.update(tri) n_vertices = len(vertices) # Create vertex mapping vertex_list = sorted(vertices) vertex_to_idx = {v: i for i, v in enumerate(vertex_list)} # Build adjacency sets (graph representation) adjacency = {i: set() for i in range(n_vertices)} for v0, v1, v2 in triangles: i0 = vertex_to_idx[v0] i1 = vertex_to_idx[v1] i2 = vertex_to_idx[v2] # Add edges adjacency[i0].add(i1) adjacency[i0].add(i2) adjacency[i1].add(i0) adjacency[i1].add(i2) adjacency[i2].add(i0) adjacency[i2].add(i1) # Create pynauty graph g = pynauty.Graph(number_of_vertices=n_vertices, directed=False, adjacency_dict=adjacency) # Compute canonical labeling canonical_label = pynauty.canon_label(g) # Return as string (hashable) return str(canonical_label) def check_response( target_triangulation: List[Tuple[int, int, int]], llm_response: Optional[np.ndarray], verbose: bool = True ) -> Dict: """ Check if LLM's response is correct. Args: target_triangulation: The triangulation from the benchmark llm_response: Either None (LLM says impossible) or np.ndarray of points verbose: If True, print diagnostic info Returns: Dict with: - 'correct': bool, whether LLM answer is correct - 'reason': str, explanation - 'details': dict with additional info """ if verbose: print("="*70) print("CHECKING LLM RESPONSE") print("="*70) print() # Compute canonical hash of target target_hash = compute_canonical_hash(target_triangulation) if verbose: print(f"Target triangulation:") print(f" Vertices: {len(set(v for tri in target_triangulation for v in tri))}") print(f" Triangles: {len(target_triangulation)}") print(f" Canonical hash: {target_hash[:50]}...") print() if llm_response is None: # LLM claims triangulation is not realizable if verbose: print("LLM response: None (claims triangulation is not realizable)") print() print("Verifying claim by checking Rivin constraints...") from ideal_poly_volume_toolkit.rivin_delaunay import check_delaunay_realizability result = check_delaunay_realizability(target_triangulation, verbose=False) if not result['realizable']: # LLM correctly identified non-realizable triangulation if verbose: print(" ✓ Confirmed: Triangulation is NOT realizable") print() return { 'correct': True, 'reason': 'LLM correctly identified non-realizable triangulation', 'details': { 'llm_said': 'None', 'actual_realizable': False, 'lp_result': result, }, } else: # LLM incorrectly said it's not realizable if verbose: print(" ✗ ERROR: Triangulation IS realizable!") print(f" Min angle: {np.degrees(result['min_angle_radians']):.2f}°") print() return { 'correct': False, 'reason': 'LLM incorrectly claimed triangulation is not realizable', 'details': { 'llm_said': 'None', 'actual_realizable': True, 'lp_result': result, }, } # LLM provided a point set if verbose: print(f"LLM response: Point set with {llm_response.shape[0]} vertices") print() # Check dimensions if llm_response.ndim != 2 or llm_response.shape[1] != 2: if verbose: print(f" ✗ ERROR: Expected shape (n, 2), got {llm_response.shape}") print() return { 'correct': False, 'reason': f'Invalid point set shape: {llm_response.shape}', 'details': {'llm_said': 'points', 'error': 'invalid_shape'}, } n_target_vertices = len(set(v for tri in target_triangulation for v in tri)) if llm_response.shape[0] != n_target_vertices: if verbose: print(f" ✗ ERROR: Expected {n_target_vertices} vertices, got {llm_response.shape[0]}") print() return { 'correct': False, 'reason': f'Wrong number of vertices: expected {n_target_vertices}, got {llm_response.shape[0]}', 'details': { 'llm_said': 'points', 'expected_vertices': n_target_vertices, 'got_vertices': llm_response.shape[0], }, } # Compute Delaunay triangulation of LLM's points if verbose: print("Computing Delaunay triangulation of proposed points...") try: tri = Delaunay(llm_response) llm_triangulation = [tuple(simplex) for simplex in tri.simplices] if verbose: print(f" Triangulation: {len(llm_triangulation)} triangles") print() except Exception as e: if verbose: print(f" ✗ ERROR: Could not compute Delaunay triangulation: {e}") print() return { 'correct': False, 'reason': f'Delaunay triangulation failed: {e}', 'details': {'llm_said': 'points', 'error': 'delaunay_failed'}, } # Compute canonical hash of LLM's triangulation llm_hash = compute_canonical_hash(llm_triangulation) if verbose: print("Checking combinatorial equivalence (graph isomorphism)...") print(f" Target hash: {target_hash[:50]}...") print(f" LLM hash: {llm_hash[:50]}...") print() # Check if hashes match if target_hash == llm_hash: if verbose: print(" ✓ SUCCESS: Triangulations are isomorphic!") print(" LLM provided a valid point set with correct combinatorics") print() return { 'correct': True, 'reason': 'LLM provided valid point set with correct combinatorics', 'details': { 'llm_said': 'points', 'isomorphic': True, 'target_triangles': len(target_triangulation), 'llm_triangles': len(llm_triangulation), }, } else: if verbose: print(" ✗ INCORRECT: Triangulations are NOT isomorphic") print(f" Target: {len(target_triangulation)} triangles") print(f" LLM: {len(llm_triangulation)} triangles") print() return { 'correct': False, 'reason': 'Point set produces different combinatorial structure', 'details': { 'llm_said': 'points', 'isomorphic': False, 'target_triangles': len(target_triangulation), 'llm_triangles': len(llm_triangulation), }, } def load_benchmark(filepath: str) -> Dict: """Load benchmark JSON file.""" with open(filepath, 'r') as f: return json.load(f) def main(): import argparse parser = argparse.ArgumentParser( description="Check LLM response for geometric reasoning benchmark" ) parser.add_argument( "benchmark", type=str, help="Path to benchmark JSON file", ) parser.add_argument( "challenge_idx", type=int, help="Challenge index (0-based)", ) parser.add_argument( "--points", type=str, default=None, help="Path to NPY file with proposed points, or 'None' if claiming non-realizable", ) args = parser.parse_args() # Load benchmark benchmark = load_benchmark(args.benchmark) if args.challenge_idx < 0 or args.challenge_idx >= len(benchmark['challenges']): print(f"Error: Invalid challenge index {args.challenge_idx}") print(f"Valid range: 0 to {len(benchmark['challenges'])-1}") return 1 challenge = benchmark['challenges'][args.challenge_idx] print() print("#"*70) print("# LLM Response Checker") print("#"*70) print() print(f"Challenge: {challenge['label']}") print() # Load LLM response if args.points is None or args.points.lower() == 'none': llm_response = None else: try: llm_response = np.load(args.points) except Exception as e: print(f"Error loading points file: {e}") return 1 # Check response target_triangulation = [tuple(tri) for tri in challenge['triangles']] result = check_response(target_triangulation, llm_response, verbose=True) # Print summary print("="*70) print("RESULT") print("="*70) if result['correct']: print("✓ CORRECT") else: print("✗ INCORRECT") print() print(f"Reason: {result['reason']}") print("="*70) return 0 if result['correct'] else 1 if __name__ == "__main__": sys.exit(main())