#!/usr/bin/perl
# Asks every question in questions.txt through the running chat server and scores it.
# usage: perl rag/evaluate.pl [--check] [results.tsv]
#   --check only lists questions whose required terms don't appear on any book page.
# Scoring is by keyword, so it is a screen for errors; read results.tsv for real accuracy.
use strict;
use warnings;
use utf8;
use FindBin qw($Bin);
use HTTP::Tiny;
use JSON::PP;

binmode STDOUT, ':utf8';
my $check = @ARGV && $ARGV[0] eq '--check' ? shift : 0;
my $out_path = shift // "$Bin/data/results.tsv";
my $json = JSON::PP->new->utf8;
my $http = HTTP::Tiny->new(timeout => 300);
my $not_found = 'could not find this in your A/L ICT books';

open(my $pf, '<:raw', "$Bin/data/pages.json") or die "pages.json: $!\n";
my $pages = do { local $/; $json->decode(<$pf>) };
close $pf;
my %titles = ('eALSyl ICT.pdf' => 'A/L ICT Syllabus', 'eGr12TG ICT.pdf' => 'Grade 12 Teacher\'s Guide', 'eOMAL ICT Resource book.pdf' => 'Grade 13 Resource Book');

sub matches {
    my ($text, $groups) = @_;
    for my $g (@$groups) {
        return 0 unless grep { index(lc $text, lc $_) >= 0 } @$g;
    }
    return 1;
}

my @qs;
my $q_file = $ENV{Q_FILE} // "$Bin/questions.txt";
my $think = $ENV{THINK} ? JSON::PP::true : JSON::PP::false;
open(my $qf, '<:encoding(UTF-8)', $q_file) or die "$q_file: $!\n";
while (<$qf>) {
    next if /^\s*(#|$)/;
    chomp;
    my ($unit, $q, $terms) = map { s/^\s+|\s+$//gr } split /\|/, $_, 3;
    my @groups = map { [ map { s/^\s+|\s+$//gr } split m{/}, $_ ] } grep { /\S/ } split /;/, $terms // '';
    my %gold;
    for my $p (@$pages) {
        $gold{"$titles{$p->{book}}|$p->{page}"} = 1 if @groups && matches($p->{text}, \@groups);
    }
    push @qs, { unit => $unit, q => $q, groups => \@groups, gold => \%gold };
}
close $qf;

if ($check) {
    for (@qs) {
        next if $_->{unit} eq 'REFUSE' || %{ $_->{gold} };
        print "No page has all terms: $_->{q}\n";
    }
    print scalar(@qs), " questions checked\n";
    exit 0;
}

open(my $out, '>:encoding(UTF-8)', $out_path) or die "$out_path: $!\n";
$out->autoflush(1);
print $out join("\t", qw(result unit retrieved secs question answer sources)), "\n";
my (%tot, %ok, %ret, %cit);
for my $item (@qs) {
    my $t0 = time;
    my $res = $http->post('http://127.0.0.1:8765/api/ask', {
        headers => { 'Content-Type' => 'application/json' },
        content => $json->encode({ think => $think, messages => [ { role => 'user', content => $item->{q} } ] }),
    });
    my ($answer, @src) = ('');
    if ($res->{success}) {
        for my $line (split /\n/, $res->{content}) {
            my $c = eval { $json->decode($line) } or next;
            @src = @{ $c->{sources} } if $c->{sources};
            $answer .= $c->{message}{content} if $c->{message} && defined $c->{message}{content};
        }
    }
    my $refused = index($answer, $not_found) >= 0;
    my $hit = grep { $item->{gold}{"$_->{book}|$_->{page}"} } @src;
    my ($unit, $pass) = ($item->{unit});
    if ($unit eq 'REFUSE') {
        $pass = $refused;
    } else {
        $pass = !$refused && ($unit eq 'SI' ? $hit : matches($answer, $item->{groups}));
        $ret{$unit} += $hit ? 1 : 0;
        $cit{$unit} += $answer =~ /\[\d/ ? 1 : 0;
    }
    $tot{$unit}++;
    $ok{$unit} += $pass ? 1 : 0;
    (my $flat = $answer) =~ s/\s+/ /g;
    print $out join("\t", ($pass ? 'PASS' : 'FAIL'), $unit, ($hit ? 'yes' : 'no'), time - $t0, $item->{q}, $flat,
        join(', ', map { "$_->{book} p$_->{page}" } @src)), "\n";
    printf "%s [%s] %s\n", ($pass ? 'PASS' : 'FAIL'), $unit, $item->{q};
}
close $out;

my ($all, $good, $tret, $tcit, $ans) = (0, 0, 0, 0, 0);
for my $u (sort { ($a =~ /^\d/ && $b =~ /^\d/) ? $a <=> $b : $a cmp $b } keys %tot) {
    printf "%-7s %d/%d  retrieved %d  cited %d\n", $u, $ok{$u}, $tot{$u}, $ret{$u} // 0, $cit{$u} // 0;
    $all += $tot{$u};
    $good += $ok{$u};
    $tret += $ret{$u} // 0;
    $tcit += $cit{$u} // 0;
    $ans += $tot{$u} if $u ne 'REFUSE';
}
printf "TOTAL %d/%d passed (%.1f%%), right page retrieved %d/%d, cited %d/%d\n", $good, $all, 100 * $good / $all, $tret, $ans, $tcit, $ans;