| |
| |
| |
| |
| |
| use strict; |
| use warnings; |
| use utf8; |
| use FindBin qw($Bin); |
| use HTTP::Tiny; |
| use JSON::PP; |
|
|
| binmode STDOUT, ':utf8'; |
| my $check = @ARGV && $ARGV[0] eq '--check' ? shift : 0; |
| my $out_path = shift // "$Bin/data/results.tsv"; |
| my $json = JSON::PP->new->utf8; |
| my $http = HTTP::Tiny->new(timeout => 300); |
| my $not_found = 'could not find this in your A/L ICT books'; |
|
|
| open(my $pf, '<:raw', "$Bin/data/pages.json") or die "pages.json: $!\n"; |
| my $pages = do { local $/; $json->decode(<$pf>) }; |
| close $pf; |
| my %titles = ('eALSyl ICT.pdf' => 'A/L ICT Syllabus', 'eGr12TG ICT.pdf' => 'Grade 12 Teacher\'s Guide', 'eOMAL ICT Resource book.pdf' => 'Grade 13 Resource Book'); |
|
|
| sub matches { |
| my ($text, $groups) = @_; |
| for my $g (@$groups) { |
| return 0 unless grep { index(lc $text, lc $_) >= 0 } @$g; |
| } |
| return 1; |
| } |
|
|
| my @qs; |
| my $q_file = $ENV{Q_FILE} // "$Bin/questions.txt"; |
| my $think = $ENV{THINK} ? JSON::PP::true : JSON::PP::false; |
| open(my $qf, '<:encoding(UTF-8)', $q_file) or die "$q_file: $!\n"; |
| while (<$qf>) { |
| next if /^\s*( |
| chomp; |
| my ($unit, $q, $terms) = map { s/^\s+|\s+$//gr } split /\|/, $_, 3; |
| my @groups = map { [ map { s/^\s+|\s+$//gr } split m{/}, $_ ] } grep { /\S/ } split /;/, $terms // ''; |
| my %gold; |
| for my $p (@$pages) { |
| $gold{"$titles{$p->{book}}|$p->{page}"} = 1 if @groups && matches($p->{text}, \@groups); |
| } |
| push @qs, { unit => $unit, q => $q, groups => \@groups, gold => \%gold }; |
| } |
| close $qf; |
|
|
| if ($check) { |
| for (@qs) { |
| next if $_->{unit} eq 'REFUSE' || %{ $_->{gold} }; |
| print "No page has all terms: $_->{q}\n"; |
| } |
| print scalar(@qs), " questions checked\n"; |
| exit 0; |
| } |
|
|
| open(my $out, '>:encoding(UTF-8)', $out_path) or die "$out_path: $!\n"; |
| $out->autoflush(1); |
| print $out join("\t", qw(result unit retrieved secs question answer sources)), "\n"; |
| my (%tot, %ok, %ret, %cit); |
| for my $item (@qs) { |
| my $t0 = time; |
| my $res = $http->post('http://127.0.0.1:8765/api/ask', { |
| headers => { 'Content-Type' => 'application/json' }, |
| content => $json->encode({ think => $think, messages => [ { role => 'user', content => $item->{q} } ] }), |
| }); |
| my ($answer, @src) = (''); |
| if ($res->{success}) { |
| for my $line (split /\n/, $res->{content}) { |
| my $c = eval { $json->decode($line) } or next; |
| @src = @{ $c->{sources} } if $c->{sources}; |
| $answer .= $c->{message}{content} if $c->{message} && defined $c->{message}{content}; |
| } |
| } |
| my $refused = index($answer, $not_found) >= 0; |
| my $hit = grep { $item->{gold}{"$_->{book}|$_->{page}"} } @src; |
| my ($unit, $pass) = ($item->{unit}); |
| if ($unit eq 'REFUSE') { |
| $pass = $refused; |
| } else { |
| $pass = !$refused && ($unit eq 'SI' ? $hit : matches($answer, $item->{groups})); |
| $ret{$unit} += $hit ? 1 : 0; |
| $cit{$unit} += $answer =~ /\[\d/ ? 1 : 0; |
| } |
| $tot{$unit}++; |
| $ok{$unit} += $pass ? 1 : 0; |
| (my $flat = $answer) =~ s/\s+/ /g; |
| print $out join("\t", ($pass ? 'PASS' : 'FAIL'), $unit, ($hit ? 'yes' : 'no'), time - $t0, $item->{q}, $flat, |
| join(', ', map { "$_->{book} p$_->{page}" } @src)), "\n"; |
| printf "%s [%s] %s\n", ($pass ? 'PASS' : 'FAIL'), $unit, $item->{q}; |
| } |
| close $out; |
|
|
| my ($all, $good, $tret, $tcit, $ans) = (0, 0, 0, 0, 0); |
| for my $u (sort { ($a =~ /^\d/ && $b =~ /^\d/) ? $a <=> $b : $a cmp $b } keys %tot) { |
| printf "%-7s %d/%d retrieved %d cited %d\n", $u, $ok{$u}, $tot{$u}, $ret{$u} // 0, $cit{$u} // 0; |
| $all += $tot{$u}; |
| $good += $ok{$u}; |
| $tret += $ret{$u} // 0; |
| $tcit += $cit{$u} // 0; |
| $ans += $tot{$u} if $u ne 'REFUSE'; |
| } |
| printf "TOTAL %d/%d passed (%.1f%%), right page retrieved %d/%d, cited %d/%d\n", $good, $all, 100 * $good / $all, $tret, $ans, $tcit, $ans; |
|
|
