File size: 5,218 Bytes
fd49381 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 |
#!/usr/bin/env perl
# Converts AT&T FSA format to 'python lattice format'.
# Note that the input FSA needs to be epsilon-free and topologically sorted.
# This script checks for topological sortedness.
# The start node has to have the index 0.
# All path ends are assumed to be final nodes, not just the explicitly stated
# final nodes.
# Note that the output format may not contain any spaces.
# Ondrej Bojar, bojar@ufal.mff.cuni.cz
#
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
use warnings;
use strict;
use Getopt::Long;
binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
binmode(STDERR, ":utf8");
my $filelist;
my $ignore_final_state_cost = 0;
my $mangle_weights = undef;
GetOptions(
"ignore-final-state-cost" => \$ignore_final_state_cost,
# sometimes, final states have a cost (e.g. "45 0.05\n")
# instead of dying there, ignore the problem
"filelist|fl=s" => \$filelist,
"mangle-weights=s" => \$mangle_weights,
) or exit 1;
my @infiles;
if (defined $filelist) {
my $fh = my_open($filelist);
while (<$fh>) {
chomp;
push @infiles, $_;
}
close $fh;
}
push @infiles, @ARGV;
@ARGV = ();
if (0 == scalar(@infiles)) {
print STDERR "Reading input from stdin\n";
push @infiles, "-";
}
my $err = 0;
foreach my $inf (@infiles) {
my $nr = 0;
NEXTLATTICE:
my %usedids = (); # collect all used ids for densification
my %usedtgtids = (); # collect all used ids for densification
my @outnodes = ();
my $fh = my_open($inf);
my %is_final; # remember which nodes were final
while (<$fh>) {
chomp;
$nr++;
last if $_ eq ""; # assume a blank line delimits lattices
my ($src, $tgt, $label, $weight) = split /\s+/;
die "$inf:$nr:Bad src node index: $src" if $src !~ /^[0-9]+$/;
if (!defined $label && !defined $weight) {
# explicit final node, warn at the end if there are any intermed. final
# nodes
$is_final{$src};
# final nodes can have a cost
die "$inf:$nr:Final state $src has cost $tgt. Unsupported, use --ignore-final-state-cost"
if defined $tgt && !$ignore_final_state_cost;
next;
}
$weight = 0 if !defined $weight;
$usedids{$src} = 1;
$usedtgtids{$tgt} = 1;
# process the weight
# when reading RWTH FSA output, the weights are negated natural logarithms
# we need to negate them back
if (defined $mangle_weights) {
if ($mangle_weights eq "expneg") {
$weight = join(",", map {exp(-$_)} split /,/, $weight);
} else {
die "Bad weights mangling: $mangle_weights";
}
}
# remember the node
my $targetnode = $tgt-$src;
die "$inf:$nr:Not topologically sorted, got arc from $src to $tgt"
if $targetnode <= 0;
push @{$outnodes[$src]}, [ $label, $weight, $tgt ];
}
if (eof($fh)) {
close $fh;
$fh = undef;
}
# Assign our dense IDs: source node ids are assigned first
my %denseids = (); # maps node ids from the file to dense ids
my $nextid = 0;
foreach my $id (sort {$a<=>$b} keys %usedids) {
$denseids{$id} = $nextid;
$nextid++;
}
# All unseen target nodes then get the same next id, the final node id
foreach my $id (keys %usedtgtids) {
next if defined $denseids{$id};
$denseids{$id} = $nextid;
}
foreach my $f (keys %is_final) {
if (defined $outnodes[$f]) {
print STDERR "$inf:Node $f is final but it has outgoing edges!\n";
$err = 1;
}
}
# # Verbose: print original to dense IDs mapping
# foreach my $src (sort {$a<=>$b} keys %denseids) {
# print STDERR "$src ...> $denseids{$src}\n";
# }
print "(";
for(my $origsrc = 0; $origsrc < @outnodes; $origsrc++) {
my $src = $denseids{$origsrc};
next if !defined $src; # this original node ID is not used at all
next if $src == $nextid; # this is the ultimate merged final node
my $outnode = $outnodes[$origsrc];
print "(";
foreach my $arc (@$outnode) {
my $origtgt = $arc->[2];
my $tgt = $denseids{$origtgt};
if (!defined $tgt) {
# this was a final node only
$tgt = $denseids{$origtgt} = $nextid;
$nextid++;
}
my $step_to_target = $tgt - $src;
die "$inf:Bug, I damaged top-sortedness (orig $origsrc .. $origtgt; curr $src .. $tgt)." if $step_to_target <= 0;
print "('".apo($arc->[0])."',$arc->[1],$step_to_target),";
}
print "),";
}
print ")\n";
goto NEXTLATTICE if defined $fh && ! eof($fh);
}
die "There were errors." if $err;
sub apo {
my $s = shift;
# protects apostrophy and backslash
$s =~ s/\\/\\\\/g;
$s =~ s/(['])/\\$1/g;
return $s;
}
sub my_open {
my $f = shift;
if ($f eq "-") {
binmode(STDIN, ":utf8");
return *STDIN;
}
die "Not found: $f" if ! -e $f;
my $opn;
my $hdl;
my $ft = `file '$f'`;
# file might not recognize some files!
if ($f =~ /\.gz$/ || $ft =~ /gzip compressed data/) {
$opn = "zcat '$f' |";
} elsif ($f =~ /\.bz2$/ || $ft =~ /bzip2 compressed data/) {
$opn = "bzcat '$f' |";
} else {
$opn = "$f";
}
open $hdl, $opn or die "Can't open '$opn': $!";
binmode $hdl, ":utf8";
return $hdl;
}
|