Add files using upload-large-folder tool

SCTK/src/acomp/acomp.pl

@@ -0,0 +1,328 @@
+#!/usr/bin/perl -w
+
+my $Version="1.1";
+
+#####
+#  Version 1.0  Released September 9, 1997
+#        - Initial Release
+#  Version 1.1 Released October 24, 1997
+#        - added the -f flag to interpret fragments
+
+
+my $Usage="Usage: acompl.pl [ -t -s -f ] [ -i fmt ] -m min -l lex Infile|- OutFile|-\n".
+    "Version: $Version\n".
+    "Desc: The 'acomp' program expands compound words in German orthography into\n".
+    "      the constituent parts of those compound words.  The program uses the\n".
+    "      LDC German Lexicon as a basis for the expansions.  Input is read from\n".
+    "      InFile or STDIN if infile is '-' and output is written to Outfile or\n".
+    "      STDOUT if OutFile is '-'\n".
+    "Options:\n".
+    "      -m min   Set the minimum length on a compound costituent part to 'min'.\n".
+    "               Default is 2.\n".
+    "      -l lex   File name of the LDC German Dictionary\n".
+    "      -t       Disable the processing of triplet consonants\n".
+    "      -s       Disable the processing of -s and -en insertions\n".
+    "      -f       Divide word fragments as best as you can\n".
+    "      -i fmt   Set the input file formant to 'fmt'.  The possible choices are:\n".
+    "                  txt -> plain text, the default\n".
+    "                  ctm -> CTM format, ignores all but the 5th column, and if\n".
+    "                         a division occurs and a confidence score is present,\n".
+    "                         the confidence score is copied to all parts.\n".
+    "                  stm -> STM format, change only the text field of the stm record\n".
+    "\n";
+$Lex = "";
+$MaxLen = 0;
+%LexHash = ();
+%CompHash = ();
+$MinCompLen = 2;
+$DoTriplet = 1;
+$DoInserts = 1;
+$InFmt = "txt";
+$InFile = "";
+$OutFile = "";
+$Frag = 0;
+
+$| = 1; # This will cause stdout to be flushed whenever we print to it.
+
+use Getopt::Long;
+my $ret = GetOptions ("l=s",
+		      "m=s",
+		      "t",
+		      "s",
+		      "i:s",
+		      "f");
+die "\n$Usage\nError: Failed to parse arguments" if (! $ret);
+
+
+if (defined($opt_l)) {  $Lex = $opt_l; } else { die("$Usage\n\nError: Lexicon required via -l.\n"); }
+if (defined($opt_m)) {  $MinCompLen = $opt_m; }
+if (defined($opt_t)) {  $DoTriplet = 0; $opt_t = 0;}
+if (defined($opt_s)) {  $DoInserts = 0; $opt_s = 0;}
+if (defined($opt_f)) {  $Frag = 1; $opt_f = 0;}
+if (defined($opt_i)) {
+    die("$Usage\n\nError: Undefined input format '$opt_i'") 
+	if ($opt_i !~ /^(txt|ctm|stm)$/);
+    $InFmt = $opt_i;
+}
+#### The main functions arguments:
+if ($#ARGV > 1) { print "\n$Usage\nToo many arguments\n\n"; exit 1; }
+if ($#ARGV == 0) { print "\n$Usage\nOutput Not Specified\n\n"; exit 1; } 
+if ($#ARGV == -1) { print "\n$Usage\nInput and Output Not Specified\n\n";
+		    exit 1; } 
+
+$InFile=$ARGV[0];
+$OutFile=$ARGV[1];
+die("$Usage\nError: Input file $InFile does not exist\n")
+    if ($InFile ne "-" && ! -r $InFile);
+
+##########################   MAIN   #####################D
+
+&LoadLex();
+
+open(IN,"$InFile") || die("Error: Unable to open input file '$InFile'");
+open(OUT,">$OutFile") || die("Error: Unable to open output file '$OutFile'");
+
+while (<IN>){
+    chop;
+    if ($_ =~ /^;;/){
+	print OUT "$_\n";
+    } elsif ($InFmt eq "txt"){
+	print OUT &String_LU($_)."\n";
+    } elsif ($InFmt eq "ctm"){
+	s/^\s+//;
+	local(@ctm);
+	local(@new_words);
+	local($i);	    
+	local($newt);
+	local($conf);
+
+	@ctm = split(/\s+/,$_);
+	if ($#ctm <= 4) { $conf = "" ; } else { $conf = $ctm[5]; }
+
+	$newt = &String_LU($ctm[4]);
+	@new_words = split(/\s+/,$newt);
+	for ($i=0; $i<=$#new_words; $i++){
+	    printf OUT ("%s %s %.2f %.2f %s %s\n",$ctm[0],$ctm[1],
+			$ctm[2] + ($i * ($ctm[3] / ($#new_words+1))),
+			$ctm[3] / ($#new_words + 1), $new_words[$i], $conf);
+	}
+    } elsif ($InFmt eq "stm"){
+	s/^\s+//;
+	local(@sl) = split(/\s+/,$_,7);
+        local($txt);
+	die "Error: Not enough fields in stm line '$_'\n" if ($#sl < 4);
+
+	if ($#sl == 4){
+	    print OUT "$sl[0] $sl[1] $sl[2] $sl[3] $sl[4]\n";
+	} else {
+	    print OUT "$sl[0] $sl[1] $sl[2] $sl[3] $sl[4]";
+	    if ($sl[5] =~ /^<.*>$/){
+		print OUT " $sl[5]";
+		$txt = "";
+		if ($#sl == 6){  $txt = $sl[6];  }
+	    } else {
+		$txt = $sl[5];
+		if ($#sl == 6){  $txt .= " ".$sl[6];  }
+	    }
+	    if ($txt ne ""){
+		print OUT " ".&String_LU($txt);
+	    }
+	    print OUT "\n";
+	}
+    }
+}
+
+close (IN); close(OUT);
+
+#################   END OF MAIN   #########################
+###########################################################
+
+sub String_LU{
+    local($s) = @_;
+    local($new) = "";
+    local($ne);
+    $s =~ tr/a-z/A-Z/;
+    $s =~ tr/\340-\377/\300-\337/; 
+    foreach $word(split(/\s+/,$s)){
+	if ($word =~ /^\(.*\)$/){
+	    $word =~ s/^\((.*)\)$/$1/; 
+	    local($lookup) = &Comp_LU($word);
+	    if ($word !~ /-$/ || (! $Frag)){
+		foreach $ne(split(/\s+/,$lookup)){
+		    $new .= "(".$ne.") ";
+		}
+	    } else {
+		$lookup =~ s/(\S+-)$/aq($1)/; 
+		$new .= "$lookup ";
+	    }
+	} else {
+	    $new .= &Comp_LU($word)." ";
+	}
+    }
+    $new =~ s/ $//;
+    $new;
+}
+
+sub Comp_LU{
+    local($text) = @_;
+    local(@Comps) = ();
+    local($comp);
+    local($db) = 0;
+
+    if ($CompHash{$text} !~ /^$/){
+	print "hashed $text = $CompHash{$text}\n" if ($db);
+	$CompHash{$text};
+    } else {
+	if ($Frag && $text =~ /-$/) {  
+	    print "Fragment Lookup '$text'\n" if ($db);
+	    
+	    local($len) = length($text);
+	    local($htext) = "";
+	    local($nocomp) = 1;
+	    local($i, $j);
+	    for ($i=$len-2; $i>=$MinCompLen && $nocomp; $i--){
+		$htext = substr($text,0,$i);
+		&is_Compound("",$htext,"",*Comps);
+		if ($#Comps != -1){
+		    print "Fragment match @Comps\n" if ($db);
+		    foreach ($j=0; $j <= $#Comps; $j++){
+			$Comps[$j] .= " ".substr($text,$i);
+		    }
+		    $nocomp = 0;
+		}
+	    }	
+	} else {
+	    print "compute $text  " if ($db);
+	    &is_Compound("",$text,"",*Comps);
+	}
+	
+	if ($#Comps == -1) {
+	    $comp = $text;
+	} elsif ($#Comps > 0) {
+	    $comp = "{";
+	    foreach $c (@Comps){
+		$comp .= " $c /";
+	    }
+	    $comp =~ s:/$:}:;
+        } else {
+	    $comp = $Comps[0];
+	}
+        $CompHash{$text} = $comp;
+        print "compute $text = $comp\n" if ($db);
+        $comp;
+    }
+}
+
+sub is_Compound{
+    local($hist,$val,$ind,*comps) = @_;
+    local($sval);
+    local($resid);
+    local($vallen) = length($val);
+    local($srchlen) = $MaxLen;
+    local($db) = 0;
+    local($stop_recur) = 0;
+    local($i);
+    
+    $srchlen = $vallen if ($vallen < $MaxLen);
+
+    printf $ind."is_Compound hist: '$hist' val: '$val' slen: $srchlen\n" if ($db);
+    if ($LexHash{$val} eq "1"){
+	### The word is already in the hashed list, return it.
+	local($co);
+	($co = "$hist $val") =~ s/^ //;
+	print "$ind*******  SUCCESS Search value pre-defined '$co'\n" if ($db);
+	push(@comps,$co);
+	1;
+    } else {
+	### loop through all of the characters in a 
+	for ($i=$srchlen; $i>=$MinCompLen && ! $stop_recur; $i--){
+	    $sval = substr($val,0,$i);
+	    print $ind."   $sval\n" if ($db);
+	    if ($LexHash{$sval} eq "1"){
+		$resid = substr($val,$i);
+		print $ind."      Possible subword '$sval' residual '$resid'\n" if ($db);
+		if (length($resid) >= $MinCompLen){
+		    if (&is_Compound($hist." $sval",$resid, $ind."   |",*comps) == 1){
+			$stop_recur = 1;
+			print "$ind**** Curtailing search for normal lookup\n" if ($db);
+		    } else {
+			print "$ind**** Failed to Curtail search\n" if ($db);
+		    }
+		} else {
+		    print "$ind**** Residual length < MinCompLen $MinCompLen\n" if ($db);
+		}
+		#### handle the geminate constant conditional
+		if (! $stop_recur && $DoTriplet && $sval =~ /([bcdfghjklmnpqrstvwxyz])\1$/){
+		    $resid = substr($val,$i-1);
+		    print $ind."      Possible geminate consonate subword residual $resid\n" if ($db);
+		    if (length($resid) >= $MinCompLen){
+			if (&is_Compound($hist." $sval",$resid, $ind."   |",*comps) == 1){
+			    $stop_recur = 1;
+			    print "$ind**** Curtailing search for triplet consonants\n" if ($db);
+			} else {
+			    print "$ind**** Failed to Curtail search\n" if ($db);
+			}
+		    }		    
+		}
+		#### handle the insertion of syllables
+		if (! $stop_recur && $DoInserts && ( $resid =~ /^([sS]|[eE][nN])/ ) ){
+		    local($gen);
+		    if ($resid =~ /^[sS]/){  $resid = substr($val,$i+1); $gen = "s" } 
+		    if ($resid =~ /^[eE][nN]/){  $resid = substr($val,$i+2); $gen = "en" } 
+		    print $ind."      Possible additional '$gen' on residual '$gen'$resid\n" if ($db);
+		    if (length($resid) >= $MinCompLen){
+			if (&is_Compound($hist." $sval",$resid, $ind."   |",*comps) == 1){
+			    $stop_recur = 1;
+			    print "$ind**** Curtailing search for insertion of syllables\n" if ($db);
+			} else {
+			    print "$ind**** Failed to Curtail search\n" if ($db);
+			}
+		    }		    
+		}
+	    }
+	}
+	$stop_recur;
+    }
+}
+
+sub LoadLex{
+    local($len);
+    local($db) = 0;
+    open(LEX,$Lex) || die("Unable to open Lexicon '$Lex'");
+
+    print "Reading Lexicon: '$Lex'\n   Each . is 10000 entries:   " if ($db);
+    while (<LEX>){
+	chop;
+	($word) = split;
+	$word =~ tr/a-z/A-Z/;
+	$word =~ tr/\340-\377/\300-\337/; 
+
+	if ($word =~ /[-_]/){
+	    ($sep = $word) =~ s/[-_]+/ /g;
+	    ($cmp = $word) =~ s/_//g;
+	    foreach $sw(split(/\s+/,$sep)){
+		$LexHash{$sw} = "1";
+		$len = length($sw);
+		$MaxLen = $len if ($MaxLen < $len);
+	    }
+	    $CompHash{$cmp} = $sep;
+	    if ($cmp =~ /-/){
+		$cmp =~ s/-//g;
+		$CompHash{$cmp} = $sep;
+	    }
+	} else {
+	    $LexHash{$word} = "1";
+	    $CompHash{$word} = $word;
+	}
+	if (($. + 1) % 10000 == 0){
+	    print "." if ($db);
+	}
+    }
+    print "\n   $. entries loaded, Largest word is $MaxLen\n" if ($db);
+
+    close(LEX);
+    
+#    print "LexHash:\n"; foreach $key(sort(keys %LexHash)) { print "   $key -> $LexHash{$key}\n"; }
+#    print "CompHash:\n"; foreach $key(sort(keys %CompHash)) { print "   $key -> $CompHash{$key}\n"; }
+}
+

SCTK/src/acomp/makefile

@@ -0,0 +1,27 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+T=../test_suite
+PERL=/slp/sware/Linux/bin/perl
+
+all: acomp.pl
+
+acomp.pl: 
+	chmod +x acomp.pl
+
+setTests:
+
+check: all
+	@echo "Testing acomp.pl"
+	@echo "   No tests defined for acomp.pl"
+
+install: all
+	cp acomp.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/acomp/makefile.in

@@ -0,0 +1,27 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=
+bindir = $(PREFIX)/bin
+T=../test_suite
+PERL=/slp/sware/Linux/bin/perl
+
+all: acomp.pl
+
+acomp.pl: 
+	chmod +x acomp.pl
+
+setTests:
+
+check: all
+	@echo "Testing acomp.pl"
+	@echo "   No tests defined for acomp.pl"
+
+install: all
+	cp acomp.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/align2html/Segment.pm

@@ -0,0 +1,325 @@
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+package Segment;
+use strict;
+use Data::Dumper;
+
+use Token;
+
+sub new
+{
+    my $class = shift;
+    my $self = {};
+
+	$self->{ID} = shift;
+	$self->{SPKRID} = shift;
+	$self->{TOKENS} = {};
+	$self->{MINX} = 10000000;
+	$self->{MAXX} = 0;
+	$self->{HEIGHT} = 0;
+	$self->{HASFAKETIME} = 0;
+	
+    bless $self;
+    return $self;
+}
+
+sub AddToken
+{
+	my ($self, $tkn) = @_;
+	$self->{TOKENS}{$tkn->{ID}} = $tkn;
+}
+
+sub SetY
+{
+	my ($self, $ypos) = @_;
+		
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		$self->{TOKENS}{$tknid}->SetYPos($ypos);
+	}
+}
+
+sub isTokenInSegment
+{
+	my ($self, $tokenid) = @_;
+	
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		return 1 if($tknid == $tokenid);
+	}
+	
+	return 0;
+}
+
+sub CalculateMinMaxX
+{
+	my ($self) = @_;
+	
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		next if( ($self->{TOKENS}{$tknid}->{XSTARTPOS} == 0) && ($self->{TOKENS}{$tknid}->{XENDPOS} == 0) && ($self->{TOKENS}{$tknid}->{TEXT} eq "") );
+		$self->{MINX} = $self->{TOKENS}{$tknid}->{XSTARTPOS} if($self->{MINX} > $self->{TOKENS}{$tknid}->{XSTARTPOS});
+		$self->{MAXX} = $self->{TOKENS}{$tknid}->{XENDPOS}   if($self->{MAXX} < $self->{TOKENS}{$tknid}->{XENDPOS});
+	}
+}
+
+sub HasOnlyOneFakeToken
+{
+    my ($self) = @_;
+    
+    foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		return 0 if( !( ($self->{TOKENS}{$tknid}->{XSTARTPOS} == 0) && ($self->{TOKENS}{$tknid}->{XENDPOS} == 0) && ($self->{TOKENS}{$tknid}->{TEXT} eq "") ) );
+	}
+	
+	return 1;
+}
+
+sub CleanSegmentPrevNext
+{
+	my ($self) = @_;
+	
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		my $myToken = $self->{TOKENS}{$tknid};
+		
+		for(my $i=0; $i< @{ $myToken->{PREVTKNID} }; $i++)
+		{
+			delete $myToken->{PREVTKNID}[$i] if(!$self->isTokenInSegment($myToken->{PREVTKNID}[$i]));
+		}
+		
+		for(my $i=0; $i< @{ $myToken->{NEXTTKNID} }; $i++)
+		{
+			delete $myToken->{NEXTTKNID}[$i] if(!$self->isTokenInSegment($myToken->{NEXTTKNID}[$i]));
+		}
+		
+		$myToken->{NUMPREV} = scalar( @{ $myToken->{PREVTKNID} } );
+		$myToken->{NUMNEXT} = scalar( @{ $myToken->{NEXTTKNID} } );
+	}
+}
+
+sub MultiGraphXY
+{
+	my ($self) = @_;
+	
+	my $maxheigh = 0;
+	
+	if(scalar(keys %{ $self->{TOKENS} }) != 0)
+	{
+		my %tokenPositions;
+		my @listTokenID;
+		my $currentlevel = 0;
+		my $maxlengh = 0;
+	
+		foreach my $tknid(sort {$a <=> $b} keys %{ $self->{TOKENS} })
+		{
+			$tokenPositions{$tknid}{XB} = 0;
+			$tokenPositions{$tknid}{XE} = 1;
+			$tokenPositions{$tknid}{Y} = 0;
+			$tokenPositions{$tknid}{TEXT} = $self->{TOKENS}{$tknid}->{TEXT};
+			
+			push(@listTokenID, $tknid);
+			
+			if( scalar(@{ $self->{TOKENS}{$tknid}->{PREVTKNID} }) == 0 )
+			{
+				$tokenPositions{$tknid}{Y} = $currentlevel;
+				$currentlevel++;
+			}
+		}
+			
+		for(my $i=0; $i<@listTokenID; $i++)
+		{
+			my $tokenid = $listTokenID[$i];
+			my $token = $self->{TOKENS}{$tokenid};
+			my @prevtkn = @{ $token->{PREVTKNID} };
+			my @nexttkn = @{ $token->{NEXTTKNID} };
+			
+			my $tokenBegin = $tokenPositions{$tokenid}{XB};
+			my $tokenEnd = $tokenPositions{$tokenid}{XE};
+			my $tokenY = $tokenPositions{$tokenid}{Y};
+			
+			for(my $j=0; $j<@nexttkn; $j++)
+			{
+				my $nexttokenid = $nexttkn[$j];
+				
+				my $nexttokenBegin = $tokenPositions{$nexttokenid}{XB};
+				my $nexttokenEnd = $tokenPositions{$nexttokenid}{XE};
+				my $nextTokenY = $tokenPositions{$nexttokenid}{Y};
+				
+				if( $nexttokenBegin < $tokenEnd)
+				{
+					$tokenPositions{$nexttokenid}{XB} = $tokenEnd;
+					$tokenPositions{$nexttokenid}{XE} = $tokenPositions{$nexttokenid}{XB} + 1;
+					$i = -1;
+				}
+				
+				my $nbrprev = scalar( @{ $self->{TOKENS}{$nexttokenid}->{PREVTKNID} } ) - 1;
+				
+				if($nextTokenY < ($tokenY + $j - $nbrprev))
+				{
+					$tokenPositions{$nexttokenid}{Y} = $tokenY + $j - $nbrprev;
+					$i = -1;
+				}
+			}
+			
+			for(my $j=0; $j<@prevtkn; $j++)
+			{
+				my $prevtokenid = $prevtkn[$j];
+				
+				my $prevtokenBegin = $tokenPositions{$prevtokenid}{XB};
+				my $prevtokenEnd = $tokenPositions{$prevtokenid}{XE};
+				
+				if( $prevtokenEnd != $tokenBegin)
+				{
+					$tokenPositions{$prevtokenid}{XE} = $tokenBegin;
+					$i = -1;
+				}
+			}
+		}
+		
+		foreach my $tken (sort {$a <=> $b} keys %tokenPositions)
+		{
+			$maxheigh = $tokenPositions{$tken}{Y} if($tokenPositions{$tken}{Y} > $maxheigh);
+			$maxlengh = $tokenPositions{$tken}{XE} if($tokenPositions{$tken}{XE} > $maxlengh);
+		}
+		
+		foreach my $tknid(sort keys %{ $self->{TOKENS} })
+		{
+			my $updown = 1;
+			my $base = 0;
+			
+			if($self->{TOKENS}{$tknid}->{REFORSYS} eq "REF")
+			{
+				$updown = -1;
+				$base = $maxheigh;
+			}
+		
+			if($self->{TOKENS}{$tknid}->{REALTIME} == 0)
+			{
+				$self->{TOKENS}{$tknid}->MakeFakeTime($tokenPositions{$tknid}{XB}, $tokenPositions{$tknid}{XE}, $maxlengh);
+				$self->{HASFAKETIME} = 1;
+			}
+			
+			$self->{TOKENS}{$tknid}->{YPOS} += $base*30 + $updown*30*$tokenPositions{$tknid}{Y};
+		}
+	}
+	
+	$self->{HEIGHT} = ($maxheigh+1)*30;
+}
+
+sub Compute
+{
+	my ($self) = @_;
+	
+	# Calculate Min and Max X
+	$self->CalculateMinMaxX();
+	
+	# Clean up the previous and Next of Tokens
+	$self->CleanSegmentPrevNext();
+
+	# Assign X and Y for graph
+	$self->MultiGraphXY();	
+}
+
+sub GetDraw
+{
+	my ($self, $minx) = @_;
+	my $output2 = "";
+	
+	my @listfirsts;
+	my @listlasts;
+	
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{
+		my @prevtkn = @{ $self->{TOKENS}{$tknid}->{PREVTKNID} };
+		my @nexttkn = @{ $self->{TOKENS}{$tknid}->{NEXTTKNID} };
+		
+		if(scalar(@prevtkn) > 1)
+		{
+			for(my $i=1; $i<@prevtkn; $i++)
+			{
+				my $end1x = $self->{TOKENS}{$prevtkn[$i-1]}->{XENDPOS} - $minx;
+				my $end1y = $self->{TOKENS}{$prevtkn[$i-1]}->{YPOS};
+				my $end2x = $self->{TOKENS}{$prevtkn[$i]}->{XENDPOS} - $minx;
+				my $end2y = $self->{TOKENS}{$prevtkn[$i]}->{YPOS};
+				
+				$output2 .= sprintf("jg.drawLine(scale*%.0f, %.0f, scale*%.0f, %.0f);\n", $end1x, $end1y, $end2x, $end2y);
+			}
+		}
+		
+		push(@listfirsts, $tknid) if(scalar(@prevtkn) == 0);
+		
+		if(scalar(@nexttkn) > 1)
+		{
+			for(my $i=1; $i<@nexttkn; $i++)
+			{
+				my $begin1x = $self->{TOKENS}{$nexttkn[$i-1]}->{XSTARTPOS} - $minx;
+				my $begin1y = $self->{TOKENS}{$nexttkn[$i-1]}->{YPOS};
+				my $begin2x = $self->{TOKENS}{$nexttkn[$i]}->{XSTARTPOS} - $minx;
+				my $begin2y = $self->{TOKENS}{$nexttkn[$i]}->{YPOS};
+				
+				$output2 .= sprintf("jg.drawLine(scale*%.0f, %.0f, scale*%.0f, %.0f);\n", $begin1x, $begin1y, $begin2x, $begin2y);
+			}			
+		}
+		
+		push(@listlasts, $tknid) if(scalar(@nexttkn) == 0);
+	}
+	
+	if(scalar(@listfirsts) > 1)
+	{
+		for(my $i=1; $i<@listfirsts; $i++)
+		{
+			my $begin1x = $self->{TOKENS}{$listfirsts[$i-1]}->{XSTARTPOS} - $minx;
+			my $begin1y = $self->{TOKENS}{$listfirsts[$i-1]}->{YPOS};
+			my $begin2x = $self->{TOKENS}{$listfirsts[$i]}->{XSTARTPOS} - $minx;
+			my $begin2y = $self->{TOKENS}{$listfirsts[$i]}->{YPOS};
+			
+			$output2 .= sprintf("jg.drawLine(scale*%.0f, %.0f, scale*%.0f, %.0f);\n", $begin1x, $begin1y, $begin2x, $begin2y);
+		}			
+	}
+	
+	if(scalar(@listlasts) > 1)
+	{
+		for(my $i=1; $i<@listlasts; $i++)
+		{
+			my $end1x = $self->{TOKENS}{$listlasts[$i-1]}->{XENDPOS} - $minx;
+			my $end1y = $self->{TOKENS}{$listlasts[$i-1]}->{YPOS};
+			my $end2x = $self->{TOKENS}{$listlasts[$i]}->{XENDPOS} - $minx;
+			my $end2y = $self->{TOKENS}{$listlasts[$i]}->{YPOS};
+			
+			$output2 .= sprintf("jg.drawLine(scale*%.0f, %.0f, scale*%.0f, %.0f);\n", $end1x, $end1y, $end2x, $end2y);
+		}			
+	}
+	
+	my $output = "";
+	
+	if($output2 ne "")
+	{
+		$output .= sprintf("jg.setColor(\"grey\");\n");
+		$output .= sprintf("jg.setStroke(Stroke.DOTTED);\n");
+		$output .= $output2;
+	}
+	
+	foreach my $tknid(sort keys %{ $self->{TOKENS} })
+	{		
+		$output .= $self->{TOKENS}{$tknid}->GetDraw($minx);
+	}
+	
+	return $output;
+}
+
+1;
+

SCTK/src/align2html/SegmentGroup.pm

@@ -0,0 +1,407 @@
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+package SegmentGroup;
+use strict;
+use Data::Dumper;
+
+sub new
+{
+    my $class = shift;
+    my $self = {};
+
+	$self->{ID} = shift;
+    $self->{REF} = {};
+    $self->{SYS} = {};
+    $self->{STARTYPOS} = 25;
+    $self->{YPOSSTEP} = 20;
+    $self->{MINX} = 10000000;
+	$self->{MAXX} = 0;
+	$self->{HEIGHT} = 0;
+	$self->{SPKRHEIGHT} = {};
+	$self->{HASFAKETIME} = 0;
+	$self->{SPKRYS} = {};
+	$self->{FILE} = shift;
+	$self->{CHANNEL} = shift;
+	$self->{NCORR} = 0;
+	$self->{NINS} = 0;
+	$self->{NSUB} = 0;
+	$self->{NDEL} = 0;
+	$self->{NREF} = 0;
+	$self->{NSPKRERR} = 0;
+	$self->{ET} = 0;
+	$self->{BT} = 0;
+	$self->{ALIGNED} = 0;
+	
+    bless $self;    
+    return $self;
+}
+
+sub unique
+{
+	my %saw;
+	return grep(!$saw{$_}++, sort @_);
+}
+
+sub addRefSegment
+{
+	my ($self, $seg) = @_;
+	$self->{REF}{$seg->{ID}} = $seg;
+}
+
+sub addSysSegment
+{
+	my ($self, $seg) = @_;
+	$self->{SYS}{$seg->{ID}} = $seg;
+}
+
+sub ContainsFakeTimes
+{
+	my ($self) = @_;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		$self->{HASFAKETIME} = 1 if($self->{REF}{$segrefid}->{HASFAKETIME} == 1);
+		return;
+	}
+		
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		$self->{HASFAKETIME} = 1 if($self->{SYS}{$segsysid}->{HASFAKETIME} == 1);
+		return;
+	}
+}
+
+sub GetNumHypWords
+{
+	my ($self) = @_;
+	
+	my $number = 0;
+	foreach my $segsysid(keys %{ $self->{SYS} }) { $number += scalar keys %{ $self->{SYS}{$segsysid}->{TOKENS} } if(!($self->{SYS}{$segsysid}->HasOnlyOneFakeToken())); }
+	return $number;
+}
+
+sub GetToken
+{
+	my ($self, $tokenID) = @_;
+	
+	foreach my $segrefid(keys %{ $self->{REF} })
+	{
+		return $self->{REF}{$segrefid}->{TOKENS}{$tokenID} if(exists($self->{REF}{$segrefid}->{TOKENS}{$tokenID}));
+	}
+		
+	foreach my $segsysid(keys %{ $self->{SYS} })
+	{
+		return $self->{SYS}{$segsysid}->{TOKENS}{$tokenID} if(exists($self->{SYS}{$segsysid}->{TOKENS}{$tokenID}));
+	}
+	
+	return undef;
+}
+
+sub CalculateBeginEndTime
+{
+	my ($self) = @_;
+	
+	$self->{ET} = 0;
+	$self->{BT} = 1000000;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		foreach my $tknid(sort keys %{ $self->{REF}{$segrefid}->{TOKENS} })
+		{
+			next if( ($self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGBT} == 0) && ($self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGET} == 0) && ($self->{REF}{$segrefid}->{TOKENS}{$tknid}->{TEXT} eq "") );
+			$self->{BT} = $self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGBT} if($self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGBT} < $self->{BT});
+			$self->{ET} = $self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGET} if($self->{REF}{$segrefid}->{TOKENS}{$tknid}->{SEGET} > $self->{ET});
+		}
+	}
+	
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		foreach my $tknid(sort keys %{ $self->{SYS}{$segsysid}->{TOKENS} })
+		{
+			next if( ($self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} == 0) && ($self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} + $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{DUR} == 0) && ($self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{TEXT} eq "") );
+			$self->{BT} = $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} if($self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} < $self->{BT});
+			$self->{ET} = sprintf("%.3f", $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} + $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{DUR}) if(sprintf("%.3f", $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{BT} + $self->{SYS}{$segsysid}->{TOKENS}{$tknid}->{DUR}) > $self->{ET});
+		}
+	}
+}
+
+sub CalculateMinMaxX
+{
+	my ($self) = @_;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		$self->{MINX} = $self->{REF}{$segrefid}->{MINX} if($self->{MINX} > $self->{REF}{$segrefid}->{MINX});
+		$self->{MAXX} = $self->{REF}{$segrefid}->{MAXX} if($self->{MAXX} < $self->{REF}{$segrefid}->{MAXX});
+	}
+		
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		$self->{MINX} = $self->{SYS}{$segsysid}->{MINX} if($self->{MINX} > $self->{SYS}{$segsysid}->{MINX});
+		$self->{MAXX} = $self->{SYS}{$segsysid}->{MAXX} if($self->{MAXX} < $self->{SYS}{$segsysid}->{MAXX});
+	}
+}
+
+sub CalculateHeigh
+{
+	my ($self) = @_;
+	
+	my %spkrYpos;
+	
+	$self->{HEIGHT} = 2 * $self->{STARTYPOS} + $self->{YPOSSTEP};
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		my $spkrID = $self->{REF}{$segrefid}->{SPKRID};
+
+		if(!exists($spkrYpos{REF}{$spkrID}))
+		{
+			$spkrYpos{REF}{$spkrID} = 1;
+			$self->{SPKRHEIGHT}{$spkrID} = $self->{REF}{$segrefid}->{HEIGHT};
+		}
+		else
+		{
+			$self->{SPKRHEIGHT}{$spkrID} = $self->{REF}{$segrefid}->{HEIGHT} if($self->{REF}{$segrefid}->{HEIGHT} > $self->{SPKRHEIGHT}{$spkrID});
+		}
+	}
+		
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		my $spkrID = $self->{SYS}{$segsysid}->{SPKRID};
+		
+		if(!exists($spkrYpos{SYS}{$spkrID}))
+		{
+			$spkrYpos{SYS}{$spkrID} = 1;
+			$self->{SPKRHEIGHT}{$spkrID} = $self->{SYS}{$segsysid}->{HEIGHT};
+		}
+		else
+		{
+			$self->{SPKRHEIGHT}{$spkrID} = $self->{SYS}{$segsysid}->{HEIGHT} if($self->{SYS}{$segsysid}->{HEIGHT} > $self->{SPKRHEIGHT}{$spkrID});
+		}
+	}
+	
+	foreach my $idd (keys %{ $self->{SPKRHEIGHT} } )
+	{
+		$self->{HEIGHT} += $self->{SPKRHEIGHT}{$idd} + $self->{YPOSSTEP};
+	}
+	
+	return $self->{HEIGHT};
+}
+
+sub SetYs
+{
+	my ($self) = @_;
+	
+	my $output = "";
+	my $currYpos = $self->{STARTYPOS};
+	my $YposStep = $self->{YPOSSTEP};
+	
+	my %spkrYpos;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		my $spkrID = $self->{REF}{$segrefid}->{SPKRID};		
+		my $Ypos;
+		
+		if(!exists($spkrYpos{REF}{$spkrID}))
+		{
+			$Ypos = $currYpos;
+			$spkrYpos{REF}{$spkrID} = $Ypos;
+			$currYpos += $self->{SPKRHEIGHT}{$spkrID} + $YposStep;
+		}
+		else
+		{
+			$Ypos = $spkrYpos{REF}{$spkrID};
+		}
+		
+		$self->{REF}{$segrefid}->SetY($Ypos);
+		$self->{SPKRYS}{$spkrID} = $Ypos;
+	}
+	
+	$currYpos += 2*$self->{YPOSSTEP};
+	
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		my $spkrID = $self->{SYS}{$segsysid}->{SPKRID};		
+		my $Ypos;
+		
+		if(!exists($spkrYpos{SYS}{$spkrID}))
+		{
+			$Ypos = $currYpos;
+			$spkrYpos{SYS}{$spkrID} = $Ypos;
+			$currYpos += $self->{SPKRHEIGHT}{$spkrID} + $YposStep;
+		}
+		else
+		{
+			$Ypos = $spkrYpos{SYS}{$spkrID};
+		}
+		
+		$self->{SYS}{$segsysid}->SetY($Ypos);
+		$self->{SPKRYS}{$spkrID} = $Ypos;
+	}
+}
+
+sub GetFillREFHYP
+{
+	my ($self, $maxscalewidth) = @_;	
+	my $output = "";
+	my @valuessep1;
+	
+	foreach my $spkrID(sort keys %{ $self->{SPKRYS} })
+	{
+		push(@valuessep1, $self->{SPKRYS}{$spkrID} - 25);
+		push(@valuessep1, $self->{SPKRYS}{$spkrID} + $self->{SPKRHEIGHT}{$spkrID});
+	}
+		
+	my @valuessep = sort {$a <=> $b} @valuessep1;
+	
+	my @refspkrs;
+	foreach my $segref (keys %{ $self->{REF} }) { push(@refspkrs, $self->{REF}{$segref}->{SPKRID}); }
+	my $nrefspkr = scalar unique @refspkrs;
+	
+	my @sysspkrs;
+	foreach my $segsys (keys %{ $self->{SYS} }) { push(@sysspkrs, $self->{SYS}{$segsys}->{SPKRID}); }
+	my $nsysspkr = scalar unique @sysspkrs;
+	
+	if($nrefspkr == 0)
+	{
+		my $y1 = $valuessep[0];
+		my $y2 = $valuessep[scalar(@valuessep) - 1];
+		$output .= sprintf("jg.setColor(\"#ffeeee\");\n");
+		$output .= sprintf("jg.fillRect(0, %.0f, scale*%.0f - 1, %.0f);\n", $y1 + 1, $maxscalewidth, $y2 - $y1 - 1);
+	}
+	elsif($nsysspkr == 0)
+	{
+		my $y1 = $valuessep[0];
+		my $y2 = $valuessep[scalar(@valuessep) - 1];
+		$output .= sprintf("jg.setColor(\"#eeffee\");\n");
+		$output .= sprintf("jg.fillRect(0, %.0f, scale*%.0f - 1, %.0f);\n", $y1 + 1, $maxscalewidth, $y2 - $y1 - 1);
+	}
+	else
+	{
+		my $y1 = $valuessep[0];
+		my $ym = ($valuessep[2*$nrefspkr - 1] + $valuessep[2*$nrefspkr])/2;
+		my $y2 = $valuessep[scalar(@valuessep) - 1];
+		$output .= sprintf("jg.setColor(\"#eeffee\");\n");
+		$output .= sprintf("jg.fillRect(0, %.0f, scale*%.0f - 1, %.0f);\n", $y1 + 1, $maxscalewidth, $ym - $y1 - 1);
+		$output .= sprintf("jg.setColor(\"#ffeeee\");\n");
+		$output .= sprintf("jg.fillRect(0, %.0f, scale*%.0f - 1, %.0f);\n", $ym - $y1 + 1, $maxscalewidth, $y2 - $ym -1);
+	}
+
+	return $output;
+}
+
+sub GetSeparationLines
+{
+	my ($self, $maxscalewidth) = @_;
+	my $output = "";
+	
+	$output .= sprintf("jg.setColor(\"black\");\n");
+	$output .= sprintf("jg.setFont(\"verdana\",\"12px\",Font.PLAIN);\n");
+	
+	foreach my $spkrID(sort keys %{ $self->{SPKRYS} })
+	{
+		my $spkName = $spkrID;
+		$spkName =~ s/^ref://;
+		$spkName =~ s/^hyp://;
+		$output .= sprintf("jg.drawStringRect(\"%s\",%.0f, %.0f, scale*%.0f, \"left\");\n", $spkName, 0, $self->{SPKRYS}{$spkrID} - 8 + ($self->{SPKRHEIGHT}{$spkrID}-30)/2, $maxscalewidth);
+	}
+	
+	$output .= sprintf("jg.setColor(\"yellow\");\n");
+	$output .= sprintf("jg.setStroke(1);\n");
+	
+	my @valuessep1;
+	
+	foreach my $spkrID(sort keys %{ $self->{SPKRYS} })
+	{
+		push(@valuessep1, $self->{SPKRYS}{$spkrID} - 25);
+		push(@valuessep1, $self->{SPKRYS}{$spkrID} + $self->{SPKRHEIGHT}{$spkrID});
+	}
+	
+	my @valuessep = sort {$a <=> $b} @valuessep1;
+		
+	$output .= sprintf("jg.drawLine(0, %.0f, scale*%.0f - 1, %.0f);\n", $valuessep[0], $maxscalewidth, $valuessep[0]);
+	
+	for(my $i=1; $i<scalar(@valuessep)-1; $i += 2)
+	{
+		my $ypostmp = ($valuessep[$i] + $valuessep[$i + 1])/2;
+		
+		$output .= sprintf("jg.drawLine(0, %.0f, scale*%.0f - 1, %.0f);\n", $ypostmp, $maxscalewidth, $ypostmp);
+	}
+	
+	$output .= sprintf("jg.drawLine(0, %.0f, scale*%.0f - 1, %.0f);\n", $valuessep[scalar(@valuessep) - 1], $maxscalewidth, $valuessep[scalar(@valuessep) - 1]);
+	
+	return $output;
+}
+
+sub Compute
+{
+	my ($self) = @_;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		$self->{REF}{$segrefid}->Compute();
+	}
+	
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		$self->{SYS}{$segsysid}->Compute();
+	}
+	
+	$self->CalculateMinMaxX();
+	$self->CalculateHeigh();
+	$self->ContainsFakeTimes();
+	$self->SetYs();
+	$self->CalculateBeginEndTime();
+}
+
+sub isInSpeakers
+{
+	my ($self, $spkr) = @_;
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		return 1 if($self->{REF}{$segrefid}->{SPKRID} =~ /^$spkr$/i);
+	}
+		
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		return 1 if($self->{SYS}{$segsysid}->{SPKRID} =~ /^$spkr$/i);
+	}
+	
+	return 0;
+}
+
+sub GetDraw
+{
+	my ($self, $decalx) = @_;
+	my $output = "";
+	
+	foreach my $segrefid(sort keys %{ $self->{REF} })
+	{
+		$output .= $self->{REF}{$segrefid}->GetDraw($self->{MINX} - $decalx);		
+	}
+		
+	foreach my $segsysid(sort keys %{ $self->{SYS} })
+	{
+		$output .= $self->{SYS}{$segsysid}->GetDraw($self->{MINX} - $decalx);	
+	}
+	
+	return $output;
+}
+
+1;
+

SCTK/src/align2html/Token.pm

@@ -0,0 +1,166 @@
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+package Token;
+use strict;
+use Data::Dumper;
+
+sub new
+{
+    my $class = shift;
+    my $self = {};
+
+	$self->{ID} = shift;
+	$self->{BT} = shift;
+	$self->{DUR} = shift;
+	$self->{MID} = 0;
+	$self->{TEXT} = shift;
+	$self->{LENGTH} = 0;
+	$self->{XSTARTPOS} = 0;
+	$self->{XENDPOS} = 0;
+	$self->{YPOS} = 0;
+	$self->{XMIDPOS} = 0;
+	$self->{TEXTPOS} = "top";
+	$self->{WIDTHLINE} = 1;
+	$self->{PIXPERSEC} = 100;
+	$self->{REFORSYS} = shift;
+	$self->{PREVTKNID} = [()];
+	$self->{NEXTTKNID} = [()];
+	$self->{DISPLAY} = 0;
+	$self->{NUMPREV} = 0;
+	$self->{NUMNEXT} = 0;
+	$self->{SEGGRPID} = shift;
+	$self->{SEGID} = shift;
+	$self->{SPKRID} = shift;
+	$self->{CONF} = shift;
+	$self->{REALTIME} = 1;	
+	
+	my $tmpprev = shift;
+	my $tmpnext = shift;
+	
+	$self->{SEGBT} = shift;
+	$self->{SEGET} = shift;
+	
+	bless $self;
+	
+	$self->{REALTIME} = 0 if( ($self->{BT} eq "") || ($self->{DUR} eq "") );
+	
+	if($self->{REALTIME} == 0)
+	{
+		$self->{BT} = $self->{SEGBT};
+		$self->{DUR} = sprintf("%.3f", $self->{SEGET}-$self->{SEGBT});
+	}
+	
+    $self->{MID} = sprintf("%.0f", $self->{BT} + $self->{DUR}/2);
+    $self->{XSTARTPOS} = sprintf("%.0f", $self->{PIXPERSEC}*$self->{BT});
+	$self->{XMIDPOS} = sprintf("%.0f", $self->{PIXPERSEC}*($self->{BT}+$self->{DUR}/2));
+	$self->{XENDPOS} = sprintf("%.0f", $self->{PIXPERSEC}*($self->{BT}+$self->{DUR}));
+	$self->{LENGTH} = sprintf("%.0f", $self->{PIXPERSEC}*$self->{DUR});
+	$self->{TEXTPOS} = "bottom" if($self->{REFORSYS} eq "SYS");
+	
+	push(@{ $self->{PREVTKNID} }, sort {$a <=> $b} split(/\|/, $tmpprev) );
+	push(@{ $self->{NEXTTKNID} }, sort {$a <=> $b} split(/\|/, $tmpnext) );
+	
+	$self->{NUMPREV} = scalar( @{ $self->{PREVTKNID} } );
+	$self->{NUMNEXT} = scalar( @{ $self->{NEXTTKNID} } );
+	
+    return $self;
+}
+
+sub MakeFakeTime
+{
+	my ($self, $beginindex, $endindex, $maxind) = @_;
+	
+	$self->{DUR} = sprintf("%.3f", ($self->{SEGET} - $self->{SEGBT})*( ($endindex - $beginindex)/$maxind ) );
+	$self->{BT} = sprintf("%.3f", $self->{SEGBT} + ($self->{SEGET} - $self->{SEGBT})*$beginindex/$maxind);
+	$self->{MID} = sprintf("%.0f", $self->{BT} + $self->{DUR}/2);
+    $self->{XSTARTPOS} = sprintf("%.0f", $self->{PIXPERSEC}*$self->{BT});
+	$self->{XMIDPOS} = sprintf("%.0f", $self->{PIXPERSEC}*($self->{BT}+$self->{DUR}/2));
+	$self->{XENDPOS} = sprintf("%.0f", $self->{PIXPERSEC}*($self->{BT}+$self->{DUR}));
+	$self->{LENGTH} = sprintf("%.0f", $self->{PIXPERSEC}*$self->{DUR});
+}
+
+sub DoDisplay
+{
+	my ($self) = @_;
+	$self->{DISPLAY} = 1;
+}
+
+sub AddPrevToken
+{
+	my ($self, $prevtknid) = @_;
+	push(@{ $self->{PREVTKNID} }, $prevtknid);
+}
+
+sub AddNextToken
+{
+	my ($self, $nexttknid) = @_;
+	push(@{ $self->{NEXTTKNID} }, $nexttknid);
+}
+
+sub SetYPos
+{
+	my ($self, $ypos) = @_;
+	$self->{YPOS} += $ypos;
+}
+
+
+sub SetWidthLine
+{
+	my ($self, $lwidth) = @_;
+	$self->{WIDTHLINE} = $lwidth;
+}
+
+sub IsFakeToken
+{
+    my ($self) = @_;
+    return( ($self->{SEGBT} == 0) && ($self->{SEGET} == 0) && ($self->{TEXT} eq "") );
+}
+
+sub GetDraw
+{
+	my ($self, $minx) = @_;
+	my $output = "";
+
+	return $output if(! $self->{DISPLAY});
+	
+	if($self->{TEXT} ne "")
+	{
+		my $modiftexty = ($self->{TEXTPOS} eq "top" ? -17 : 6);
+		my $bullet = "images/white_dk.gif";
+		$bullet = "images/white_lt.gif" if($self->{REALTIME} == 0);
+		
+		$output .= sprintf("jg.setColor(\"DimGray\");\n");
+		$output .= sprintf("jg.setStroke(%.0f);\n", $self->{WIDTHLINE});
+		$output .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $self->{XSTARTPOS} - $minx, $self->{YPOS}, $self->{XENDPOS} - $minx, $self->{YPOS});
+		$output .= sprintf("jg.setFont(\"arial\",\"11px\",Font.PLAIN);\n");
+		$output .= sprintf("jg.drawStringRect(\"%s\", %.0f*scale + 1, %.0f, %.0f*scale - 1, \"center\");\n", $self->{TEXT}, $self->{XSTARTPOS} - $minx, $self->{YPOS}+$modiftexty, $self->{LENGTH});
+		$output .= sprintf("jg.drawImage(\"%s\", %.0f*scale - 7, %.0f - 7, 14, 14);\n", $bullet, $self->{XSTARTPOS} - $minx, $self->{YPOS});
+		$output .= sprintf("jg.drawImage(\"%s\", %.0f*scale - 7, %.0f - 7, 14, 14);\n", $bullet, $self->{XENDPOS} - $minx, $self->{YPOS});
+	}
+	elsif(!( ($self->{SEGBT} == 0) && ($self->{SEGET} == 0) && ($self->{TEXT} eq "") ) )
+	{
+		$output .= sprintf("jg.setColor(\"brown\");\n");
+		$output .= sprintf("jg.setStroke(%.0f);\n", $self->{WIDTHLINE});
+		$output .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $self->{XSTARTPOS} - $minx, $self->{YPOS}, $self->{XENDPOS} - $minx, $self->{YPOS});
+		$output .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $self->{XSTARTPOS} - $minx, $self->{YPOS} - 4, $self->{XSTARTPOS} - $minx, $self->{YPOS} + 4);
+		$output .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $self->{XENDPOS} - $minx, $self->{YPOS} - 4, $self->{XENDPOS} - $minx, $self->{YPOS} + 4);
+	}
+	
+	return $output;
+}
+
+1;
+

SCTK/src/align2html/align2html-main.pl

@@ -0,0 +1,1436 @@
+#!/usr/bin/perl -w
+
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use File::Path;
+
+### Begin Local Modules
+use SegmentGroup;
+use Segment;
+use Token;
+### End Local Modules
+
+Getopt::Long::Configure(qw( auto_abbrev no_ignore_case ));
+
+my $VERSION = "0.6";
+my $AlignFile = "";
+my $MapFile = "";
+my $Outputdir = "";
+my $NonLinearSegments = 0;
+my $pixelpersec = 100;
+my $maxscalewidth = 0;
+
+## This variable says if the program is installed as a standalone executable
+my $Installed = 0;
+
+sub usage
+{
+	print "perl align2html.pl [OPTIONS] -i alignfile -o outputdir\n";
+	print "Version: $VERSION\n";
+	print "\n";
+	print "Required file arguments:\n";
+	print "  -a, --alignfile         Path to the Alignment file.\n";
+	print "  -o, --outputdir         Path to the output directory.\n";
+	print "\n";
+	print "Options\n";
+	print "  -m, --mapfile           Path to the Mapping file.\n";
+	print "\n";
+}
+
+sub extractJavaImages
+{
+    my ($outDir) = @_;
+    die "HERE DOCUMENT NOT BUILT";
+}
+
+sub unique
+{
+	my %saw;
+	return grep(!$saw{$_}++, sort @_);
+}
+
+my %SegGroups;
+my %Alignments;
+my %FileChannelSG;
+my %Overlap;
+my %CumulOverlap;
+
+sub addSGLine
+{
+	my ($inputline) = @_;
+	
+	my $SG;
+	my $Segmnt;
+	my $SYSREF = "";
+	my $SegID;
+	my $Tokn;
+	my $SpkrID;
+	
+	my $SG_SegGrpID	  = $inputline->[1];
+	my $SG_SegGrpFile = $inputline->[2];
+	my $SG_SegGrpChan = $inputline->[3];
+	my $SG_RefSegID	  = $inputline->[5];
+	my $SG_HypSegID	  = $inputline->[16];
+	
+	my $ok = 0;
+	
+	if($SG_RefSegID ne "" && $inputline->[9] ne "")
+	{
+		$SYSREF = "REF";
+		$SegID = $SG_RefSegID;
+		$SpkrID = "ref:" . $inputline->[8];
+		
+		my $dur = 0;
+		$dur = sprintf("%.3f", $inputline->[11]-$inputline->[10]) if( ($inputline->[10] ne "") || ($inputline->[11] ne "") );
+		
+		$Tokn = new Token($inputline->[9], $inputline->[10], $dur, $inputline->[12], $SYSREF, $inputline->[1], $inputline->[5], $SpkrID, $inputline->[13], $inputline->[14], $inputline->[15], $inputline->[6], $inputline->[7]);
+		$ok = 1;
+	}
+	
+	if($SG_HypSegID ne "" && $inputline->[20] ne "")
+	{
+		$SYSREF = "SYS";
+		$SegID = $SG_HypSegID;
+		$SpkrID = "hyp:" . $inputline->[19];
+		
+		my $dur = 0;
+		$dur = sprintf("%.3f", $inputline->[22]-$inputline->[21]) if( ($inputline->[21] ne "") || ($inputline->[22] ne "") );
+		
+		$Tokn = new Token($inputline->[20], $inputline->[21], $dur, $inputline->[23], $SYSREF, $inputline->[1], $inputline->[16], $SpkrID, $inputline->[24], $inputline->[25], $inputline->[26], $inputline->[17], $inputline->[18]);
+		$ok = 1;
+	}
+	
+	if($ok)
+	{
+		$SegGroups{$SG_SegGrpID} = new SegmentGroup($SG_SegGrpID, $SG_SegGrpFile, $SG_SegGrpChan) if(!exists($SegGroups{$SG_SegGrpID}));
+		$SG = $SegGroups{$SG_SegGrpID};
+			
+		if(!exists( $SG->{$SYSREF}{$SegID} ))
+		{
+			$Segmnt = new Segment($SegID, $SpkrID);
+			
+			$SG->addSysSegment($Segmnt) if($SYSREF eq "SYS");
+			$SG->addRefSegment($Segmnt) if($SYSREF eq "REF");
+		}
+		else
+		{
+			$Segmnt = $SG->{$SYSREF}{$SegID};
+		}
+		
+		$Tokn->DoDisplay();
+		$Segmnt->AddToken($Tokn);
+	}
+}
+
+sub addAlignments
+{
+	my ($inputline) = @_;
+	
+	my $SG_SegGrpID	  = $inputline->[1];
+	my $SG_Eval		  = $inputline->[4];
+	my $SG_RefTknID	  = $inputline->[9];
+	my $SG_HypTknID	  = $inputline->[20];
+	
+	push( @{ $Alignments{$SG_SegGrpID} }, [($SG_Eval, $SG_RefTknID, $SG_HypTknID)]);
+	$SegGroups{$SG_SegGrpID}->{ALIGNED} = 1;
+	
+	if($SG_RefTknID ne "")
+	{
+		$SegGroups{$SG_SegGrpID}->GetToken($SG_RefTknID)->SetWidthLine(2);
+		$SegGroups{$SG_SegGrpID}->GetToken($SG_RefTknID)->DoDisplay();
+	}
+	
+	if($SG_HypTknID ne "")
+	{
+		$SegGroups{$SG_SegGrpID}->GetToken($SG_HypTknID)->SetWidthLine(2);
+		$SegGroups{$SG_SegGrpID}->GetToken($SG_HypTknID)->DoDisplay();
+	}
+}
+
+sub loadAlignFile
+{
+	my ($alignfile) = @_;
+	my $linenum = 0;
+	
+	open(ALIGNFILE, $alignfile) or die "Unable to open for read alignfile file '$alignfile'";
+    
+    while (<ALIGNFILE>)
+    {
+        chomp;
+        my $fileline = $_;
+        $linenum++;
+        
+        next if($fileline !~ /^\[ALIGNMENT\]/);
+        next if($fileline =~ /^\[ALIGNMENT\]\s*Aligned/);
+        
+        $fileline =~ s/^\[ALIGNMENT\]//g;
+    	$fileline =~ s/\s+//g;
+    	$fileline =~ s/,/, /g;
+    	
+    	my @spliteedline = split(",", $fileline);
+
+		foreach (@spliteedline) { $_ =~ s/\s+//g; }
+    	       	
+    	addSGLine(\@spliteedline)     if($spliteedline[0] eq "SG" );
+    	addAlignments(\@spliteedline) if($spliteedline[0] eq "YES");
+    }
+    
+    close(ALIGNFILE);
+}
+
+sub GetHTMLHeader
+{
+	my ($ID) = @_;
+	
+	my $out = "";
+	
+	$out .= "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">\n";
+	$out .= "\n";
+	$out .= "<html>\n";
+	$out .= "<head>\n";
+	$out .= "<meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\">\n";
+	$out .= "<title>Segment Group ID $ID</title>\n";
+	$out .= "<script type=\"text/javascript\" src=\"js/wz_jsgraphics.js\"></script>\n";
+	$out .= "<script type=\"text/javascript\">\n";
+	$out .= "<!--\n";
+	$out .= "var jg;\n";
+	$out .= "var scale = 1;\n";
+	$out .= "\n";
+	$out .= "function Init()\n";
+	$out .= "{\n";
+	$out .= "jg = new jsGraphics(\"Graph\");\n";
+	$out .= "scale = 1;\n";
+	$out .= "}\n";
+	$out .= "\n";
+	$out .= "function myDraw()\n";
+	$out .= "{\n";
+	$out .= "jg.clear();\n";
+	
+	return $out;	
+}
+
+sub GetHTMLFooter
+{
+	my ($end, $heigh, $warning, $SG) = @_;
+	
+	my $out = "";
+	
+	$out .= "jg.paint();\n";
+	$out .= "}\n";
+	$out .= "\n";
+	$out .= "//-->\n";
+	$out .= "</script>\n";
+	$out .= "</head>\n";
+	$out .= "\n";
+	$out .= "<body bgcolor=\"white\" onload=\"Init(); myDraw()\">\n";
+	$out .= "\n";
+	$out .= sprintf("<div id=\"Graph\" style=\"background-color:white;height:%dpx;width:%dpx;border-style:solid;border-width:0;position:relative;top:0;left:0;\"></div>\n", $heigh, $end);
+	$out .= "\n";
+	$out .= "<font face=\"courier\">\n";
+	$out .= "<a href=\"#\" onclick=\"scale -= 0.5; myDraw(); return false\">[-5]</a>\n";
+	$out .= "<a href=\"#\" onclick=\"scale -= 0.1; myDraw(); return false\">[-]</a>\n";
+	$out .= "<a href=\"#\" onclick=\"scale  = 1; myDraw(); return false\">[Reset]</a>\n";
+	$out .= "<a href=\"#\" onclick=\"scale += 0.1; myDraw(); return false\">[+]</a>\n";
+	$out .= "<a href=\"#\" onclick=\"scale += 0.5; myDraw(); return false\">[+5]</a>\n";
+	
+	
+	
+	$out .= "</font>\n";
+	$out .= PrintMapping($SG);
+	$out .= PrintWarning() if($warning == 1);
+	$out .= "</body>\n";
+	$out .= "</html>\n";
+
+	return $out;	
+}
+
+sub CalculateMaxScale
+{
+	my ($start, $end, $stepsec, $heigh) = @_;
+	my $beginxline = $pixelpersec*sprintf("%.0f", $start/$pixelpersec);
+	my $endxline = $pixelpersec*sprintf("%.0f", $end/$pixelpersec);
+	
+	$beginxline -= $pixelpersec if( ($beginxline > $start) && ($beginxline > $pixelpersec) );
+	$endxline += $pixelpersec if($endxline < $end);	
+	$maxscalewidth = $endxline - $start + $pixelpersec;
+}
+
+sub GetDrawScale
+{
+	my ($start, $end, $stepsec, $heigh) = @_;
+		
+	my $output = "";
+	$output .= sprintf("jg.setColor(\"orange\");\n");
+	$output .= sprintf("jg.setStroke(Stroke.DOTTED);\n");
+	$output .= sprintf("jg.setFont(\"arial\",\"8px\",Font.BOLD);\n");
+	
+	my $beginxline = $pixelpersec*sprintf("%.0f", $start/$pixelpersec);
+	my $endxline = $pixelpersec*sprintf("%.0f", $end/$pixelpersec);
+		
+	$beginxline -= $pixelpersec if( ($beginxline > $start) && ($beginxline > $pixelpersec) );
+	$endxline += $pixelpersec if($endxline < $end);
+			
+	for(my $i=$beginxline; $i<=$endxline; $ i += $stepsec*$pixelpersec)
+	{
+		$output .= sprintf("jg.drawLine(scale*%.0f, 0, scale*%.0f, %.0f);\n", $i - $start + $pixelpersec, $i - $start + $pixelpersec, $heigh);
+		$output .= sprintf("jg.drawStringRect(\"%s\", scale*%.1f - 9, %.0f + 2, 20, \"center\");\n", $i/$pixelpersec, $i - $start + $pixelpersec, $heigh);
+	}
+		
+	return $output;
+}
+
+sub LinkToken
+{
+	my ($tkn1x, $tkn1y, $tkn2x, $tkn2y, $decalx, $type) = @_;
+	
+	my $color = "black";
+	$color = "blue" if($type eq "spkrerr");
+	$color = "red" if($type eq "sub");
+	$color = "green"if($type eq "corr");
+	
+	my $out = "";
+	$out .= sprintf("jg.setColor(\"%s\");\n", $color);
+	$out .= sprintf("jg.setStroke(2);\n");
+	$out .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $tkn1x - $decalx, $tkn1y, $tkn2x - $decalx, $tkn2y);
+	
+	return $out;
+}
+
+sub InsSubToken
+{
+	my ($tknx, $tkny, $decalx, $type) = @_;
+	
+	my $multi = 1;
+	$multi = -1 if($type eq "SYS");
+	
+	my $color = "black";
+	
+	my $out = "";
+	$out .= sprintf("jg.setColor(\"%s\");\n", $color);
+	$out .= sprintf("jg.setStroke(1);\n");
+	$out .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $tknx - $decalx, $tkny, $tknx - $decalx, $tkny + $multi*13);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 8, %.0f, %.0f*scale + 8, %.0f);\n", $tknx - $decalx, $tkny + $multi*13 , $tknx - $decalx, $tkny + $multi*13);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 5, %.0f, %.0f*scale + 5, %.0f);\n", $tknx - $decalx, $tkny + $multi*15 , $tknx - $decalx, $tkny + $multi*15);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 2, %.0f, %.0f*scale + 2, %.0f);\n", $tknx - $decalx, $tkny + $multi*17 , $tknx - $decalx, $tkny + $multi*17);
+	
+	return $out;
+}
+
+sub InsOptSubToken
+{
+	my ($tknx, $tkny, $decalx, $type) = @_;
+	
+	my $multi = 1;
+	$multi = -1 if($type eq "SYS");
+	
+	my $color = "green";
+	
+	my $out = "";
+	$out .= sprintf("jg.setColor(\"%s\");\n", $color);
+	$out .= sprintf("jg.setStroke(1);\n");
+	$out .= sprintf("jg.drawLine(%.0f*scale, %.0f, %.0f*scale, %.0f);\n", $tknx - $decalx, $tkny, $tknx - $decalx, $tkny + $multi*13);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 8, %.0f, %.0f*scale + 8, %.0f);\n", $tknx - $decalx, $tkny + $multi*13 , $tknx - $decalx, $tkny + $multi*13);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 5, %.0f, %.0f*scale + 5, %.0f);\n", $tknx - $decalx, $tkny + $multi*15 , $tknx - $decalx, $tkny + $multi*15);
+	$out .= sprintf("jg.drawLine(%.0f*scale - 2, %.0f, %.0f*scale + 2, %.0f);\n", $tknx - $decalx, $tkny + $multi*17 , $tknx - $decalx, $tkny + $multi*17);
+	
+	return $out;
+}
+
+sub DrawLinkToken
+{
+	my ($sg, $tknrefid, $tknhypid, $decalx, $type) = @_;
+	
+	my $tknref;
+	my $tknhyp;
+	
+	if($type eq "I")
+	{
+		$sg->{NINS}++;
+		$tknhyp = $sg->GetToken($tknhypid);
+		return InsSubToken($tknhyp->{XMIDPOS}, $tknhyp->{YPOS}, $sg->{MINX} - $pixelpersec, $tknhyp->{REFORSYS});
+	}
+	
+	if($type eq "D")
+	{
+		$sg->{NDEL}++;
+		$sg->{NREF}++;
+		$tknref = $sg->GetToken($tknrefid);
+		return InsSubToken($tknref->{XMIDPOS}, $tknref->{YPOS}, $sg->{MINX} - $pixelpersec, $tknref->{REFORSYS});
+	}
+	
+	if($type eq "C")
+	{
+		$sg->{NCORR}++;
+		$tknref = $sg->GetToken($tknrefid);
+		$tknhyp = $sg->GetToken($tknhypid);
+		
+		my $refdef = 0;
+		$refdef = 1 if($tknrefid ne "");
+		
+		my $hypdef = 0;
+		$hypdef = 1 if($tknhypid ne "");
+		
+		$sg->{NREF}++ if($refdef);
+		
+		return LinkToken($tknref->{XMIDPOS}, $tknref->{YPOS}, $tknhyp->{XMIDPOS}, $tknhyp->{YPOS}, $sg->{MINX} - $pixelpersec, "corr") if($refdef && $hypdef);
+		return InsOptSubToken($tknref->{XMIDPOS}, $tknref->{YPOS}, $sg->{MINX} - $pixelpersec, $tknref->{REFORSYS}) if($refdef && !$hypdef);
+		return InsOptSubToken($tknhyp->{XMIDPOS}, $tknhyp->{YPOS}, $sg->{MINX} - $pixelpersec, $tknhyp->{REFORSYS}) if(!$refdef && $hypdef);	
+	}
+	
+	if($type eq "S")
+	{
+		$sg->{NSUB}++;
+		$sg->{NREF}++;
+		$tknref = $sg->GetToken($tknrefid);
+		$tknhyp = $sg->GetToken($tknhypid);
+		return LinkToken($tknref->{XMIDPOS}, $tknref->{YPOS}, $tknhyp->{XMIDPOS}, $tknhyp->{YPOS}, $sg->{MINX} - $pixelpersec, "sub");
+	}
+	
+	if($type eq "P")
+	{
+		$sg->{NSPKRERR}++;
+		$sg->{NREF}++;
+		$tknref = $sg->GetToken($tknrefid);
+		$tknhyp = $sg->GetToken($tknhypid);
+		return LinkToken($tknref->{XMIDPOS}, $tknref->{YPOS}, $tknhyp->{XMIDPOS}, $tknhyp->{YPOS}, $sg->{MINX} - $pixelpersec, "spkrerr");
+	}
+	
+	die "Unknown type $type";
+}
+
+sub PrintWarning
+{
+	return "<p>WARNING: Some words do not have times specified for them.  For these cases, word times <img src='images/white_lt.gif' border='0' /> are estimated from the segment times.</p>\n";
+}
+
+sub PrintMapping
+{
+	my($SG) = @_;
+	my $out = "";
+	my @listmapping;
+		
+	if($MapFile ne "")
+	{
+		$out .= "<br><br>MAPPING:<br>\n";
+	
+		open(MAPFILE, $MapFile) or die "$?";
+		
+		while (<MAPFILE>)
+		{
+			chomp;
+			next if($_ =~ /^File,Channel,RefSpeaker,SysSpeaker,isMapped,timeOverlap$/);
+			
+			my @info = split(/,/, $_);
+			
+			if($info[4] eq "mapped")
+			{
+				my $tmp1 = "ref:" . uc($info[2]);
+				my $tmp2 = "hyp:" . uc($info[3]);
+				
+				if($SG->isInSpeakers($tmp1) == 1 && $SG->isInSpeakers($tmp2) == 1)
+				{
+					$tmp1 =~ s/^ref://;
+					$tmp1 =~ s/^hyp://;
+					$tmp2 =~ s/^ref://;
+					$tmp2 =~ s/^hyp://;
+					push(@listmapping, "$tmp1 <=> $tmp2<br>\n");
+				}
+			}
+		}
+			
+		close MAPFILE;
+		
+		my @listsorted = sort(@listmapping);
+		
+		foreach my $mapp (@listsorted)
+		{
+			$out .= "$mapp\n";
+		}
+	}
+		
+	return $out;
+}
+
+sub DrawKey
+{
+	my ($ystart) = @_;
+	
+	my $ystartpos = $ystart;
+	
+	my $out = "";
+	$out .= sprintf("jg.setFont(\"verdana\",\"12px\",Font.PLAIN);\n");
+	$out .= sprintf("jg.setStroke(1);\n");
+	
+	$out .= sprintf("jg.setColor(\"black\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 15, $ystartpos      , 15, $ystartpos + 13);
+	$out .= sprintf("jg.drawLine(%.0f - 8, %.0f, %.0f + 8, %.0f);\n", 15, $ystartpos + 13 , 15, $ystartpos + 13);
+	$out .= sprintf("jg.drawLine(%.0f - 5, %.0f, %.0f + 5, %.0f);\n", 15, $ystartpos + 15 , 15, $ystartpos + 15);
+	$out .= sprintf("jg.drawLine(%.0f - 2, %.0f, %.0f + 2, %.0f);\n", 15, $ystartpos + 17 , 15, $ystartpos + 17);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Deletion", 50, $ystartpos + 4);
+	
+	$out .= sprintf("jg.setColor(\"green\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 215, $ystartpos      , 215, $ystartpos + 13);
+	$out .= sprintf("jg.drawLine(%.0f - 8, %.0f, %.0f + 8, %.0f);\n", 215, $ystartpos + 13 , 215, $ystartpos + 13);
+	$out .= sprintf("jg.drawLine(%.0f - 5, %.0f, %.0f + 5, %.0f);\n", 215, $ystartpos + 15 , 215, $ystartpos + 15);
+	$out .= sprintf("jg.drawLine(%.0f - 2, %.0f, %.0f + 2, %.0f);\n", 215, $ystartpos + 17 , 215, $ystartpos + 17);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 200, \"left\");\n", "Optionally Deletable", 250, $ystartpos + 4);
+	
+	$ystartpos += 50;
+	
+	$out .= sprintf("jg.setColor(\"black\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 15, $ystartpos      , 15, $ystartpos - 13);
+	$out .= sprintf("jg.drawLine(%.0f - 8, %.0f, %.0f + 8, %.0f);\n", 15, $ystartpos - 13 , 15, $ystartpos - 13);
+	$out .= sprintf("jg.drawLine(%.0f - 5, %.0f, %.0f + 5, %.0f);\n", 15, $ystartpos - 15 , 15, $ystartpos - 15);
+	$out .= sprintf("jg.drawLine(%.0f - 2, %.0f, %.0f + 2, %.0f);\n", 15, $ystartpos - 17 , 15, $ystartpos - 17);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Insertion", 50, $ystartpos - 17);
+	
+	$out .= sprintf("jg.setColor(\"green\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 215, $ystartpos      , 215, $ystartpos - 13);
+	$out .= sprintf("jg.drawLine(%.0f - 8, %.0f, %.0f + 8, %.0f);\n", 215, $ystartpos - 13 , 215, $ystartpos - 13);
+	$out .= sprintf("jg.drawLine(%.0f - 5, %.0f, %.0f + 5, %.0f);\n", 215, $ystartpos - 15 , 215, $ystartpos - 15);
+	$out .= sprintf("jg.drawLine(%.0f - 2, %.0f, %.0f + 2, %.0f);\n", 215, $ystartpos - 17 , 215, $ystartpos - 17);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 200, \"left\");\n", "Optionally Insertable", 250, $ystartpos - 17);
+	
+	$ystartpos += 15;
+	
+	$out .= sprintf("jg.setStroke(2);\n");
+	
+	$out .= sprintf("jg.setColor(\"green\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 205, $ystartpos      , 225, $ystartpos);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Correct", 250, $ystartpos - 7);
+	
+	$out .= sprintf("jg.setColor(\"red\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 5, $ystartpos      , 25, $ystartpos);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Substitution", 50, $ystartpos - 7);
+	
+	$ystartpos += 25;
+	
+	$out .= sprintf("jg.setColor(\"blue\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 5, $ystartpos      , 25, $ystartpos);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Speaker error", 50, $ystartpos - 7);
+	
+	$out .= sprintf("jg.setStroke(1);\n");
+	$out .= sprintf("jg.setColor(\"brown\");\n");
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 205, $ystartpos      , 225, $ystartpos);
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 205, $ystartpos -4   , 205, $ystartpos + 4);
+	$out .= sprintf("jg.drawLine(%.0f, %.0f, %.0f, %.0f);\n"        , 225, $ystartpos -4   , 225, $ystartpos + 4);
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Empty Segment", 250, $ystartpos - 7);
+	
+	$ystartpos += 25;
+	
+	$out .= sprintf("jg.setColor(\"#eeffee\");\n");
+	$out .= sprintf("jg.fillRect(%.0f, %.0f, %.0f, %.0f);\n", 5, $ystartpos - 3     , 25, 8);
+	$out .= sprintf("jg.setColor(\"green\");\n");
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "References", 50, $ystartpos - 7);
+	
+	$ystartpos += 25;
+	
+	$out .= sprintf("jg.setColor(\"#ffeeee\");\n");
+	$out .= sprintf("jg.fillRect(%.0f, %.0f, %.0f, %.0f);\n", 5, $ystartpos - 3    , 25, 8);
+	$out .= sprintf("jg.setColor(\"red\");\n");
+	$out .= sprintf("jg.drawStringRect(\"%s\", %.0f, %.0f, 100, \"left\");\n", "Systems", 50, $ystartpos - 7);
+	
+	return $out;
+}
+
+GetOptions
+(
+    'alignfile=s' => \$AlignFile,
+    'mapfile=s'   => \$MapFile,
+    'outputdir=s' => \$Outputdir,
+    'version'     => sub { print "align2html version: $VERSION\n"; exit },
+    'help'        => sub { usage (); exit },
+);
+
+die "ERROR: An Align file must be set." if($AlignFile eq "");
+die "ERROR: An Output directory must be set." if($Outputdir eq "");
+
+system("mkdir -p $Outputdir");
+
+loadAlignFile($AlignFile);
+
+foreach my $SegGrpID ( sort keys %SegGroups )
+{
+	my $out = "";
+	my $outputfilename = "segmentgroup-$SegGrpID.html";
+	my $mySG = $SegGroups{$SegGrpID};
+	
+	$mySG->Compute();
+	
+	$out .= GetHTMLHeader($SegGrpID);
+	CalculateMaxScale($mySG->{MINX}, $mySG->{MAXX}, 1, $mySG->{HEIGHT});
+	$out .= $mySG->GetFillREFHYP($maxscalewidth);
+	$out .= GetDrawScale($mySG->{MINX}, $mySG->{MAXX}, 1, $mySG->{HEIGHT});
+	$out .= $mySG->GetSeparationLines($maxscalewidth);
+	$out .= $mySG->GetDraw($pixelpersec);
+	
+	foreach (@{ $Alignments{$SegGrpID} })
+	{
+		my $typ = $_->[0];
+		my $tknrefid = $_->[1];
+		my $tknhypid = $_->[2];
+			
+		$out .= DrawLinkToken($mySG, $tknrefid, $tknhypid, $mySG->{MINX} - $pixelpersec, $typ);
+	}
+	
+	$out .= DrawKey($mySG->{HEIGHT} + 25);
+	
+	$out .= GetHTMLFooter($mySG->{MAXX} - $mySG->{MINX} + $pixelpersec, $mySG->{HEIGHT} + 185, $mySG->{HASFAKETIME}, $mySG);
+	
+	open(FILESG, ">$Outputdir/$outputfilename") or die "Can't open file $Outputdir/$outputfilename for write";
+	print FILESG "$out\n";
+	close FILESG;
+	
+	my @refspkrs;
+	foreach my $segref (sort keys %{ $mySG->{REF} }){ push(@refspkrs, $mySG->{REF}{$segref}->{SPKRID}) if($mySG->{REF}{$segref}->{SPKRID} ne "ref:INTER_SEGMENT_GAP"); }
+	my $countSpeakers = scalar unique @refspkrs;
+	
+	my @sysspkrs;
+	foreach my $segsys (sort keys %{ $mySG->{SYS} }){ push(@refspkrs, $mySG->{SYS}{$segsys}->{SPKRID}) if(!($mySG->{SYS}{$segsys}->HasOnlyOneFakeToken())); }
+	my $counthypSpeakers = scalar unique @sysspkrs;
+	
+	$mySG->{ALIGNED} = 1 if( ($countSpeakers == 0) && ($counthypSpeakers == 0) );
+	$mySG->{ALIGNED} = 1 if( ($mySG->{ALIGNED} == 0) && ($countSpeakers == 1) && ($counthypSpeakers == 0) && ($mySG->{HASFAKETIME} == 1) );
+	
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NCORR} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NCORR}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NINS} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NINS}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSUB} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSUB}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NDEL} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NDEL}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NREF} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NREF}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSPKRERR} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSPKRERR}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIME} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIME}));
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIMEALIGNED} = 0 if(!exists($FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIMEALIGNED}));
+	
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NCORR} += $mySG->{NCORR};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NINS} += $mySG->{NINS};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSUB} += $mySG->{NSUB};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NDEL} += $mySG->{NDEL};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NREF} += $mySG->{NREF};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{NSPKRERR} += $mySG->{NSPKRERR};
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIME} += $mySG->{ET} - $mySG->{BT};
+	push( @{ $FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{LISTSG} }, $SegGrpID);
+	push( @{ $FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{LISTSGALIGNED} }, $SegGrpID) if($mySG->{ALIGNED} == 1);
+	$FileChannelSG{$mySG->{FILE}}{$mySG->{CHANNEL}}{TOTTIMEALIGNED} += $mySG->{ET} - $mySG->{BT} if($mySG->{ALIGNED} == 1);
+	
+	$Overlap{$countSpeakers}{NREF} = 0 if(!exists($Overlap{$countSpeakers}{NREF}));
+	$Overlap{$countSpeakers}{NREF} += $mySG->{NREF};
+	
+	$Overlap{$countSpeakers}{NCORR} = 0 if(!exists($Overlap{$countSpeakers}{NCORR}));
+	$Overlap{$countSpeakers}{NCORR} += $mySG->{NCORR};
+	$Overlap{$countSpeakers}{PCORR} = "NA";
+	$Overlap{$countSpeakers}{PCORR} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NCORR}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{NSUB} = 0 if(!exists($Overlap{$countSpeakers}{NSUB}));
+	$Overlap{$countSpeakers}{NSUB} += $mySG->{NSUB};
+	$Overlap{$countSpeakers}{PSUB} = "NA";
+	$Overlap{$countSpeakers}{PSUB} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NSUB}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{NINS} = 0 if(!exists($Overlap{$countSpeakers}{NINS}));
+	$Overlap{$countSpeakers}{NINS} += $mySG->{NINS};	
+	$Overlap{$countSpeakers}{PINS} = "NA";
+	$Overlap{$countSpeakers}{PINS} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NINS}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{NDEL} = 0 if(!exists($Overlap{$countSpeakers}{NDEL}));
+	$Overlap{$countSpeakers}{NDEL} += $mySG->{NDEL};
+	$Overlap{$countSpeakers}{PDEL} = "NA";
+	$Overlap{$countSpeakers}{PDEL} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NDEL}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{NSPKRERR} = 0 if(!exists($Overlap{$countSpeakers}{NSPKRERR}));
+	$Overlap{$countSpeakers}{NSPKRERR} += $mySG->{NSPKRERR};
+	$Overlap{$countSpeakers}{PSPKRERR} = "NA";
+	$Overlap{$countSpeakers}{PSPKRERR} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NSPKRERR}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{NERR} = $Overlap{$countSpeakers}{NSUB} + $Overlap{$countSpeakers}{NINS} + $Overlap{$countSpeakers}{NDEL} + $Overlap{$countSpeakers}{NSPKRERR};
+	$Overlap{$countSpeakers}{PERR} = "NA";
+	$Overlap{$countSpeakers}{PERR} = sprintf("%.1f", 100*$Overlap{$countSpeakers}{NERR}/$Overlap{$countSpeakers}{NREF}) if($Overlap{$countSpeakers}{NREF} != 0);
+	
+	$Overlap{$countSpeakers}{TOTTIME} = 0 if(!exists($Overlap{$countSpeakers}{TOTTIME}));
+	$Overlap{$countSpeakers}{TOTTIME} += $mySG->{ET} - $mySG->{BT};
+	
+	push( @{ $Overlap{$countSpeakers}{LISTSG} }, $SegGrpID);
+	$Overlap{$countSpeakers}{NLISTSG} = scalar @{ $Overlap{$countSpeakers}{LISTSG} };
+	
+	$Overlap{$countSpeakers}{TOTTIMEALIGNED} = 0 if(!exists($Overlap{$countSpeakers}{TOTTIMEALIGNED}));
+	$Overlap{$countSpeakers}{TOTTIMEALIGNED} += $mySG->{ET} - $mySG->{BT} if($mySG->{ALIGNED} == 1);
+
+	push( @{ $Overlap{$countSpeakers}{LISTSGALIGNED} }, $SegGrpID) if($mySG->{ALIGNED} == 1);
+	$Overlap{$countSpeakers}{NLISTSGALIGNED} = 0;
+	$Overlap{$countSpeakers}{NLISTSGALIGNED} = scalar(@{ $Overlap{$countSpeakers}{LISTSGALIGNED} }) if(exists($Overlap{$countSpeakers}{LISTSGALIGNED}));
+	
+	$Overlap{$countSpeakers}{NUMHYPTOKENS} = 0 if(!exists($Overlap{$countSpeakers}{NUMHYPTOKENS}));
+	$Overlap{$countSpeakers}{NUMHYPTOKENS} += $SegGroups{$SegGrpID}->GetNumHypWords();
+}
+
+open(FILEINDEX, ">$Outputdir/index.html") or die "$?";
+
+print FILEINDEX "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">\n";
+print FILEINDEX "<html>\n";
+print FILEINDEX "<head>\n";
+print FILEINDEX "<meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\">\n";
+print FILEINDEX "<title>Overall Performance Summary by Meeting</title>\n";
+print FILEINDEX "</head>\n";
+print FILEINDEX "<body>\n";
+print FILEINDEX "<center>\n";
+print FILEINDEX "<h2>Overall Performance Summary by Meeting</h2>\n";
+print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+print FILEINDEX "<tbody>\n";
+print FILEINDEX "<tr>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: left;\"><b>File\/Channel</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total<br>Time</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total Time<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+print FILEINDEX "</tr>\n";
+
+my $TnbrSG = 0;
+my $Ttottime = 0;
+my $TnbrSGAligned = 0;
+my $Ttottimealigned = 0;
+my $Tnumref = 0;
+my $TpercCorr = 0;
+my $TpercSub = 0;
+my $TpercIns = 0;
+my $TpercDel = 0;
+my $TpercSpErr = 0;
+my $TpercErr = 0;
+my $TnumCorr = 0;
+my $TnumSub = 0;
+my $TnumIns = 0;
+my $TnumDel = 0;
+my $TnumSpErr = 0;
+my $TnumErr = 0;
+
+my $ccount = 0;
+
+foreach my $file(sort keys %FileChannelSG)
+{
+	foreach my $channel(sort keys %{ $FileChannelSG{$file} })
+	{
+		my $cleanfile = $file;
+		$cleanfile =~ s/\//\_/g;		
+		my $filename = "$cleanfile" . "_" . "$channel" . ".html";
+		
+		my $percCorr = "NA";
+		my $percSub = "NA";
+		my $percIns = "NA";
+		my $percDel = "NA";
+		my $percErr = "NA";
+		my $percSpErr = "NA";
+		
+		my $numref = $FileChannelSG{$file}{$channel}{NREF};
+		$Tnumref += $numref;
+		
+		my $nbrSG = scalar(@{ $FileChannelSG{$file}{$channel}{LISTSG} });
+		$TnbrSG += $nbrSG;
+		
+		my $tottime = sprintf("%.3f", $FileChannelSG{$file}{$channel}{TOTTIME});
+		$Ttottime += $tottime;
+		
+		my $nbrSGAligned = scalar(@{ $FileChannelSG{$file}{$channel}{LISTSGALIGNED} });
+        $TnbrSGAligned += $nbrSGAligned;
+		
+		my $tottimealigned = sprintf("%.3f", $FileChannelSG{$file}{$channel}{TOTTIMEALIGNED});
+		$Ttottimealigned += $tottimealigned;
+		
+		if($FileChannelSG{$file}{$channel}{NREF} != 0)
+		{
+			$percCorr = sprintf("%.1f%% (%d)", 100*$FileChannelSG{$file}{$channel}{NCORR}/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NCORR});
+			$percSub = sprintf("%.1f%% (%d)", 100*$FileChannelSG{$file}{$channel}{NSUB}/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NSUB});
+			$percIns = sprintf("%.1f%% (%d)", 100*$FileChannelSG{$file}{$channel}{NINS}/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NINS});
+			$percDel = sprintf("%.1f%% (%d)", 100*$FileChannelSG{$file}{$channel}{NDEL}/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NDEL});
+			$percErr = sprintf("%.1f%% (%d)", 100*($FileChannelSG{$file}{$channel}{NDEL}+$FileChannelSG{$file}{$channel}{NINS}+$FileChannelSG{$file}{$channel}{NSUB}+$FileChannelSG{$file}{$channel}{NSPKRERR})/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NDEL}+$FileChannelSG{$file}{$channel}{NINS}+$FileChannelSG{$file}{$channel}{NSUB}+$FileChannelSG{$file}{$channel}{NSPKRERR});
+			$percSpErr = sprintf("%.1f%% (%d)", 100*$FileChannelSG{$file}{$channel}{NSPKRERR}/$FileChannelSG{$file}{$channel}{NREF}, $FileChannelSG{$file}{$channel}{NSPKRERR});
+			
+			$TpercCorr += sprintf("%.3f", 100*$FileChannelSG{$file}{$channel}{NCORR}/$FileChannelSG{$file}{$channel}{NREF});
+			$TpercSub += sprintf("%.3f", 100*$FileChannelSG{$file}{$channel}{NSUB}/$FileChannelSG{$file}{$channel}{NREF});
+			$TpercIns += sprintf("%.3f", 100*$FileChannelSG{$file}{$channel}{NINS}/$FileChannelSG{$file}{$channel}{NREF});
+			$TpercDel += sprintf("%.3f", 100*$FileChannelSG{$file}{$channel}{NDEL}/$FileChannelSG{$file}{$channel}{NREF});
+			$TpercSpErr += sprintf("%.3f", 100*$FileChannelSG{$file}{$channel}{NSPKRERR}/$FileChannelSG{$file}{$channel}{NREF});
+			$TpercErr += sprintf("%.3f", 100*($FileChannelSG{$file}{$channel}{NDEL}+$FileChannelSG{$file}{$channel}{NINS}+$FileChannelSG{$file}{$channel}{NSUB}+$FileChannelSG{$file}{$channel}{NSPKRERR})/$FileChannelSG{$file}{$channel}{NREF});
+			$TnumCorr += $FileChannelSG{$file}{$channel}{NCORR};
+			$TnumSub += $FileChannelSG{$file}{$channel}{NSUB};
+			$TnumIns += $FileChannelSG{$file}{$channel}{NINS};
+			$TnumDel += $FileChannelSG{$file}{$channel}{NDEL};
+			$TnumSpErr += $FileChannelSG{$file}{$channel}{NSPKRERR};
+			$TnumErr += $FileChannelSG{$file}{$channel}{NDEL}+$FileChannelSG{$file}{$channel}{NINS}+$FileChannelSG{$file}{$channel}{NSUB}+$FileChannelSG{$file}{$channel}{NSPKRERR};
+			
+			$ccount++;
+		}
+		
+		print FILEINDEX "<tr>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: left;\"><a href=\"$filename\" target=\"_blank\">$file\/$channel</a></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$nbrSG</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$tottime</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$nbrSGAligned</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$tottimealigned</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$numref</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percCorr</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percSub</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percIns</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percDel</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percSpErr</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$percErr</td>\n";
+		print FILEINDEX "</tr>\n";		
+	}
+}
+
+my $dTpercCorr = sprintf("%.1f%% (%d)",  100*$TnumCorr/$Tnumref, $TnumCorr);
+my $dTpercSub = sprintf("%.1f%% (%d)",   100*$TnumSub/$Tnumref, $TnumSub);
+my $dTpercIns = sprintf("%.1f%% (%d)",   100*$TnumIns/$Tnumref, $TnumIns);
+my $dTpercDel = sprintf("%.1f%% (%d)",   100*$TnumDel/$Tnumref, $TnumDel);
+my $dTpercSpErr = sprintf("%.1f%% (%d)", 100*$TnumSpErr/$Tnumref, $TnumSpErr);
+my $dTpercErr = sprintf("%.1f%% (%d)",   100*$TnumErr/$Tnumref, $TnumErr);
+
+print FILEINDEX "<tr>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: left;\"><b>Ensemble</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$TnbrSG</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$Ttottime</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$TnbrSGAligned</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$Ttottimealigned</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$Tnumref</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercCorr</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercSub</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercIns</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercDel</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercSpErr</td>\n";
+print FILEINDEX "<td style=\"vertical-align: top; text-align: center;\">$dTpercErr</td>\n";
+print FILEINDEX "</tr>\n";
+print FILEINDEX "</tbody>\n";
+print FILEINDEX "</table>\n";
+
+print FILEINDEX "<br>\n";
+
+my $Total_regs = 0;
+my $Total_tottime = 0;
+my $Total_regsalign = 0;
+my $Total_tottimealign = 0;
+my $Total_nref = 0;
+my $Total_ncorr = 0;
+my $Total_nsub = 0;
+my $Total_nins = 0;
+my $Total_ndel = 0;
+my $Total_nspkerr = 0;
+my $Total_nerr = 0;
+
+print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+print FILEINDEX "<tbody>\n";
+print FILEINDEX "<tr>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Speaker<br>Overlap</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total<br>Time</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total Time<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+print FILEINDEX "</tr>\n";
+
+foreach my $spkover (sort {$a <=> $b} keys %Overlap)
+{
+	my $display_regs = $Overlap{$spkover}{NLISTSG};
+	my $display_tottime = sprintf("%.3f", $Overlap{$spkover}{TOTTIME});
+	my $display_regsalign = $Overlap{$spkover}{NLISTSGALIGNED};
+	my $display_tottimealign = sprintf("%.3f", $Overlap{$spkover}{TOTTIMEALIGNED});
+	my $display_nref = "-";
+	my $display_ncorr = "-";
+	my $display_nsub = "-";
+	my $display_nins = "-";
+	my $display_ndel = "-";
+	my $display_nspkerr = "-";
+	my $display_nerr = "-";
+	
+	$Total_regs += $Overlap{$spkover}{NLISTSG};
+	$Total_tottime += $Overlap{$spkover}{TOTTIME};
+	$Total_regsalign += $Overlap{$spkover}{NLISTSGALIGNED};
+	$Total_tottimealign += $Overlap{$spkover}{TOTTIMEALIGNED};
+	
+	if($spkover == 0)
+	{
+		$display_nins = sprintf("(%d)", $Overlap{$spkover}{NUMHYPTOKENS});
+		
+		$Total_nins += $Overlap{$spkover}{NUMHYPTOKENS};
+		$Total_nerr += $Total_nins;
+	}
+	
+	if($Overlap{$spkover}{NREF} != 0)
+	{
+		$display_nref = $Overlap{$spkover}{NREF};
+		$display_ncorr = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PCORR}, $Overlap{$spkover}{NCORR});
+		$display_nsub = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PSUB}, $Overlap{$spkover}{NSUB});
+		$display_nins = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PINS}, $Overlap{$spkover}{NINS});
+		$display_ndel = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PDEL}, $Overlap{$spkover}{NDEL});
+		$display_nspkerr = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PSPKRERR}, $Overlap{$spkover}{NSPKRERR});
+		$display_nerr = sprintf("%.1f%% (%d)", $Overlap{$spkover}{PERR}, $Overlap{$spkover}{NERR});
+		
+		$Total_nref += $Overlap{$spkover}{NREF};
+		$Total_ncorr += $Overlap{$spkover}{NCORR};
+		$Total_nsub += $Overlap{$spkover}{NSUB};
+		$Total_nins += $Overlap{$spkover}{NINS};
+		$Total_ndel += $Overlap{$spkover}{NDEL};
+		$Total_nspkerr += $Overlap{$spkover}{NSPKRERR};
+		$Total_nerr += $Overlap{$spkover}{NERR};
+	}
+	
+	print FILEINDEX "<tr>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>$spkover<b></td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regs</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottime</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regsalign</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottimealign</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nref</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ncorr</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nsub</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nins</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ndel</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nspkerr</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nerr</td>\n";
+	print FILEINDEX "</tr>\n";
+	
+	# data for Cumuloverlap table
+	$CumulOverlap{$spkover}{NLISTSG} = $Total_regs;
+	$CumulOverlap{$spkover}{TOTTIME} = $Total_tottime;
+	$CumulOverlap{$spkover}{NLISTSGALIGNED} = $Total_regsalign;
+	$CumulOverlap{$spkover}{TOTTIMEALIGNED} = $Total_tottimealign;
+	$CumulOverlap{$spkover}{NREF} = $Total_nref;
+	$CumulOverlap{$spkover}{NCORR} = $Total_ncorr;
+	$CumulOverlap{$spkover}{NSUB} = $Total_nsub;
+	$CumulOverlap{$spkover}{NINS} = $Total_nins;
+	$CumulOverlap{$spkover}{NDEL} = $Total_ndel;
+	$CumulOverlap{$spkover}{NSPKRERR} = $Total_nspkerr;
+	$CumulOverlap{$spkover}{NERR} = $Total_nerr;
+}
+
+my $display_Total_regs = $Total_regs;
+my $display_Total_tottime = sprintf("%.3f", $Total_tottime);
+my $display_Total_regsalign = $Total_regsalign;
+my $display_Total_tottimealign = sprintf("%.3f", $Total_tottimealign);
+my $display_Total_nref = $Total_nref;
+my $display_Total_ncorr = sprintf("%.1f%% (%d)", 100*$Total_ncorr/$Total_nref, $Total_ncorr);
+my $display_Total_nsub = sprintf("%.1f%% (%d)", 100*$Total_nsub/$Total_nref, $Total_nsub);
+my $display_Total_nins = sprintf("%.1f%% (%d)", 100*$Total_nins/$Total_nref, $Total_nins);
+my $display_Total_ndel = sprintf("%.1f%% (%d)", 100*$Total_ndel/$Total_nref, $Total_ndel);
+my $display_Total_nspkerr = sprintf("%.1f%% (%d)", 100*$Total_nspkerr/$Total_nref, $Total_nspkerr);
+my $display_Total_nerr = sprintf("%.1f%% (%d)", 100*$Total_nerr/$Total_nref, $Total_nerr);
+
+print FILEINDEX "<tr>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ensemble</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_regs</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_tottime</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_regsalign</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_tottimealign</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nref</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_ncorr</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nsub</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nins</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_ndel</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nspkerr</td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nerr</td>\n";
+print FILEINDEX "</tr>\n";
+
+print FILEINDEX "</tbody>\n";
+print FILEINDEX "</table>\n";
+
+print FILEINDEX "<br>\n";
+
+print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+print FILEINDEX "<tbody>\n";
+print FILEINDEX "<tr>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Speaker<br>Cumulative<br>Overlap</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total<br>Time</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total Time<br>Aligned</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+print FILEINDEX "</tr>\n";
+
+foreach my $cumulspkover (sort {$a <=> $b} keys %CumulOverlap)
+{
+	my $display_regs = $CumulOverlap{$cumulspkover}{NLISTSG};
+	my $display_tottime = sprintf("%.3f", $CumulOverlap{$cumulspkover}{TOTTIME});
+	my $display_regsalign = $CumulOverlap{$cumulspkover}{NLISTSGALIGNED};
+	my $display_tottimealign = sprintf("%.3f", $CumulOverlap{$cumulspkover}{TOTTIMEALIGNED});
+	my $display_nref = "-";
+	my $display_ncorr = "-";
+	my $display_nsub = "-";
+	my $display_nins = "-";
+	my $display_ndel = "-";
+	my $display_nspkerr = "-";
+	my $display_nerr = "-";
+	
+	if($cumulspkover == 0)
+	{
+		$display_nins = sprintf("(%d)", $CumulOverlap{$cumulspkover}{NINS});
+	}
+	
+	if($CumulOverlap{$cumulspkover}{NREF} != 0)
+	{
+		$display_nref = $CumulOverlap{$cumulspkover}{NREF};
+		$display_ncorr = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NCORR}     *100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NCORR});
+		$display_nsub = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NSUB}       *100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NSUB});
+		$display_nins = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NINS}       *100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NINS});
+		$display_ndel = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NDEL}       *100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NDEL});
+		$display_nspkerr = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NSPKRERR}*100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NSPKRERR});
+		$display_nerr = sprintf("%.1f%% (%d)", $CumulOverlap{$cumulspkover}{NERR}       *100/$CumulOverlap{$cumulspkover}{NREF}, $CumulOverlap{$cumulspkover}{NERR});
+	}
+
+	print FILEINDEX "<tr>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>$cumulspkover<b></td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regs</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottime</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regsalign</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottimealign</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nref</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ncorr</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nsub</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nins</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ndel</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nspkerr</td>\n";
+	print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nerr</td>\n";
+	print FILEINDEX "</tr>\n";
+}
+
+
+print FILEINDEX "</tbody>\n";
+print FILEINDEX "</table>\n";
+
+
+print FILEINDEX "</center>\n";
+print FILEINDEX "</body>\n";
+print FILEINDEX "</html>\n";
+
+close FILEINDEX;
+
+foreach my $file(sort keys %FileChannelSG)
+{
+	foreach my $channel(sort keys %{ $FileChannelSG{$file} })
+	{
+		my %overallmeeting;
+		my %cumuloverallmeeting;
+		my $cleanfile = $file;
+		$cleanfile =~ s/\//\_/g;		
+		my $filename = "$cleanfile" . "_" . "$channel" . ".html";
+		my $bottom = "";
+	
+		open(FILEINDEX, ">$Outputdir/$filename") or die "$?";
+
+		print FILEINDEX "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">\n";
+		print FILEINDEX "<html>\n";
+		print FILEINDEX "<head>\n";
+		print FILEINDEX "<meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\">\n";
+		print FILEINDEX "<title>Overall Performance Summary for Meeting $file\/$channel</title>\n";
+		print FILEINDEX "</head>\n";
+		print FILEINDEX "<body>\n";
+		
+		print FILEINDEX "<center>\n";
+		print FILEINDEX "<h2>Overall Performance Summary for Meeting $file\/$channel</h2>\n";
+		print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+		print FILEINDEX "<tbody>\n";
+		
+		print FILEINDEX "<tr>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: left;\"><b>Region ID</b></b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Begin<br>Time</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>End<br>Time</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Duration</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref<br>Speaker</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Hyp<br>Speaker</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+		print FILEINDEX "</tr>\n";
+			
+		foreach my $SG_ID (sort {$a <=> $b} @{ $FileChannelSG{$file}{$channel}{LISTSG} })
+		{
+			my $BT = sprintf("%.3f", $SegGroups{$SG_ID}->{BT});
+			my $ET = sprintf("%.3f", $SegGroups{$SG_ID}->{ET});
+			my $Duration = sprintf("%.3f", $SegGroups{$SG_ID}->{ET} - $SegGroups{$SG_ID}->{BT});
+			
+			my @refspkrs;
+			my @sysspkrs;
+			
+			foreach my $segref (sort keys %{ $SegGroups{$SG_ID}->{REF} }) { push(@refspkrs, $SegGroups{$SG_ID}->{REF}{$segref}->{SPKRID}) if($SegGroups{$SG_ID}->{REF}{$segref}->{SPKRID} ne "ref:INTER_SEGMENT_GAP"); }
+			foreach my $segsys (sort keys %{ $SegGroups{$SG_ID}->{SYS} }) { push(@sysspkrs, $SegGroups{$SG_ID}->{SYS}{$segsys}->{SPKRID}) if(!($SegGroups{$SG_ID}->{SYS}{$segsys}->HasOnlyOneFakeToken())); }
+			
+			my $nrefspkr = scalar unique @refspkrs;
+			my $nhypspkr = scalar unique @sysspkrs;
+			
+			$SegGroups{$SG_ID}->{ALIGNED} = 1 if( ($nrefspkr == 0) && ($nhypspkr == 0) );
+			$SegGroups{$SG_ID}->{ALIGNED} = 1 if( ($SegGroups{$SG_ID}->{ALIGNED} == 0) && ($nrefspkr == 1) && ($nhypspkr == 0) && ($SegGroups{$SG_ID}->{HASFAKETIME} == 1) );
+			
+			my $titleSG = "<a href=\"segmentgroup-$SG_ID.html\" target=\"_blank\">$SG_ID</a>";
+			my $nref = "-";
+			my $percCorr = "-";
+			my $percSub = "-";
+			my $percIns = "-";
+			my $percDel = "-";
+			my $percErr = "-";
+			my $percSpErr = "-";
+			
+			if($SegGroups{$SG_ID}->{NREF} != 0)
+			{
+				$percCorr = sprintf("%.1f%% (%d)", 100*$SegGroups{$SG_ID}->{NCORR}/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NCORR});
+				$percSub = sprintf("%.1f%% (%d)", 100*$SegGroups{$SG_ID}->{NSUB}/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NSUB});
+				$percIns = sprintf("%.1f%% (%d)", 100*$SegGroups{$SG_ID}->{NINS}/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NINS});
+				$percDel = sprintf("%.1f%% (%d)", 100*$SegGroups{$SG_ID}->{NDEL}/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NDEL});
+				$percErr = sprintf("%.1f%% (%d)", 100*($SegGroups{$SG_ID}->{NDEL}+$SegGroups{$SG_ID}->{NINS}+$SegGroups{$SG_ID}->{NSUB}+$SegGroups{$SG_ID}->{NSPKRERR})/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NDEL}+$SegGroups{$SG_ID}->{NINS}+$SegGroups{$SG_ID}->{NSUB}+$SegGroups{$SG_ID}->{NSPKRERR});
+				$percSpErr = sprintf("%.1f%% (%d)", 100*$SegGroups{$SG_ID}->{NSPKRERR}/$SegGroups{$SG_ID}->{NREF}, $SegGroups{$SG_ID}->{NSPKRERR});
+			}
+			else
+			{
+                $percIns = sprintf("(%d)", $SegGroups{$SG_ID}->{NINS}) if($SegGroups{$SG_ID}->{ALIGNED} == 1);
+			}
+						
+            if($SegGroups{$SG_ID}->{ALIGNED} == 0)
+			{
+				$titleSG = "*<i>$titleSG</i>";
+				$nrefspkr = "<i>$nrefspkr</i>";
+				$nhypspkr = "<i>$nhypspkr</i>";
+				$BT = "<i>$BT</i>";
+				$ET = "<i>$ET</i>";
+				$Duration = "<i>$Duration</i>";
+				$bottom = "<br>*: Segment Group ignored for scoring.<br>";
+			}
+			else
+			{
+                $nref = $SegGroups{$SG_ID}->{NREF};
+			}
+					
+			print FILEINDEX "<tr>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: left;\">$titleSG</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$BT</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$ET</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$Duration</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$nrefspkr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$nhypspkr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$nref</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percCorr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percSub</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percIns</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percDel</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percSpErr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$percErr</td>\n";
+			print FILEINDEX "</tr>\n";
+			
+			$BT =~ s/<i>//;
+			$ET =~ s/<i>//;
+			$nrefspkr =~ s/<i>//;
+			$nhypspkr =~ s/<i>//;
+			$BT =~ s/<\/i>//;
+			$ET =~ s/<\/i>//;
+			$nrefspkr =~ s/<\/i>//;
+			$nhypspkr =~ s/<\/i>//;
+			
+			my $countSpeakers = $nrefspkr;
+			
+			$overallmeeting{$countSpeakers}{NREF} = 0 if(!exists($overallmeeting{$countSpeakers}{NREF}));
+			$overallmeeting{$countSpeakers}{NREF} += $SegGroups{$SG_ID}->{NREF};
+			
+			$overallmeeting{$countSpeakers}{NCORR} = 0 if(!exists($overallmeeting{$countSpeakers}{NCORR}));
+			$overallmeeting{$countSpeakers}{NCORR} += $SegGroups{$SG_ID}->{NCORR};
+			$overallmeeting{$countSpeakers}{PCORR} = "NA";
+			$overallmeeting{$countSpeakers}{PCORR} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NCORR}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{NSUB} = 0 if(!exists($overallmeeting{$countSpeakers}{NSUB}));
+			$overallmeeting{$countSpeakers}{NSUB} += $SegGroups{$SG_ID}->{NSUB};
+			$overallmeeting{$countSpeakers}{PSUB} = "NA";
+			$overallmeeting{$countSpeakers}{PSUB} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NSUB}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{NINS} = 0 if(!exists($overallmeeting{$countSpeakers}{NINS}));
+			$overallmeeting{$countSpeakers}{NINS} += $SegGroups{$SG_ID}->{NINS};
+			$overallmeeting{$countSpeakers}{PINS} = "NA";
+			$overallmeeting{$countSpeakers}{PINS} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NINS}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{NDEL} = 0 if(!exists($overallmeeting{$countSpeakers}{NDEL}));
+			$overallmeeting{$countSpeakers}{NDEL} += $SegGroups{$SG_ID}->{NDEL};
+			$overallmeeting{$countSpeakers}{PDEL} = "NA";
+			$overallmeeting{$countSpeakers}{PDEL} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NDEL}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{NSPKRERR} = 0 if(!exists($overallmeeting{$countSpeakers}{NSPKRERR}));
+			$overallmeeting{$countSpeakers}{NSPKRERR} += $SegGroups{$SG_ID}->{NSPKRERR};
+			$overallmeeting{$countSpeakers}{PSPKRERR} = "NA";
+			$overallmeeting{$countSpeakers}{PSPKRERR} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NSPKRERR}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{NERR} = $overallmeeting{$countSpeakers}{NSUB} + $overallmeeting{$countSpeakers}{NINS} + $overallmeeting{$countSpeakers}{NDEL} + $overallmeeting{$countSpeakers}{NSPKRERR};
+			$overallmeeting{$countSpeakers}{PERR} = "NA";
+			$overallmeeting{$countSpeakers}{PERR} = sprintf("%.1f", 100*$overallmeeting{$countSpeakers}{NERR}/$overallmeeting{$countSpeakers}{NREF}) if($overallmeeting{$countSpeakers}{NREF} != 0);
+			
+			$overallmeeting{$countSpeakers}{TOTTIME} = 0 if(!exists($overallmeeting{$countSpeakers}{TOTTIME}));
+			$overallmeeting{$countSpeakers}{TOTTIME} += $SegGroups{$SG_ID}->{ET} - $SegGroups{$SG_ID}->{BT};
+			
+			push( @{ $overallmeeting{$countSpeakers}{LISTSG} }, $SG_ID);
+			$overallmeeting{$countSpeakers}{NLISTSG} = scalar @{ $overallmeeting{$countSpeakers}{LISTSG} };
+			
+			$overallmeeting{$countSpeakers}{TOTTIMEALIGNED} = 0 if(!exists($overallmeeting{$countSpeakers}{TOTTIMEALIGNED}));
+			$overallmeeting{$countSpeakers}{TOTTIMEALIGNED} += $SegGroups{$SG_ID}->{ET} - $SegGroups{$SG_ID}->{BT} if($SegGroups{$SG_ID}->{ALIGNED} == 1);
+		
+			push( @{ $overallmeeting{$countSpeakers}{LISTSGALIGNED} }, $SG_ID) if($SegGroups{$SG_ID}->{ALIGNED} == 1);
+			$overallmeeting{$countSpeakers}{NLISTSGALIGNED} = 0;
+			$overallmeeting{$countSpeakers}{NLISTSGALIGNED} = scalar(@{ $overallmeeting{$countSpeakers}{LISTSGALIGNED} }) if(exists($overallmeeting{$countSpeakers}{LISTSGALIGNED}));
+			
+			$overallmeeting{$countSpeakers}{NUMHYPTOKENS} = 0 if(!exists($overallmeeting{$countSpeakers}{NUMHYPTOKENS}));
+			$overallmeeting{$countSpeakers}{NUMHYPTOKENS} += $SegGroups{$SG_ID}->GetNumHypWords();
+		}
+		
+		print FILEINDEX "</tbody>\n";
+		print FILEINDEX "</table>\n";
+		print FILEINDEX "$bottom";
+		print FILEINDEX "<br>\n";
+		
+		my $Total_regs = 0;
+		my $Total_tottime = 0;
+		my $Total_regsalign = 0;
+		my $Total_tottimealign = 0;
+		my $Total_nref = 0;
+		my $Total_ncorr = 0;
+		my $Total_nsub = 0;
+		my $Total_nins = 0;
+		my $Total_ndel = 0;
+		my $Total_nspkerr = 0;
+		my $Total_nerr = 0;
+		
+		print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+		print FILEINDEX "<tbody>\n";
+		print FILEINDEX "<tr>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Speaker<br>Overlap</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total<br>Time</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions<br>Aligned</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total Time<br>Aligned</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+		print FILEINDEX "</tr>\n";
+		
+		foreach my $spkover (sort {$a <=> $b} keys %overallmeeting)
+		{
+			my $display_regs = $overallmeeting{$spkover}{NLISTSG};
+			my $display_tottime = sprintf("%.3f", $overallmeeting{$spkover}{TOTTIME});
+			my $display_regsalign = $overallmeeting{$spkover}{NLISTSGALIGNED};
+			my $display_tottimealign = sprintf("%.3f", $overallmeeting{$spkover}{TOTTIMEALIGNED});
+			my $display_nref = "-";
+			my $display_ncorr = "-";
+			my $display_nsub = "-";
+			my $display_nins = "-";
+			my $display_ndel = "-";
+			my $display_nspkerr = "-";
+			my $display_nerr = "-";
+			
+			$Total_regs += $overallmeeting{$spkover}{NLISTSG};
+			$Total_tottime += $overallmeeting{$spkover}{TOTTIME};
+			$Total_regsalign += $overallmeeting{$spkover}{NLISTSGALIGNED};
+			$Total_tottimealign += $overallmeeting{$spkover}{TOTTIMEALIGNED};
+			
+			if($spkover == 0)
+			{
+				$display_nins = sprintf("(%d)", $overallmeeting{$spkover}{NUMHYPTOKENS});
+				$Total_nins += $overallmeeting{$spkover}{NUMHYPTOKENS};
+				$Total_nerr += $Total_nins;
+			}
+			
+			if($overallmeeting{$spkover}{NREF} != 0)
+			{
+				$display_nref = $overallmeeting{$spkover}{NREF};
+				$display_ncorr = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PCORR}, $overallmeeting{$spkover}{NCORR});
+				$display_nsub = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PSUB}, $overallmeeting{$spkover}{NSUB});
+				$display_nins = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PINS}, $overallmeeting{$spkover}{NINS});
+				$display_ndel = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PDEL}, $overallmeeting{$spkover}{NDEL});
+				$display_nspkerr = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PSPKRERR}, $overallmeeting{$spkover}{NSPKRERR});
+				$display_nerr = sprintf("%.1f%% (%d)", $overallmeeting{$spkover}{PERR}, $overallmeeting{$spkover}{NERR});
+				$Total_nref += $overallmeeting{$spkover}{NREF};
+				$Total_ncorr += $overallmeeting{$spkover}{NCORR};
+				$Total_nsub += $overallmeeting{$spkover}{NSUB};
+				$Total_nins += $overallmeeting{$spkover}{NINS};
+				$Total_ndel += $overallmeeting{$spkover}{NDEL};
+				$Total_nspkerr += $overallmeeting{$spkover}{NSPKRERR};
+				$Total_nerr += $overallmeeting{$spkover}{NERR};
+			}
+			
+			print FILEINDEX "<tr>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>$spkover<b></td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regs</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottime</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regsalign</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottimealign</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nref</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ncorr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nsub</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nins</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ndel</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nspkerr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nerr</td>\n";
+			print FILEINDEX "</tr>\n";
+			
+			# data for Cumuloverlap table
+			$cumuloverallmeeting{$spkover}{NLISTSG} = $Total_regs;
+			$cumuloverallmeeting{$spkover}{TOTTIME} = $Total_tottime;
+			$cumuloverallmeeting{$spkover}{NLISTSGALIGNED} = $Total_regsalign;
+			$cumuloverallmeeting{$spkover}{TOTTIMEALIGNED} = $Total_tottimealign;
+			$cumuloverallmeeting{$spkover}{NREF} = $Total_nref;
+			$cumuloverallmeeting{$spkover}{NCORR} = $Total_ncorr;
+			$cumuloverallmeeting{$spkover}{NSUB} = $Total_nsub;
+			$cumuloverallmeeting{$spkover}{NINS} = $Total_nins;
+			$cumuloverallmeeting{$spkover}{NDEL} = $Total_ndel;
+			$cumuloverallmeeting{$spkover}{NSPKRERR} = $Total_nspkerr;
+			$cumuloverallmeeting{$spkover}{NERR} = $Total_nerr;
+		}
+		
+		my $display_Total_regs = $Total_regs;
+		my $display_Total_tottime = sprintf("%.3f", $Total_tottime);
+		my $display_Total_regsalign = $Total_regsalign;
+		my $display_Total_tottimealign = sprintf("%.3f", $Total_tottimealign);
+		my $display_Total_nref = $Total_nref;
+		my $display_Total_ncorr = sprintf("%.1f%% (%d)", 100*$Total_ncorr/$Total_nref, $Total_ncorr);
+		my $display_Total_nsub = sprintf("%.1f%% (%d)", 100*$Total_nsub/$Total_nref, $Total_nsub);
+		my $display_Total_nins = sprintf("%.1f%% (%d)", 100*$Total_nins/$Total_nref, $Total_nins);
+		my $display_Total_ndel = sprintf("%.1f%% (%d)", 100*$Total_ndel/$Total_nref, $Total_ndel);
+		my $display_Total_nspkerr = sprintf("%.1f%% (%d)", 100*$Total_nspkerr/$Total_nref, $Total_nspkerr);
+		my $display_Total_nerr = sprintf("%.1f%% (%d)", 100*$Total_nerr/$Total_nref, $Total_nerr);
+		
+		print FILEINDEX "<tr>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ensemble</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_regs</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_tottime</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_regsalign</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_tottimealign</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nref</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_ncorr</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nsub</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nins</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_ndel</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nspkerr</td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_Total_nerr</td>\n";
+		print FILEINDEX "</tr>\n";
+		
+		print FILEINDEX "</tbody>\n";
+		print FILEINDEX "</table>\n";
+		
+		print FILEINDEX "<br>\n";
+
+		print FILEINDEX "<table style=\"text-align: left;\" border=\"1\" cellpadding=\"1\" cellspacing=\"0\">\n";
+		print FILEINDEX "<tbody>\n";
+		print FILEINDEX "<tr>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Speaker<br>Cumulative<br>Overlap</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total<br>Time</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Regions<br>Aligned</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Total Time<br>Aligned</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>\#Ref Aligned<br>Words</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Corr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Sub</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Ins</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Del</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>SpErr</b></td>\n";
+		print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>Err</b></td>\n";
+		print FILEINDEX "</tr>\n";
+		
+		foreach my $cumulspkover (sort {$a <=> $b} keys %cumuloverallmeeting)
+		{
+			my $display_regs = $cumuloverallmeeting{$cumulspkover}{NLISTSG};
+			my $display_tottime = sprintf("%.3f", $cumuloverallmeeting{$cumulspkover}{TOTTIME});
+			my $display_regsalign = $cumuloverallmeeting{$cumulspkover}{NLISTSGALIGNED};
+			my $display_tottimealign = sprintf("%.3f", $cumuloverallmeeting{$cumulspkover}{TOTTIMEALIGNED});
+			my $display_nref = "-";
+			my $display_ncorr = "-";
+			my $display_nsub = "-";
+			my $display_nins = "-";
+			my $display_ndel = "-";
+			my $display_nspkerr = "-";
+			my $display_nerr = "-";
+			
+			if($cumulspkover == 0)
+			{
+				$display_nins = sprintf("(%d)", $CumulOverlap{$cumulspkover}{NINS});
+			}
+			
+			if($cumuloverallmeeting{$cumulspkover}{NREF} != 0)
+			{
+				$display_nref = $cumuloverallmeeting{$cumulspkover}{NREF};
+				$display_ncorr = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NCORR}     *100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NCORR});
+				$display_nsub = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NSUB}       *100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NSUB});
+				$display_nins = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NINS}       *100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NINS});
+				$display_ndel = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NDEL}       *100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NDEL});
+				$display_nspkerr = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NSPKRERR}*100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NSPKRERR});
+				$display_nerr = sprintf("%.1f%% (%d)", $cumuloverallmeeting{$cumulspkover}{NERR}       *100/$cumuloverallmeeting{$cumulspkover}{NREF}, $cumuloverallmeeting{$cumulspkover}{NERR});
+			}
+		
+			print FILEINDEX "<tr>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\"><b>$cumulspkover<b></td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regs</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottime</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_regsalign</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_tottimealign</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nref</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ncorr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nsub</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nins</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_ndel</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nspkerr</td>\n";
+			print FILEINDEX "<td style=\"vertical-align: center; text-align: center;\">$display_nerr</td>\n";
+			print FILEINDEX "</tr>\n";
+		}
+		
+		
+		print FILEINDEX "</tbody>\n";
+		print FILEINDEX "</table>\n";
+
+
+		print FILEINDEX "<center>\n";
+		print FILEINDEX "</body>\n";
+		print FILEINDEX "</html>\n";
+		
+		close FILEINDEX;
+	}
+}
+
+if (! $Installed)
+{
+    system ('cp', '-r', 'js', $Outputdir);
+    system ('cp', '-r', 'images', $Outputdir);
+}
+else
+{
+    extractJavaImages($Outputdir);
+}
+
+1;
+

SCTK/src/align2html/buildInstallVersion.pl

@@ -0,0 +1,69 @@
+#!/usr/bin/perl
+
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+use strict;
+
+my $exe = "align2html-main.pl";
+
+my @modules = ();
+my @file = ();
+
+open EXE,"<$exe" || die "Failed to open executable $exe";
+while (<EXE>){
+    if ($_ =~ /my \$Installed = 0;/){
+	push @file, "my \$Installed = 1;\n";
+    } elsif ($_ =~ /\#\#\# Begin Local Modules/){
+	do {
+	    if ($_ =~ /use\s+(\S+)\s*;/){
+		push @modules, $1;
+	    }
+	    $_ = <EXE>;
+	} while ($_ !~ /\#\#\# End Local Modules/);
+	my $modExp = "use\\s+(".join("|",@modules).")";
+	#print "Modes = ".join(" ",@modules)." $modExp\n";
+	### Insert the modules 
+	foreach my $mod(@modules){
+	    open (MOD, "<$mod.pm") || die "Failed to open $mod.pm";
+	    while (<MOD>){
+		push(@file, $_) if ($_ !~ /$modExp/);
+	    }
+	    close MOD;
+	}
+	### Reset the package
+	push @file, "package main;\n"
+    } elsif ($_ =~ /die "HERE DOCUMENT NOT BUILT"/){
+        push @file, 'my $here = "";'."\n";
+        push @file, "\$here = << 'EOTAR';\n";
+	open TARPACK, "packImageTarFile" || die "Failed to open the packed Image tar file";
+	while (<TARPACK>){
+	    push @file, $_;
+	}
+	close TARPACK;
+	push @file, pack("u", $_);
+        push @file, "EOTAR\n";
+
+	push @file, 'open UNTAR, "| (cd $outDir ; tar zxf -)" || die "Failed to UUDECODE/TAR"'.";\n";
+	push @file, 'binmode(UNTAR);'."\n";
+	push @file, 'print UNTAR unpack("u",$here);'."\n";
+	push @file, 'close UNTAR'."\n";
+    } else {
+	push @file, $_;
+    }
+}
+print @file;
+
+#	cat align2html.pl | perl -ne 'if ($$_ =~ /die "HERE DOCUMENT NOT BUILT"/) { system "cat DATA.tgz.uu" } else { print }' > align2html-combined.pl

SCTK/src/align2html/buildPack.pl

@@ -0,0 +1,29 @@
+#!/usr/bin/env perl
+
+# ALIGN2HTML
+# Author: Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. ALIGN2HTML is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+use strict;
+use warnings;
+
+my $str = "";
+
+open TAR, "tar zcf - --exclude CVS --exclude .DS_Store ./js/wz_jsgraphics.js ./images/*gif |" || die "TAR/UUENCODE Fialed";
+binmode TAR;
+read(TAR, $_, 90000000);
+$str .= pack("u", $_) . "\n";
+close TAR;
+
+print "$str";

SCTK/src/align2html/makefile

@@ -0,0 +1,25 @@
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+
+all: align2html.pl
+
+align2html.pl: align2html-main.pl buildInstallVersion.pl
+	perl buildInstallVersion.pl > align2html.pl
+	chmod +x align2html.pl
+
+check: all
+	@echo "No Tests for align2html.pl"
+
+install: all
+	perl buildInstallVersion.pl > $(DESTDIR)$(bindir)/align2html.pl
+	chmod +x $(DESTDIR)$(bindir)/align2html.pl
+
+buildPackFile:
+	buildPack.pl > packImageTarFile
+
+clean:
+	$(RM) align2html.pl
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/align2html/makefile.in

@@ -0,0 +1,25 @@
+SHELL = /bin/sh
+PREFIX=
+bindir = $(PREFIX)/bin
+
+all: align2html.pl
+
+align2html.pl: align2html-main.pl buildInstallVersion.pl
+	perl buildInstallVersion.pl > align2html.pl
+	chmod +x align2html.pl
+
+check: all
+	@echo "No Tests for align2html.pl"
+
+install: all
+	perl buildInstallVersion.pl > $(DESTDIR)$(bindir)/align2html.pl
+	chmod +x $(DESTDIR)$(bindir)/align2html.pl
+
+buildPackFile:
+	buildPack.pl > packImageTarFile
+
+clean:
+	$(RM) align2html.pl
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/align2html/packImageTarFile

@@ -0,0 +1,229 @@
+M'XL(`-QX"$H``^P\2VSD1G9*$`21$!D.L,@:>RI-9J;9ZH^Z6Z.QK1Y)UDCS
+MD5<>:4<RG(EWTF:3U4W.L,E>DJUN3G:`1>($"`(DU\#'/03(<6][VTO@`+DX
+MR,'W('O.+0CVDLU[KXID\=.2QI8=>Q$"HLCZO'^]]ZJZBLVU9\':]$7O63#T
+M];%E&T'S6;!PM5<+KC??W%AHM5NW;[77X;W37M_8H/)6Z];&[=NW%MJMC3<W
+M-J!THP7M-M;7UQ=8ZXKI*+TF0:C[C"WHS[SPO';!F'/#FE\OF&DE_[\EU]HJ
+M.[7L@+E>:!N<C4`<K,_9Q`V]B6%QD^DATQV'A?:(!\WEI>6EO+$PN,Z:K--<
+MO[V\=&IQYN@A!RAGW`]LSV4`7#_3;4?O.QR@+2]983C>7%N;3J?-J>Z$W'_A
+M^6[3\$;+2Y[/E%J3G]F^7E:C]#,Y$K7GC2/?'EHATXPJZX"%->!VBWU`#=F?
+M8$NV"WQ0HX#Y/.#^&3>!HSV?`\$F6V^R=KM)?5D_4GLR[0/>WYR#'ZACU>6E
+M0QU8'GFF/;"YN<DZ`.DV`;N-Y!US?^#Y(]T%$7MCD*7]0@]!.`&#8G;@`C"7
+MA^S>;.QX/O>7EX"`4\L;Z0&[[^ON<Z:[)GO7LUSVT'/,8.KYH06D#VS'.?:<
+M:`AB'O'0\DQFC\8.'W$7.0(@[^EA:-E\PAZ"$KE/"GP((F`J1>^"?DX,WQZ'
+M[('4*SNT^[[N1]#AV/?.;),'$D.PO-1@H<=,7Y\RQW9Y4`=I&J'N#AU\YHYC
+MCP-\&@O*H,/BU`XM!B/(`/&0'6!'!I09SP%`4)<PD1\%&'$=PU/1AGP6`F6/
+MCD[O;;*C,?>%,.ML,'$-(5?L"ARXAF6[0V;I9YQ!*PLTVN?<C97`S>4E`,H'
+M`]NPN6M$U`^'N@G4Z2YB#"R0-E+)O`$+O(D/$C,\DY,L#P_V[CTZ`2H.'QP?
+M8@$-)D<(#TU_X',.O0;A5/=YET7>A!D`U^>F'82^W9^$G-DAHEWS0.]D01&6
+M3%P3B`6*@5M_1,CQY<&C]]DA$`-U#[@+G#OL>-)W;`-H@0'L!IQI2$N5@>V,
+ML2:PA"E@[_M(S8FDAMWW``F)KLNXC<)97HI';:?9CE%*N'4&IJJ!.P`>?!*@
+MYP(6-Z(![\?CO5DFA919L%"7H%K>&"T`X`&S4U0\NIV`#R8.F`,T91\<G#X\
+M>O^4[3YZPC[8??QX]]'IDRY#4_*@EI]Q`0@M'L8<`Y9`WV$$9"\OO7?O\=Y#
+MZ+![]^#PX/0)DG[_X/31O9,3=O_H,=MEQ[N/3P_VWC_<?<R.WW]\?'1RK\G8
+M">>QD$&<95*.A4'C=@1CE9D\!.<F7.,3T"Z8R\0QI<5Q@]MGZ$3!8L;1*^A0
+M=SPP6QHVH2+++K,'Z*KK;.K;8#E@G@6U+B^E>JV#;S&:=;;Q-COEZ!G8L:,;
+MH,J3"79?7V_5V5TO"+'E>[N,06!NMQOM]=:;C+U_LELGOQL`<,7W#=U)T_.'
+M:\B1PP?AFD.L-*UPY"POK:ZA(,".(*@^&_9L**W3`Z=_`W"3]&!ZHGSDO0`L
+M\.#>8EM,,SUC@MZKZ>@1V!.[>9.%T9B#X)(:P]$!7\!6MM@U'"0#<"7FM6I7
+MX(T]`#.LY_L/3]_39G5F@W_^,_!",\"0@.E[9L1^_&/F3AP'^BX2C=!@IN"<
+M-6U0AA_NFL]`:FX(\`YS>&578`?)I\XK`M3-FU"UF$#2QV/NFGN6#<:1!9%M
+MF?))<>DQN$)^;@?-5ME2>FG5:C/@X0GD-^%=/D!C/0^.+?ON>2[ZUXGN0.@9
+M(M`242]*W0+JA%UZ0D'`8UZ*8.7SA8=&`8`2.`DSF'@@W*GMFMZTZ:&GEWW`
+M;D07B3#!%81@E<WWO!='8]VPP1T44+[,6<K8#?<]0Y-&@L.M.77-9D(%#35-
+MTKE#XQ&-_?$8^K#-Y)W$$K\`;95*.:X]]VS?&\7H<)R@71;19C2)L&<Y;6K4
+MQW#/1"W:[EP57I)\>Y`"9?&3:KK:[!78/.`QE^5PBX-+NR98N^>:UQ)_<24B
+M/X`6%U$3&VEM*\%P>009^SFG]>CYOGV&7BD")UYG5KX;(*_<,>TS1H:\=6WL
+M!39VW-3[@>=`!.U6:CAJ*^A[-RNL!N938Y7Q+"X/O3$51]GBJ6V&%E5,LQ46
+MQY28:JQLC0&9UR:F8UJK7JE-:U`%_RWZWZK&K?JZ\7SH8\AI&!ZDKP1)2!5?
+M&;725L28W6&5K@>9PL#QIIN6;9K<K8!"*Y6J`-:]MGWGAVO`_O9<V8%5G2^]
+M&S<JM11_K0*$SN@>T7U*=POO\U`<^^%5:*CO^9##-1)%2<%#.NC8)LN+B56^
+ME&9;WU*-'D;^1;*F7(`IPMFZEDCG&DN%(XJC3#$)1U1,,Q5"2*+&RM3TA\2F
+MJ,IJB'@A<K9_Z"K\H!?W^9"C$V9K-VYH'_YI]VFMVKWH_]JPJP@C]G!"!CX/
+M)[Z;>**FS\>8NVE((^&J$[6J)>;TT"U:9E&U@N?K[2Y9W?5.%ZWL^GI76-7U
+M6UUI3=<W@'G2(W!>+5$ET`CCYIO,0%<=D-=O)2/Q>OL"UL!*;5>;M<%,X6_6
+M@?^=))9`UM=FVU`*)8M81#&]-^M@0.YTDX(("R)1("K;]"S*Q3-`VL*NHH)>
+M1(^7,E4P*<QW&D@+Y*];.,&WFB`=+>HTHG:U'F<"2"O!Q9((9@$^YJ<14@JT
+M[[!&&P9IFY)FX@$`;T,:&668&&,G,[ISAZ`LCOT)O$-A`]M#:5444V'#%/KT
+M"#TQ,(6,`::CT+;1J`+B%@(7T!=K-=&&D(^32EDK'(`(E9[P#K,&/K2KHM=B
+MA*Z!^)(%8RP`"N6K0@8);W&1.S"ODJVHG(I+$'4`4ZT=XR);H,Y%R<Q*)1,5
+M)!,)R9!""#4R#7J_@PI2A)((+%(%%LLD)ZFX6&5A5JL1#T"]%S6B6B*NHKSR
+M`E/)BT46LZT@FPLGTU'<5<)HS"2$=6+*J&&,Y3)"*."_K%0\(9:HX4755Q5!
+MQFS*F8O!@_%$*G,OYWB3SOYOKC]!P$$\IPE"WWO.N_/<3%S(6CB+"QKK.3]!
+MK"(P+5@U9ZM$7O`CR,_:-3-:-:,UZ`T5U2KXGX86;&^WJU!:96O`FE"7Z+VB
+M!8U;0"<!,+CM:+T`IQ+T3O$$"ZH0Y-M9PV0)!4$B>]V,)460@K6.6'[X9GK-
+MFF[6@8.ORW/J)CI/B:_H/9'R"]0<9=6,^@4UH[9!S;&.9PVIYNAK4+/H6:KG
+MJXX!-1!@]VJC02^0X4`WOVGQ(":MD]#V]48$Q(\A017-%4<%0M$1."X3&?:]
+M,/Q-C`U8:?I3C`O^9'Y`^`(>]$*W28C\J5SL45=@VFE**6A;@7_GN=@+O.<<
+MAYDX[I=)S#N?G"O).K-RB:Y(+GD)E,LIB527$]`KR*4X:([.-)S2U1G,_.J,
+MYGUU)F9]RBJK#EQ0'<2/.NO#FVB"K]!DZIEQ@YM0;]&K)IK<;%=KU,A`>T=<
+M-;W.##1Z0%GKQR.A)08"O7NRP$M*=*1`TU?U*BJ0]9$$K;_:KTI]TB*XINL4
+MW[1V0^NC2C%6]:E_2/7]/M:#RG5]51--&D+QN'Z",I(:CV*=I&$W9'<R^H<"
+M4$&_#X!H6-76I<Y#J@!<6+JJ@3)SSAE_A]+"2\!#V]1U`29JM.?#CSG2T$Z)
+MJY@8-#_,8\2;!6\8,5+CF][L8.YHW>R4)4-'9S\P`\V8H;[JK#&KH5N>U4#=
+M\(;^&8"U:Q:^*D8_MS?Z9&@OND-/T'BALV+CYX#)$V$).+0*)N"HF9@2A$HB
+M),BSL27D%V5D%U!%HH%&(\H$(#5>&;,&&371H\LYL.81/"0M76R/V]=B%L[O
+M5#9@8?ISB2%;,HF@IF@\08/"F.BC%/P?#/6O<F!WDQ4:2&QQN*&I4YK;J>+/
+ME\1%8Z,=SZ`$%[*@FHFJ18]$+D/$C)E39^.9C[<0;WV\3=4($I4$#\6CQ,-N
+MKD^9ZU34\9)S*^<`+7<LK^99<JY%]2W+22HY;;1SB20(!NVGUJYIP<UTB<5*
+M)@_9-!&!6'.`!&EG(#L+,)-J4FL5P[F>I0'M:U/%OX!W\:2C@JK4Q61\3";3
+M3962<:KS',U<3R,`Y6>,&6<3%/U,SM'HR$HC0.;F]RG/F7HT.,4PT6#8M,D$
+MJM(5+/;ZZ=@OK9>#IH>BZ8E1TQ,#OJ>O]N20[XDQW^NO]I)1#S/24#1+!WXO
+M&?F]OH0E!G\O&?T]$FY/">QR?$(SET_9KN_KD28SO5E84MJ#T9LM[DH0'[:>
+M(C?R-12O_6[<2;SW^L*1?O7CGDB"OZ;#W6%H/54L$:F#/Z6F8)9?MZ_X4L1^
+M^3&4R+X7E:X%]%2=)%ZF)Z31(W'T<HHAXZL)ZXM%T)NEM6@2>%,8ZT72.(HN
+MKA=>G@!42"]FLZ>H9`Y)R;C`QGG%O`*A.=J$)N2(B_)0I39ZJ3IZT5R<C?;3
+M1J.XYO$R72XJ";\]>I9#CTIPE*=&EL1H`HQ;SC2$9-,@AG]WF`/_:K7<Z)1[
+M80BN_32_>Z=@.++9'8;D0$J!YDSO7I13YDQ0![6Q&"Z;W_;BZ"-^QH$@VU/6
+MX3-!2$0A040B[5102:&ZLI0?8@5"LZ%GI@22DB0WWSZ?YY9W*492E8>+HV`9
+MU%<E8\[TF):4OJ(I<IPVOW+.W,DDS;J.^_^@]/P4NG/I'+J3FQUGEYZ^:7-D
+M8/\R\^-.66@Z9S[8F3,A)'29R6!A\24_I&'HQK/>DCEO=I&HU`P?X[:3PDZ/
+M.=.\P@H0=`GR,U`@B.J".)_-]:J!C4]K05E/K`U$ST90/DE%>L5:;.GN%-R(
+M?@@>558+6E(\:;4D4ORO665-)*T7-($&276>W&?!\+[GAIE-:,>'NP>/0+25
+M`=0TIF(7A(N;_AW:^B1:W3TZW,\WZGN.J30Y.-T]/-A+&M%>C4T[U!W;*+3J
+M27AJ1[F+!FLR:'L)X#P`P2':!G(ED]F$QVZ1^1.RG`S[^T>GI_>0%$H$)#C1
+M+@48]RN"C$]":#8NRKBF"CK@X1YM"1)P[LM>6D7WAY4ZJRB;AF"L^<-FZ!UZ
+M4^[OZ0'BJLA9?0PK(2I&K\T2GZ2,BR2\X%!=`<-J5Q6_DK$7:)K\EM!-JW%4
+M4Y4("=UL1S1XJHTMO\2=S1KMW'2@%&]GOQQKIKR`<ZXW*T-1CN!2X%]*;2<*
+M./9M-Z1S*8H.0&]9+8R55E"9J$)N4\W+A#P-(:=-BPIE8I^7@+*3[IMBFW('
+M6$[N&6BT4WY'[)N##@0AWK,([_38+>-1#J2$O8$^`O_WHHY;K[)\#L+[.FY?
+MA19=I?`$-UP'+S)%(<9S`(!I(R(0/J>('W6!!Y4<H<!,F51J:OO"IR<TI6GO
+M5HM!NGMGEJ3<C+)?5BU:B3:#S`L`T1V>:VWQ!O^K1?+PW)$TE1P9TNLKTBGL
+MWYW/+1[+RD',PE+%(JJ[F3K)2<HP,9%YG>',`0I;97PAC'OB_)0J]:,SW;D<
+MJT<7<(J"RR'`HCP"-?54L<0YIS;%I*1=3=).S4H+L*'(/+6I6)U-5FR3Q=KS
+M,L]"ZEGRT\RKIJ,7YZ,7)Z1BS=4$@9A6XDNFX-39-W;!-9>ERE4IG%#)`A.S
+M0-D#=';!.DMI-@M`LC_QJ,..1":G/T)R)1,VV:@FD]1LNR^XWEE,I$LSZ4M,
+M\Z9E/ZO$HVIME17/=-8O/M1)Y^V>Z6=Z((YQ)GD,'KYUF6[JXY`.A,5'T(;F
+MP4@?\ON`C)LQ.O2TZK%7Q^[C^=CXT"-T?;#/VLVWFK>@Z<2EL\'*P;"^-PFY
+MX^")V+6A2>?`3A_N/OK^"1Y3^[[M!R%WV8EA39P7A`KI&,>H[1D/5L3103!*
+M/!`;<,'"P'9-/,,9<L-R[1]-N.`!.,-3F'##XW9].@?H>V=XHA-$Y(.X1GWN
+M4T?$X_,SVYL$`G0*O2X.BD9L$L1-`\<;<Y(IL">@6]P9(Q.Z^0S/1ZOD!5@.
+MB,T)2$AGKFV`_5`#V_-[?!8")7J3X;&XU&66!`<\O!CUP-F*A^Q^"+L;/T7)
+MT\AVP9I&^BPM$IM&TE?:.Y*\@ABA'NYJ41ATTQ4JEU(2HD.&&%E)":=;96)'
+M=QP!%I&$I`N&(`*,)!5+*8KCPE5;+%RY90M7<1]:?4+H<OTIAXC65,HZ;9,\
+MXDY9.N1"S,N$&*Q$P%V(#'>VA"1A>J721'LN0%<'(*>2%?5%6Y2+!?64Q5:!
+M176];<7.K:JA8A!ZLF"))9T4[OP%3-G3+O2TLSUI=T\B".@4+XO1?B"EHA-7
+MT.:T-C`1=?)K@"FL60:6V&BDU'2>%NEG$O!V$7">EA1PGOS.7)0*,>U+$&-`
+MRFJ[M"(D2MG:6G*(7WP-@,[P9[XCD`H((S3MWJ/?9",A+<G58FPW'Z*1U&JX
+M)JUL)82PA_U6(5CB!BO<R"AW5>'!EOAG7U5H:*_"2F-TVRKJK2O%G<@IAM0,
+M/#_4T*\-N=\##S_6?5XMMWOR*F#Z6[&&U=B<4/:4\LKT'3+SAE*;;L`OF;)A
+M$@N39O38B@L-\9?K8I)]N2-;D'.%O!&,=8-ONMX4)O_R--3<XUCSSF,MBE42
+MF#W93D3U3)E6I<>\9+O`?L'3`U8TS5+;E!_`P@9*EU"<>=J6G4`03'97CEZI
+M248J0%J5JT)D,T5N\=@VGK.[CC<:08B#*+CK.-Z4@ISX?(/XC@`2':<2>-(T
+M_6@#Q5,,E8X]I//YRTO4H.^%%N3MOOT"F(:T)6(:;PZ;XGL<5>IU!H'9-J@.
+M#[[3=P)T%70:1+/TEQI!NLI-I'P1F\B?1OK"!B'/7M;DYI5,%Q!.@RB$>D%J
+MK?)ML:14&91-JGJP1\,3WTA4@?M,F/Y%E/!EA8Y2F^:J2@Y()BQ7[@#I+/`-
+M<?Y/\"'.!Q;/$A8/$>*3IN\PK<(J-;TJ3D)B30I_KB`-A^N^*L2BO+;8M6O)
+M9/'<0\SQ?)\/Y#'F+"ZYP*]BB]?Z<?J$.X$B&$E1_YPU$+^&7:*P9J@K!&5-
+M^N<T<2[=)(?H9=DJJM9.5\_EJI=6@7%LZJY>'W*7G^EU2.?/.+J:>J"[02.`
+M7'U0J3/A4)H#WQOM6;J_![,+K35;AT`$]P[>WVS1_:UJ75GM2O'%*[Z5/Q+?
+M@$K7QI7SX*)@ZM)/:7"G/5SX>8-NLM-K11.?0H`J^8$#\80?7*A6DR];R!/[
+MR>^]-GU8HA(0&Q7LLV*;TC;&NDW+?^);!]A1W2<3FP_;RGQ[84>:6.:S!%#^
+MH6T^C3^=(4+\9MIOR,-[8K)Z-SHP2T%DFVA(9`Y:_%&.1<6&<64A(12_^%"P
+M^>I.V4#8E')?S`A""'8G_2`#M".I[R0?+Y`E@'I'^8@`E(H7,L'R961MH(-T
+ME=]IE%\6<CF4-JM'V?.RV@P3R2J3^^8U#4]'1=75=@+O*K__U5RST74':Y0`
+M]<SGS:$]N$H$"^+3:/B-M_+OO[5:&ZW;^>^_==H;___]MZ_C>G!P_ZVW]=<6
+M7EOXKX6%7\Z_/O_\\\\^^^S33S_]]:__YQ=T_3Q[_2R^_I&NG_[TIY]\\LG?
+MT/7QQQ^_?/ER0M>0KH\^^NC)DR<_H.LN75?"S,JO?N>W%A9^?Z&.+\C3PN\:
+M[_S5^&']>W_W3^_N7_OKT[]]_9___2</_YC=^Y=?OO/Z&]_YP]?^\HVWW_CM
+MG_SY?_S;KUZ_\7M_\=&__GS\SC_\]]V?_;WYG]>CO3_X[G??^-[T1F5E_SO_
+MVTX=HB`01'$8]P86FZZXKLS:!8M)4/0P1N]B,HH6XP2C8<($+S#!8#`8MDR8
+MJ,,#PPJV106_WPG>"]\_:26=;J^?-Z?U+%/YNCUKI$JEJ_EV-QG71I5<_Q4O
+M_2^6/]+_8$C_GU#J__Y>",%[7Q1%[/\FKF67I[-PSL7).`EKK3'F*`Y":QV7
+M8B\VHI)GZ!\```````````````!_Y@%8KS5]`'@`````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+9````````````````````````````````````
+

SCTK/src/chfilt/chfilt.pl

@@ -0,0 +1,1042 @@
+#!/usr/bin/env perl
+
+### This changes the function of a runtime warning to dump a stack trace
+#use Carp ();  local $SIG{__WARN__} = \&Carp::cluck;
+### This dies on a warning after the stack dump
+#use Carp ();  local $SIG{__WARN__} = sub { Carp::cluck(); die; };
+#use Carp ();  local $SIG{__DIE__} = sub { Carp::cluck(); die; };
+
+use strict;
+
+# File:  chfilt
+# History:  
+#    version 1.0  Released 950921
+#    version 1.1  Released 951006
+#        - For Spanish, it now converts a set of predefined words to 
+#          interjections.
+#        - Corrected the order of deleting "--" from the transcripts
+#        - Normalized improperly formatted words in (( )) markers
+#    version 1.2  Released 951025
+#        - The "add_spaces_for_japanese_punctuation" function was splitting
+#          words that contained vowel elongation characters
+#        - Japapese quotation characters are now removed
+#    version 1.3 Released 960418
+#        - changed the exit codes to be 0 for successful completion, or
+#          1 otherwise,  (thanks Barb).
+#        - added English support	 
+#        - modified the main loop to first read in the entire file, then normalize
+#          it, by deleting empty lines and concatenating continued lines together.
+#        - added the -i option
+#        - added the translation flag option "-t"
+#        - added the option -I to not make alternations for interjection words.
+#        - added the equivalence class option "-q"
+#        - added the hyphenated word alternation flag "-a"
+#    version 1.4 Released 970711
+#        - added a the pem output format
+#        - added the -u option
+#        ????? is there an unresolved issue for Japanese ????
+#    version 1.5 Release 971010
+#        - Spanish: installed hesitation filter, removing interjections.
+#        - German:  installed hesitation filter
+#        - Arabic:  updated hesitation filter for *romanized script*
+#        - removed the -i, -u and -I flags
+#        - removed all code that made altenations with NULL
+#        - added the '-d' optionally deletable flag
+#    version 1.6 Released 
+#        - For Arabic scoring, translated the arabic script for an apostrophy to a hyphen
+#    version 1.7 Released April 7, 1998
+#        - fixed a problem with Out-of-Language markers within an unintelligible markers
+#        - fixed a problem with unintelligible markers within an OOL markers
+#        - added a filter to remove empty optionally deletable flags.
+#  Version 1.10
+#       JGF updated to new version or perl 
+#  Version 1.11 
+#       JGF added support for Miss. State transcript
+#       JGF modified the PEM output to pass through the duration times as strings
+#  Version 1.12
+#       JGF commented out part of the MState code that converted 'OH' and 'AW' to hesitations
+#       JGF Correctly handle possessive ancronyms in MState transcripts
+#  Version 1.13
+#       JGF Added processing for spoken letters.  Previously they did not exist.
+#       JGF Added code to remove '^' for proper names
+#       JGF Corrected the line header extraction code to handle empty transcripts
+#       JGF Added -i and -k options
+#       JGF Added code to remove 'aside' sgml tags
+#       JGF Added -p to pad segment times with silence and -s to smooth segmentations
+#       JGF Added -E to exclude overlapping speech          
+#  Version 1.14
+#       JGF Added -c to read contractions
+#       JGF disabled all old arabic character-specific processing
+#       JGF Uses Getopt::Long now tp process the command line arguments
+#       JGF Modified the output for stdout to be "-- -".  The new Getopt
+#           is interpreting '-' as an ambiguous arguement.
+#  Version 1.15
+#       JGF added Arabic processing steps for 2004 arabic transcripts.  See the Eval
+#           plan appendix for Arabic.
+#       JGF add use strict pragma
+#  Version 1.16
+#       JGF added code to check for unannotated, non-posessive contractions IFF contractions
+#           are to be processed.
+#  Version 1.17
+#       JGF Added -C
+#  Version 1.18
+#       JGF Added two new hesitations
+
+my $Version="1.18";
+
+my $usage = "Usage: chfilt <OPTIONS> infile outfile|-- -\n".
+"Version: $Version\n".
+"\n".
+"Desc:  chfilt converts a Callhome transcription file, 'infile', to STM\n".
+"       format, written to 'outfile'.  If '-- -' is used as output, the output goes to stdout.\n".
+"\n".
+"       OPTIONS :==\n".
+"       -l <language>  Identifies the language on the transcript\n".
+"                      Current chfilt can handle:  mandarin\n".
+"                                                  spanish\n".
+"                                                  japanese\n".
+"                                                  english\n".
+"                                                  arabic\n".
+"                                                  german\n".
+"       -m             Read Miss. State Transcripts\n".
+"       -b b_time      Output segments After 'b_time'.\n".
+"       -e e_time      Output segments ending before 'e_time'.\n".
+"       -d             Output text with optionally deletable tags.\n".
+"       -q Word1=Word2\n".
+"                      Add alternations to the output such that Word1 and Word2\n".
+"                      are equivalent\n".
+"       -t Word1=Word2\n".
+"                      Translate Word1 to Word2\n".
+"       -a             Convert all hyphenated word to alternations whereby the \n".
+"                      two alternations are the word with hyphens deleted, or\n".
+"                      the word(s) with hyphens converted to spaces\n".
+"       -i             Add inter-segment gaps to the stm records.  This option\n".
+"                      requires -b and -e to be used\n".
+"       -k             Keep lines with no lexemes in the transcript rather than\n".
+"                      ignoring them\n".
+"       -o fmt         Set the output format to: \n".
+"                        stm  -> Output STM format (default)\n".
+"                        pem  -> Output PEM format\n".
+"       -s time        Smooth the segmentation (i.e., merge segments) that hava gap\n".
+"                      intersegment gaps less than 'time'\n".
+"       -p time        Pad segment boundaries with 'time' seconds of silence if\n".
+"                      the time is available\n".
+"       -c             Don't expand the contractions if they are annotated\n".
+"       -C (numbers|letters)\n".
+"                      If 'numbers', change channels A and B to 1 and 2 respectively \n".
+"                      If 'letters', change channels 1 and 2 to A and B respectively \n".
+"       -E             Exclude overlapping segments in the output STMs\n";;
+
+##########3  Globals 
+my $Language;
+my $OptDel;
+my ($NewBeginTime, $NewEndTime);
+my $Format; 
+my $KeepEmptyLines;
+my @Translate;
+my @Equivs;
+my $AltHyphens;
+my $ReadMSstate;
+my $ExcludeOverlap;
+my $PadSilence;
+my $SmoothSegments;
+my $ExpandContractions;
+my $AddInterSegGap;
+my $HES;
+my $InFile;
+my $OutFile;
+my $id;
+my %TranslateAArray;
+my %EquivsAArray;
+my @Trans;
+my $ChannelsTo = "asis";
+
+use Getopt::Long;
+$Getopt::Long::ignorecase = 0;
+my $ret = GetOptions ("s:f", "p:f", "l=s", "b:f", "e:f", "o=s", "t:s", "q:s", "a", "d", "m", "i", "k", "E", "c", "C=s");
+die "\n$usage\nError: Failed to parse argements" if (! $ret);
+if (defined($main::opt_o)) {
+    $Format = $main::opt_o;
+    if($Format !~ /^(stm|pem)$/){
+        print "\n$usage\nError: Unrecognized format type --> $Format\n\n";
+        exit 1;
+    }
+} else {
+    $Format = "stm";
+}
+if (!defined($main::opt_l)){
+    print "\n$usage\nError: Language option is required\n\n";
+    exit 1;
+} else {
+    $Language = $main::opt_l;
+    if ($Language !~ /japanese/ &&
+	$Language !~ /mandarin/ &&
+	$Language !~ /english/ &&
+	$Language !~ /spanish/ &&
+        $Language !~ /arabic/ &&
+        $Language !~ /german/) {
+	print "\n$usage\nError: Language not recognized\n\n";
+	exit 1;
+    }
+}
+
+if (defined($main::opt_b)) { $NewBeginTime = $main::opt_b;            } else { $NewBeginTime = 0.0}
+if (defined($main::opt_e)) { $NewEndTime   = $main::opt_e;            } else { $NewEndTime   = 999999;  }
+if (defined($main::opt_k)) { $KeepEmptyLines= $main::opt_k; } else { $KeepEmptyLines = 0; }
+if (defined($main::opt_t)) { @Translate    = split(/:/,$main::opt_t); } else { @Translate = (); }
+if (defined($main::opt_q)) { @Equivs       = split(/:/,$main::opt_q); } else { @Equivs = ();    }
+if (defined($main::opt_a)) { $AltHyphens   = 1;  $main::opt_a = 1; } else{ $AltHyphens = 0;$main::opt_a=0;}
+if (defined($main::opt_m)) { $ReadMSstate  = 1;  $main::opt_m = 1; } else{ $ReadMSstate = 0;$main::opt_m=0;}
+if (defined($main::opt_E)) { $ExcludeOverlap = $main::opt_E } else{ $ExcludeOverlap = 0}
+if (defined($main::opt_p)) { $PadSilence   = $main::opt_p } else{ $PadSilence = -1.0}
+if (defined($main::opt_s)) { $SmoothSegments = $main::opt_s } else{ $SmoothSegments = -1.0}
+if (defined($main::opt_c)) { $ExpandContractions = !(defined($main::opt_c))} else{ $ExpandContractions = 1}
+if (defined($main::opt_C)) { 
+    die "Error: Unknown channel conversion '$main::opt_C'" if ($main::opt_C !~ /^(numbers|letters)/);
+    $ChannelsTo = $main::opt_C;
+}
+$AddInterSegGap = defined($main::opt_i);
+die "$usage\nError: -i requires -b and -e to be used"
+    if (defined($main::opt_i) && (! defined($main::opt_b) || !defined($main::opt_e)));	
+
+if (defined($main::opt_d)) {
+    $OptDel       = 1;  $main::opt_d = 1;
+    $HES = " (%HESITATION) ";
+} else {
+    $OptDel = 0; $main::opt_d = 0;
+    $HES = "%HESITATION";
+}
+
+#### The main functions arguments:
+if ($#ARGV > 1) { print "\n$usage\nToo many arguments \n\n"; exit 1; }
+if ($#ARGV == 0) { print "\n$usage\nOutput Not Specified\n\n"; exit 1; } 
+if ($#ARGV == -1) { print "\n$usage\nInput and Output Not Specified\n\n";
+		    exit 1; } 
+
+$InFile=$ARGV[0];
+$OutFile=$ARGV[1];
+
+if (! -r $InFile){
+    print "\n$usage\nInput file $InFile does not exist\n\n";
+    exit 1;
+}
+
+if ($OutFile eq "-"){
+    $OutFile = "";
+} else {
+    $OutFile = " > ".$OutFile;
+}
+
+($id=$InFile) =~ s:^.*/::;
+$id =~ s/\.[^\.]*$//;
+
+# Check the translation classes.  If they exist, make an associative
+# array to house them
+if ($#Translate >= 0){
+    foreach my $tlate (@Translate){
+	$tlate =~ tr/a-z/A-Z/;
+	if ($Language eq "german" || $Language eq "spanish") {
+	    $tlate =~ tr/\340-\377/\300-\337/;     #Upper-Case accented letters
+	}
+	my @tl = split(/=/,$tlate);
+	$TranslateAArray{$tl[0]} = $tl[1];
+    }
+}
+
+# Check the equivalence classes.  If they exist, make an associative
+# array to house them
+if ($#Equivs >= 0){
+    foreach my $equiv (@Equivs){
+	$equiv =~ tr/a-z/A-Z/;
+	if ($Language eq "german" || $Language eq "spanish") {
+	    $equiv =~ tr/\340-\377/\300-\337/;     #Upper-Case accented letters
+	}
+	my $alt;
+ 	($alt = $equiv) =~ s/^/{ /;
+	 $alt           =~ s/$/ }/;
+	 $alt           =~ s:=: / :g;
+	foreach my $eq (split(/=/,$equiv)){
+	    $EquivsAArray{$eq} = $alt;
+	}
+    }
+}
+
+open(FILE,"< $InFile") || die("cannot open input Callhome file $InFile"); 
+#binmode FILE;
+
+my $hubscr_path = "../hubscr/hubscr.pl";
+die "ERROR: 'hubscr.pl' cannot be found, please edit 'chfilt.pl line 267 and set hubscr.pl path."if(! -e $hubscr_path);
+
+#open(OUTPUT,"| sort -t \\   +0  -1  +1  -2 +3nb -4 $OutFile") ||
+open(OUTPUT,"| perl $hubscr_path sortSTM $OutFile") || 
+    die("cannot open output file $OutFile"); 
+
+## make a place holder for the LUR reports
+print OUTPUT ';; CATEGORY "0" "" ""'."\n";
+print OUTPUT ';; LABEL "O" "Overall" "All Segments in the test set"'."\n";
+
+# read in the file, deleting blank lines:
+while (<FILE>){
+    s/\[\[[^]]*\]\]//g;            #remove comments              '[[words]]'
+
+    if ($_ !~ /^\s*$/ && $_ !~ /^#/){
+	if ($ReadMSstate){
+	    s/^\S+\s+//;
+	}
+	push(@Trans,$_);
+    }
+}
+close (FILE);
+
+# normalize the transcript so that each turn is on a line by itself
+for (my $n=0; $n<=$#Trans; $n++){
+    if ($Trans[$n] =~ /^\s*\d+\.\d*\s+\d+\.\d*\s+[AB]\d*:/){
+	my $orig = $Trans[$n];
+	if ($ExpandContractions){
+	    ### Scan the transcript looking for un-annotated non-possessive contractions
+	    my $ct = $Trans[$n];	    
+	    ### delete all tokens annotated as contractions
+	    $ct =~ s/(<contraction\s+e_form=\"([^\"]+)\">([^ \t\n\r\f\)]+))//gi;
+	    foreach my $ctt(split(/[\s\,\.\?]+/,$ct)){
+		print STDERR "Warning: Possible non-annotated contraction \"$ctt\"\n" if ($ctt =~ /\w'\w+/ && $ctt !~ /\w's$/);
+	    }
+	}
+	while ($Trans[$n] =~ /(<contraction\s+e_form=\"([^\"]+)\">([^ \t\n\r\f\)]+))/i){
+	    my ($tag, $rep, $token) = (quotemeta($1), $2, $3);
+	    my $exp;
+	    ($exp = $rep) =~ s/\[[^\[=]+=>/ /g;
+	    $exp =~ s/\]//g;
+	    $exp =~ s/^\s+//g;
+
+#	    print "Found $tag $rep $token '$exp' ------------ in $Trans[$n]";
+	    if ($ExpandContractions){
+		$Trans[$n] =~ s/$tag/$exp/;
+	    } else {
+		$Trans[$n] =~ s/$tag/$token/;
+	    }	    
+	    die "Error: unable to expand the contraction\n" if ($orig eq $Trans[$n]);
+	    $orig = $Trans[$n];
+	}
+	die "Error: Contraction not handled $Trans[$n]" if ($Trans[$n] =~ /<contrac/);
+    } else {
+#	print "no match $Trans[$n]";
+	if ($n == 0){
+	    die("Illegal format, first line does not have a begin time, end time and channel");
+	}
+	chop($Trans[$n - 1]);
+	$Trans[$n - 1] = $Trans[$n - 1]." ".$Trans[$n];
+	splice(@Trans,$n,1);
+	$n--;
+    }
+}   
+
+#print @Trans;
+
+my @STMS = ();
+
+### Normalize the text 
+foreach $_ (@Trans){
+    #match the lines header
+    chomp;
+    die "Unable to strip the line header of '$_'" unless s/^(.*[AB][0-9]*):\s*//;
+    my $line_header = $1;             #Store the header in a variable
+
+    tr/\011/ /;                    #change all tabs to spaces
+    s/ /  /g;                      #multiple spaces go to one
+    s/^/ /;                        #add spaces at begining of line
+    s/$/ /;                        #put a space at the end of each line
+    s/\s\(\(/ (( /g;                 #add spaces around unintelligble markers
+    s/\)\)\s/ )) /g;                 #add spaces around unintelligble markers
+    if ($OptDel == 0){
+        s:<foreign\s+language=[^>]+>([^><]+)</foreign>:$1:gi;
+    } else {
+        s/\(\(\s*<foreign\s+language=\"[^"]+\">([^><]+)<\/foreign>\s*\)\)/\(\($1\)\)/gi; 
+        s/<foreign\s+language=\"[^"]+\">([^><]+)<\/foreign>/\(\($1\)\)/gi;
+   } 
+#print "$_ here\n";
+    if ($Language ne "arabic") {
+       tr/a-z/A-Z/;                #Upper-Case all letters
+       if ($Language eq "german" || $Language eq "spanish") {
+	   tr/\340-\377/\300-\337/     #Upper-Case accented letters
+       }
+    }
+    s/\{[^{}]*\}//g;               #remove non-speech vocal noises         '{words}'
+
+    ### LANGUAGE Specific Normalizations
+    if ($Language eq "arabic") { 
+	my $hes;
+	if ($OptDel == 0){
+	    $hes="%\330\252\330\261\330\257\331\221\330\257";
+	} else {
+	    $hes=" (%\330\252\330\261\330\257\331\221\330\257) ";
+	}
+        s/[% ]\330\243\331\207 /$hes /g;           ## %>h
+        s/[% ]\330\245\331\212\331\207 /$hes /g;   ## %<yh
+        s/[% ]\330\243\331\205 /$hes /g;           ## >m
+        s/[% ]\330\243\331\210\331\210 /$hes /g;   ## >ww
+        s/[% ]\331\207\330\247\331\212 /$hes /g;   ## hAy
+        s/[% ]\331\205\331\207\331\205 /$hes /g;   ## mhm
+     
+        s/%*(\330\243\331\207\331\207) /$1 /g;         ## Back Channel
+        s/%\330\243\330\254\331\206\330\250\331\212 / /g;     ##  Foreign lexeme
+        s/%\330\252\330\257\330\247\330\256\331\204.*%\330\252\330\257\330\247\330\256\331\204\\ / /g;   ## End cross channel
+
+        ### Remove non speech markers
+        s/%\330\265\331\205\330\252\\* / /g;    # silence
+        s/%\330\245\331\206\331\202\330\267\330\247\330\271\\* / /g; # pause symbol
+        s/%\330\266\330\254\331\221\330\251\\* / /g;                 # noise symbol
+        s/%\330\252\331\206\331\201\331\221\330\263\\* / /g;         # breath symbol
+        s/%\330\263\330\271\330\247\331\204\\* / /g;                 # cough
+        s/%\330\266\330\255\331\203\\* / /g;                         # laugh
+        s/%\330\243\330\265\331\210\330\247\330\252\\* / /g;         # people talk
+
+#	## The LDC Transcripts have a special case encoded in them for
+#	## foreign words with an attached definite article.  The next few commands
+#	## move the definite article into the foreign tag.
+#	s/\s(\S+)(<\S+)\s+(\S+)/ $2 $1$3/g;
+#	## translate the arabic signle apostrophies to hyphens
+#	s/\254\s/- /g;
+#	s/\254$/- /g;
+#        s/\s\254/ -/g;
+#	## Arabic Semi colon
+#	s/\s\273/ /g;
+    } elsif ($Language eq "english") { 
+	;
+    } elsif ($Language eq "japanese") {
+	$_ = &add_spaces_for_japanese_punctuation($_);
+	s/\241\242 /, /g;          # Change the Japanese coma
+	s/\241\243 /. /g;          # Change the Japanese periods
+	s/\241\244 /, /g;          # Change the Japanese periods
+	s/\241\250 /; /g;          # Change the Japanese semi-colon
+	s/\241\251 /? /g;          # Change the Japanese question marks
+	s/\241\252 /! /g;          # Change the Japanese exclaimation marks
+	s/\241\327 / /g;           # Change the Japanese quotation mark
+	s/ \241\326/ /g;           # Change the Japanese quotation mark
+	s/\241\343([\000-\177])/<$1/g;      # Change the Japanese '<'
+	s/([\000-\177])\241\344/$1>/g;      # Change the Japanese '>'	 
+	# Convert Japanese col and dia flags to alternations
+	s/@([^@\[\]]+)\[\[([^,\[\]]+)\s*,[^\[\]]+\]\]/ { $1 \/ $2 } /g;
+    } elsif ($Language eq "mandarin") {
+	while (s/(<[^_>]*)_/$1 /) { # Removes underlines from OOL items
+	    ;
+	}
+	s/@([^@]+)@/$1/g;           # Removes obviously the @'s !!!
+	s/\*([^\*]+)\*/$1/g;           # Removes obviously the *'s !!!
+    }
+
+    ### Call Home filtering (General)
+
+    s/\-\-//g;                     #whatever they are??
+    s/[.,?!;]//g;                   #Remove normal punctuation
+    s/"//g;                        #Remove quotation marks
+    s/\#/ /g;                      #remove simultaneous speech markers     '#'
+    s/\[\[[^]]*\]\]//g;            #remove comments              '[[words]]'
+    s/\[[^]]*\]//g;                #remove other sounds          '[words]'
+    s/\/\///g;                     #remove speech to someone else markers  '//'
+    s/<\/*aside\/*>//g;            #remove aside sgml tags
+
+    if ($ReadMSstate){
+        s/\^(\S)/$1/g;              #remove proper name markers? '^'
+        s/\*(\S)/$1/g;              #remove ??? name markers? '*'
+        s/\@(\S)/$1/g;              #remove ??? name markers? '@'
+        s:<as/>[^<>]+</as>::ig;   #remove asides
+        while ($_ =~ /~/){          # Deal with acronyms
+            s/~(\S) /$1 /;
+            s/~(\S\'[sS]) /$1 /;       # Deal with a possessive acronym
+            s/~(\S)(\S)/$1 ~$2/;
+        } 
+    }
+
+    s/\(\(\s*\)\)//g;              #remove unintelligable speech with no words
+
+    ## Changes from new LDC transcription practices (Jan 2003)_
+    s/~([A-Z-a-z])/$1./g;          #Change the spoken letters to the Hub4 form
+    s/\^//g;                        #Remove proper names
+
+
+    if ($OptDel == 1){
+        s/\(\s+/(/g;                 #Normalize the variablity in the LDC trans
+        s/\s+\)/)/g;                 #Normalize the variablity in the LDC trans
+        my $vb = 0;
+        my $iteration = 1;
+        print "Start $_\n" if ($vb);
+        my $loop_end = "xxxxxxxxxxxxxxxxx";
+
+        my $plainWord = "[^ \t\n\r\f\(\)\{\}]+";
+        my $optDelWord = "\\([^ \t\n\r\f\(\)\{\}]+\\)";
+#        my $wordSet = "($plainWord|$optDelWord)(\s+($plainWord|$optDelWord))*";
+
+        while ($_ =~ /\(\(.*\)\)/) {
+            # if the unintell is out side of a OOL marker, just delete them
+            s/\(\((\s*<[^>]+>\s*)\)\)/$1/;
+            print "      Step 1 $_\n" if ($vb);
+
+            #  handle unintelligle markers within OOL markers
+            s/\(\((\s*<[^>]+>\s*)([^\)]*)\)\)/$1 \(\($2\)\)/;
+            print "      Step 2 $_\n" if ($vb);
+
+	    #Change 1st unintelligable word, if its already optionally deletable
+            s/\(\(\s*($optDelWord)\s+(($plainWord|$optDelWord)(\s+($plainWord|$optDelWord))*\s*\)\))/ $1 \(\($2/g;
+            print "      Step 3 $_\n" if ($vb);
+
+	    #Change 1st unintelligable word, if its an alternate
+            s:\(\(\s*({[^{}]+})\s+(($plainWord|$optDelWord)(\s+($plainWord|$optDelWord))*\s*\)\)): \($1\) \(\($2:g;
+            print "      Step 4 $_\n" if ($vb);
+
+	    #Change 1st unintelligable word, if its a plain word
+            s/\(\(\s*($plainWord)\s+(($plainWord|$optDelWord)(\s+($plainWord|$optDelWord))*\s*\)\))/ \($1\) \(\($2/g;
+            print "      Step 5 $_\n" if ($vb);
+
+            ## Change a single optionally deletable word
+            s/\(\(\s*($optDelWord)\s*\)\)/ $1 /g;
+            print "      Step 8 $_\n" if ($vb);
+
+            #alternate single unintelligable with alternate
+            s/\(\(\s*({[^{}]+})\s*\)\)/ \($1\) /g;
+            print "      Step 9 $_\n" if ($vb);
+
+            #alternate single unintelligable words
+            s/\(\(\s*($plainWord)\s*\)\)/ \($1\) /g;
+            print "      Step A $_\n" if ($vb);
+
+	    # remove empty unintellegible markers
+	    s/\(\(\s*\)\)//g;
+            $iteration ++;
+	
+	    if ($_ eq $loop_end){
+	         print STDERR "Warning: unable to fully alternate an unitelligible marker\n".
+	                      "         $_\n";
+	         last;
+	    }
+            if ($iteration > 100){
+	         die "Unable to alternate an unitelligible marker '$_'\n";
+            }
+            # Remember what the last line was so that We don't get it an infinite loop
+            $loop_end = $_;
+            print "   End $_\n" if ($vb);
+        }
+    } else {
+#        print "Delete (( $_\n";
+        s/\s\(\(/ /g;                 #Strip the unintelligible markers
+        s/\)\)\s/ /g;                 #Strip the unintelligible markers
+#        print "          $_\n";
+    }
+
+    if (! $OptDel){
+        s/<\s*\S+\s+([^<>]*)>/$1/g;  #remove out-of-language speech markers
+        s/<[^<>]*>//g;                #remove out-of-language speech markers
+    } else {
+        while ($_ =~ /(<\s*\S+\s+)([^<>]*)>/) {
+            my ($otxt, $head) = ($2, $1);
+            my $txt = $otxt;
+            $txt =~ s/^\s+//;;
+            $txt =~ s/\s+$//;;
+            my $new= "";
+            foreach my $w(split(/\s+/,$txt)){
+                if ($w !~ /^\(.+\)$/){
+                    if ($new eq ""){
+                        $new = "(".$w.")";
+                    } else {
+                        $new .= " (".$w.")";
+                    }
+                } else {
+                    $new .= " $w";
+                }
+            }
+            if ( ($_ =~ s/${head}[^>]*>/$new/) != 1){
+                die("OptDel substitution of non-language marker failed\n".
+                    "    text: $_    head: '$head'\n    otxt: '$otxt'\n    new: '$new'".
+                    "\n    merge: '${head}${otxt}>'");
+            }
+        }
+        ### delete noise tags.  These are new as of 1/20/2004
+        s/<[^ \t\n\r\f<>]+>//g;
+   }
+
+    ### Language Specific Filters
+
+    # SPANISH
+    if ($Language eq "spanish") {
+        y/\240-\277//d;	             #remove spanish punctuation
+        s/\*\*//g;		     #words marked with "**" (as neologisms or unknown
+                                     # 'words')
+        s/\+//g;		     #words marked with "++" (mispronunciations given
+                                     # in correct orthography)
+
+        # map the list of hesitiation's
+        s:%::g; # remove all hesitation markers (maybe illegal ones!)
+        # insert all hesitation markers (maybe missed ones!)
+        s/ AAA / $HES /g;
+        s/ EEE / $HES /g;
+        s/ III / $HES /g;
+        s/ MMM / $HES /g;
+        s/ EMM / $HES /g;
+        s/ AMM / $HES /g;
+        s/ IMM / $HES /g;
+        s/ EH / $HES /g;
+        s/ MMH / $HES /g;
+        s/ OH / $HES /g;
+        s/ UH / $HES /g;
+        s/ EY / $HES /g;
+        s/ UY / $HES /g;
+        s/ PSS / $HES /g;
+        s/ HM / $HES /g;
+        s/ UF / $HES /g;
+        s/ SHH / $HES /g;
+        s/ PFF / $HES /g;
+        s/ OY / $HES /g;
+        s/ HA / $HES /g;
+        s/ AH / $HES /g;
+     }
+
+     # GERMAN
+     if ($Language eq "german") {
+
+        s/\&//g;	 		 #remove proper name marker
+        s/\+//g;		     #words marked with "++" (mispronunciations given
+                                     # in correct orthography)
+         s/\*\*//g;		     #words marked with "**" (as neologisms
+                                     # or unknown 'words')
+        
+        s:%::g; # remove all hesitation markers (maybe illegal ones!)
+        # insert all hesitation markers (maybe missed ones!)
+
+        s/ �H / $HES /g;
+        s/ MM / $HES /g;
+        s/ �HM / $HES /g;
+        s/ HM / $HES /g;
+        s/ HA / $HES /g;
+        s/ EI / $HES /g;
+        s/ UH / $HES /g;
+        s/ H� / $HES /g;
+        s/ HO / $HES /g;
+        s/ UFF / $HES /g;
+        s/ OI / $HES /g;
+        s/ HUH / $HES /g;
+        s/ BAH / $HES /g;
+        s/ UI / $HES /g;
+     }
+
+
+     # ARABIC
+     if ($Language eq "arabic") {
+
+        s/\&//g;	 		 #remove proper name the marker
+        s/\s\+(\S+)\+\s/ $1 /;           #Remove the plus markers
+#        s:%\S+\s::g; # remove all hesitation markers (maybe illegal ones!)
+        s/\*\*//g;		     #words marked with "**" (as neologisms
+                                     # or unknown 'words')
+        s/$/ /;                      ## Add a space at the end of the line
+        s/ /  /;                     ## Add double spaces
+
+        # insert all hesitation markers (maybe missed ones!)
+#        s/ ah / $HES /g;
+#        s/ E / $HES /g;
+#        s/ M / $HES /g;
+#        s/ mhm / $HES /g;
+#        s/ ha / $HES /g;
+#        s/ uh / $HES /g;
+#        s/ yA / $HES /g;
+#        s/ aha / $HES /g;
+#        s/ yaa / $HES /g;
+#        s/ hm / $HES /g;
+#        s/ hum / $HES /g;
+#        s/ Ah / $HES /g;
+#        s/ ayyO  / $HES /g;
+#        s/ Eyy / $HES /g;
+#        s/ yuu / $HES /g;
+#        s/ ih / $HES /g;
+#        s/ yO / $HES /g;
+#        s/ O / $HES /g;
+#        s/ wAw / $HES /g;
+#        s/ hi / $HES /g;
+#        s/ Hay / $HES /g;
+#        s/ yOO / $HES /g;
+#        s/ OhO / $HES /g;
+#        s/ hE / $HES /g;
+#        ### Script equivalences
+#        s/ \302\347 / $HES /g;
+#        s/ \305\352\347 / $HES /g;
+#        s/ \305\352\352\347 / $HES /g;
+#        s/ \315\351 / $HES /g;
+#        s/ \345 / $HES /g;
+#        s/ \307\350 / $HES /g;
+#        s/ \307\350\347\350 / $HES /g;
+#        s/ \307\347 / $HES /g;
+#        s/ \307\347\307 / $HES /g;
+#        s/ \307\352\350     / $HES /g;
+#        s/ \347\351 / $HES /g;
+#        s/ \347\307 / $HES /g;
+#        s/ \347\351 / $HES /g;
+#        s/ \347\345 / $HES /g;
+#        s/ \347\345 / $HES /g;
+#        s/ \305\347 / $HES /g;
+#        s/ \345\345 / $HES /g;
+#        s/ \307\347 / $HES /g;
+#        s/ \350\307\350 / $HES /g;
+#        s/ \352\307\347 / $HES /g;
+#        s/ \352\350 / $HES /g;
+#        s/ \352\350\350 / $HES /g;
+#        s/ \352\307 / $HES /g;
+#        s/ \352\347 / $HES /g;
+     }
+
+     # MANDARIN
+     if ($Language eq "mandarin") {
+         s/\+//g;	 		 #remove non-standard representation marker
+         s/\&//g;	 		 #remove some the marker
+         s/\243[\254\255] / /g;      # Remove Japanese punctuation
+
+         s/\s%(\S)/ $1/g;          # Delete the all the percent signs at word head
+         s/\s\(\%/ (/g;          # Delete the all the percent signs at word head of unclear
+         s/%\s/ /g;                  # Delete the all the percent signs at word tail
+
+         # NOTE: Cannot remove all hesitation markers because some "%" are used
+         #       in the text (I think)
+         #       So we will just filter the known hesitation markers.
+         s/\s��\s/ $HES /g;
+         s/\s��\s/ $HES /g;
+         s/\s��\s/ $HES /g;
+         s/\s��\s/ $HES /g;
+         s/\s��\s/ $HES /g;
+     }	
+	
+     # JAPANESE
+     if ($Language eq "japanese") {
+         s/\+//g;		    #remove non-standard representation marker
+         s/\=/\-/g;                 #change frag= to frag-
+         s/\241[\241-\256] / /g;      # Remove Japanese punctuation
+     }
+
+     if ($Language eq "english") {
+         s/\&//g;	 		 #remove proper name the marker
+         s/\*\*//g;		     #words marked with "**" (as neologisms
+                                     # or unknown 'words')
+         s/\*//g;		     #words marked with "*" idiosyncratic words
+                                     # or unknown 'words')
+         s/\+//g;		     #words marked with "++" (mispronun-
+                                     # ciations given in correct orthography)
+         s/@//g;                     # Spoken acronyms
+         ### Mapp the hesitations
+         s/[\s\(][%]*(UH|UM|EH|MM|AH|HM|HUH|HA|ER|OOF|HEE|ACH|EEE|EW)[\s\)]/ $HES /g;
+         if ($ReadMSstate){
+#### Commented out because this should be changed in SPINE
+####         s/[\s\(][%]*(OH|AW)[\s\)]/ $HES /g;
+             s/[\s\(][%]*(MHM)[\s\)]/ $1 /g;
+         }
+     }
+
+     # This has to be done AFTER ALL THE PROCEEDING STEPS to alternate any fragment words
+     if ($OptDel){
+         s/(\S+)-(\S+)[-=]\s/ $1 \($2-\) /g;  #put in alternates for fragments    
+         s/(\S*)[-=]\s/ \($1-\) /g;  #put in alternates for fragments    
+         s/\s[-=](\S*)/ \(-$1\) /g;  #put in alternates for fragments    
+     } else {
+         s/(\S*)[-=]\s/ $1- /g;  #put in alternates for fragments    
+         s/\s[-=](\S*)/ -$1 /g;  #put in alternates for fragments    
+     }
+     s/ +/ /g;
+     if ($#Translate >= 0){	
+	$_ = &Map_AArray($_,\%TranslateAArray);
+     }	
+     if ($#Equivs >= 0){	
+	$_ = &Map_AArray($_,\%EquivsAArray);
+     }
+     if ($AltHyphens) {
+        $_ = &AlternateHyphenWords($_);
+     }
+
+     ### delete any emtpy optionally deletable words
+     if ($OptDel){         s/\(\)//g;  }
+
+     my($bt, $et, $spkr) = split(/\s+/,$line_header);
+     my $chan;
+     ($chan = $spkr) =~ s/[0-9]$//;
+     if (&in_range($bt, $et) && ($KeepEmptyLines || $_ !~ /^\s*$/)){
+         if ($Format eq "stm"){
+	     push @STMS, sprintf "%s %s %s %s %6.2f <O> %s\n",$id,makeChannel($chan),$id."_".$spkr,$bt,$et,$_; 
+         } elsif ($Format eq "pem"){
+	     printf OUTPUT "%s %s %s %s %s\n",$id,makeChannel($chan),"unknown_speaker",$bt,$et; 
+         }
+     }
+}
+
+sub makeChannel{
+    my ($chan) = @_;
+    if ($ChannelsTo eq "numbers") {
+	$chan =~ s/a/1/i;
+	$chan =~ s/b/2/i;
+    } elsif ($ChannelsTo eq "letters") {
+	$chan =~ s/1/A/i;
+	$chan =~ s/2/B/i;
+    } elsif ($ChannelsTo eq "asis") {
+	;
+    } else {
+	die "Error: unknown Channel conversion $ChannelsTo";
+    }
+    $chan;
+}
+
+sub MIN{
+    my(@a) = @_;
+    my($min) = 99e9;
+
+    while ($#a >= 0){
+        # print "  $a[0]\n";
+        $min = $a[0] if ($min > $a[0]);
+        splice(@a,0,1);
+    }
+    $min;
+}
+
+if ($Format eq "stm"){
+    my (%lastStmEnd, %lastStmEndId, %lastStmEndIndex, $nt);
+    ### Pad the STMs segments with silence if you got it
+    if ($PadSilence >= 0){
+        for (my $i=0; $i<@STMS; $i++){
+#            print "STM[$i] = $STMS[$i]";
+            my ($id, $chan, $spkr, $bt, $et, $lur, $text) = split(/\s+/,$STMS[$i],7);
+            if (! defined($lastStmEnd{$chan})){
+#                print "Set \$lastStmEnd{$chan} = $NewBeginTime;\n";
+                $lastStmEnd{$chan} = $NewBeginTime;
+                $lastStmEndId{$chan} = $id;
+		$lastStmEndIndex{$chan} = $i;
+            }
+            if ($lastStmEnd{$chan} < $bt){
+                my $shift = MIN(($bt - $lastStmEnd{$chan}) / 2.0, $PadSilence);
+                if ($lastStmEndIndex{$chan} < $i){
+                    $nt = sprintf("%.2f", $lastStmEnd{$chan} + $shift);
+#		    print "fix $nt $STMS[$lastStmEndIndex{$chan}] ";
+                    die "Can't fix silence pad time of previous line" unless ($STMS[$lastStmEndIndex{$chan}] =~ s/^(\S+\s+\S+\s+\S+\s+\S+\s+)\S+/$1$nt/);
+#		    print "->  $STMS[$lastStmEndIndex{$chan}]\n";
+                } 
+                $nt = sprintf("%.2f",$bt - $shift);
+                die "Can't fix silence pad time" unless ($STMS[$i] =~ s/^(\S+\s+\S+\s+\S+\s+)\S+/$1$nt/);
+            } 
+            $lastStmEnd{$chan} = $et;
+       	    $lastStmEndIndex{$chan} = $i;
+#            print "     \$lastStmEnd{$chan} = $et\n";
+        }
+        foreach my $chan(sort(keys %lastStmEnd)){
+            my ($id, $ccchan, $spkr, $bt, $et, $lur, $text) = split(/\s+/,$STMS[$lastStmEndIndex{$chan}],7);
+#	    print "End $STMS[$lastStmEndIndex{$chan}]";
+            if ($lastStmEnd{$chan} < $NewEndTime){
+                my $shift = MIN(($NewEndTime - $lastStmEnd{$chan}) / 2.0, $PadSilence);
+                $nt = $lastStmEnd{$chan} + $shift;
+                die "Can't fix silence pad time" unless ($STMS[$lastStmEndIndex{$chan}] =~ s/^(\S+\s+\S+\s+\S+\s+\S+\s+)\S+/$1$nt/);
+#		print "    end now $STMS[$lastStmEndIndex{$chan}]\n";
+            }
+        }
+    }
+    undef %lastStmEnd;
+    undef %lastStmEndId;
+
+    ## Smooth the segments 
+    if ($SmoothSegments >= 0){
+        for (my $i=0; $i<@STMS; $i++){
+            my ($id, $chan, $spkr, $bt, $et, $lur, $text) = split(/\s+/,$STMS[$i],7);
+            if (! defined($lastStmEnd{$chan})){
+                $lastStmEnd{$chan} = $NewBeginTime;
+                $lastStmEndId{$chan} = $id;
+		$lastStmEndIndex{$chan} = $i;
+            }
+	    my $change = 0;
+            if ($lastStmEnd{$chan} > $bt - $SmoothSegments && $lastStmEndIndex{$chan} < $i){
+		my ($lid, $lchan, $lspkr, $lbt, $let, $llur, $ltext) = split(/\s+/,$STMS[$lastStmEndIndex{$chan}],7);
+		if ($id eq $lid && $chan eq $lchan && $spkr eq $lspkr && $lur eq $llur){
+		    chomp $ltext;
+		    chomp $text;
+		    $STMS[$lastStmEndIndex{$chan}] = 
+			"$id $chan $spkr $lbt $et $lur $ltext $text\n";
+		    splice (@STMS, $i, 1);
+		    $i --;
+		    $change = 1;
+		}
+            } 
+	    $lastStmEnd{$chan} = $et;
+	    if (! $change){
+		$lastStmEndIndex{$chan} = $i;
+	    }
+        }
+    }
+    undef %lastStmEnd;
+    undef %lastStmEndId;
+
+    ## Smooth the segments 
+    if ($ExcludeOverlap){
+        for (my $i=0; $i<@STMS; $i++){
+#            print "STM[$i] = $STMS[$i]";
+            my ($id, $chan, $spkr, $bt, $et, $lur, $text) = split(/\s+/,$STMS[$i],7);
+            if (! defined($lastStmEnd{$chan})){
+                $lastStmEnd{$chan} = $NewBeginTime;
+                $lastStmEndId{$chan} = $id;
+		$lastStmEndIndex{$chan} = $i;
+            }
+	    my $change = 0;
+            if ($lastStmEnd{$chan} > $bt && $lastStmEndIndex{$chan} < $i){
+		my ($lid, $lchan, $lspkr, $lbt, $let, $llur, $ltext) = split(/\s+/,$STMS[$lastStmEndIndex{$chan}],7);
+		if ($id eq $lid && $chan eq $lchan){
+		    chomp $ltext;
+		    $STMS[$lastStmEndIndex{$chan}] = 
+			"$id $chan excluded_region $lbt $et $lur ignore_time_segment_in_scoring\n";
+		    splice (@STMS, $i, 1);
+		    $i --;
+		    $change = 1;
+		}
+            } 
+	    $lastStmEnd{$chan} = $et;
+	    if (! $change){
+		$lastStmEndIndex{$chan} = $i;
+	    }
+        }
+    }
+    undef %lastStmEnd;
+    undef %lastStmEndId;
+
+     ## Add the intersegment gaps
+     if ($AddInterSegGap){
+         for (my $i=0; $i<@STMS; $i++){
+             my ($id, $chan, $spkr, $bt, $et, $lur, $text) = split(/\s+/,$STMS[$i],7);
+             if (! defined($lastStmEnd{$chan})){
+                 $lastStmEnd{$chan} = $NewBeginTime;
+                 $lastStmEndId{$chan} = $id
+             }
+             if ($lastStmEnd{$chan} < $bt - 0.01){
+#            if (0){
+                 splice (@STMS, $i, 0, sprintf("%s %s %s %s %6.2f <O> %s\n",$id,$chan,
+                                       "${id}_${chan}_".
+                                       "inter_segment_gap",
+                                       $lastStmEnd{$chan},$bt,""));
+                 $i++;
+             }
+             $lastStmEnd{$chan} = $et;
+         }
+         foreach my $chan(sort(keys %lastStmEnd)){
+             if ($lastStmEnd{$chan} < $NewEndTime - 0.01){
+                 push(@STMS, sprintf("%s %s %s %s %6.2f <O> %s\n",$lastStmEndId{$chan},$chan,
+                   "${id}_${chan}_inter_segment_gap",
+                    $lastStmEnd{$chan},$NewEndTime,"")); 
+             }
+         }
+    }
+}
+print OUTPUT @STMS;
+close OUTPUT;
+exit 0;
+
+###############################################################################
+####                                                                       ####
+####                        END OF MAIN PROGRAM                            ####
+####                                                                       ####
+###############################################################################
+
+ 
+####
+####  Return true if the passed in begin and end times are within the 
+####  specified range of output times.  If not return false.
+####  Inputs:   - Argument 0: The begin time
+####            - Argument 1: The end time
+####  Outputs:  - Return true if:
+####              * the global variables $NewBeginTime or $NewEndTime
+####              * are set, and the begin and end times are with their
+####              * respective ranges
+####              OR
+####              * neither global variables are defined.
+sub in_range{
+    my ($bt,$et) = @_;
+    my $do_prnt = 0;
+    if (defined($NewBeginTime) && defined($NewEndTime)) {
+	if (($bt >= $NewBeginTime || &eqdelta($bt,$NewBeginTime,0.5)) &&
+	    ($et <= $NewEndTime || &eqdelta($et,$NewEndTime,0.5))) {  
+	    $do_prnt = 1;
+	}
+    } elsif (defined($NewBeginTime)) {
+	if ($bt >= $NewBeginTime ||
+	    &eqdelta($bt,$NewBeginTime,0.5)) {
+	    $do_prnt = 1;
+	} 
+    } elsif (defined($NewEndTime)) {
+	if ($et <= $NewEndTime || &eqdelta($et,$NewEndTime,0.5)) {
+	    $do_prnt = 1;
+	}
+    } else {
+	$do_prnt = 1;
+    }
+    $do_prnt;
+}
+
+
+####
+####  Compare two real numbers, they're equal if the absolute value of the
+####  difference is less than delta
+####  Inputs:  - Argument 0: real number 1.
+####           - Argument 1: real number 2.
+####           - Argument 2: The delta.
+####  Outputs: - True if they are equal, False if they are not
+sub eqdelta {
+    my($v1,$v2,$del) = @_;
+    if ($v1 < $v2) {
+        $v2 - $v1 < $del;
+    } else {
+        $v1 - $v2 < $del;
+    }
+}
+
+sub add_spaces_for_japanese_punctuation {
+    $_ = $_[0];
+    chomp;
+    my $first = 0;
+    my $c1 = "";
+    my $lc = "";
+    my $out = "";
+    
+    while (length($_) > 0){
+	s/^(.)//;
+	$c1 = $1;
+	if ($c1 =~ /^[\177-\377]/){
+	    if (!$first) { $first = 1; } else { $first = 0; }
+	}
+	$out = $out.$c1;
+	if (!$first && ($lc.$c1) =~ /^\241[\241-\270\272\273\275-\377]$/) {
+	    $out = $out." ";
+	}
+	$lc = $c1;
+    }
+    $out."\n";
+}
+
+sub Map_AArray{
+    $_ = $_[0];
+    my $AArray = $_[1];
+    my $out = "";
+    my $word = "";
+    foreach $word (split){
+	if ($AArray->{$word} !~ /^$/){
+	    $out .= " ".$AArray->{$word};
+	} else {
+	    $out .= " ".$word;
+	}
+    }
+    $out .= "\n";
+    $out;
+}
+
+sub AlternateHyphenWords{
+    $_ = $_[0];
+    my $out = "";
+    my $word = "";
+    my $x;
+    my $y;
+    my $z;
+    foreach $word (split){
+	if ($word =~ /[A-Za-z][_-][A-Za-z]/){
+	    ($x=$word) =~ s/[_-]//g;
+	    ($y=$word) =~ s/[_-]/ /g;
+	    ($z=$word) =~ s/[_-]/-/g;
+	    $word = "{ ".$y." / ".$x." / ".$z." }"; 
+	}
+	$out .= " ".$word;
+    }
+    $out .= "\n";
+    $out;
+}

SCTK/src/chfilt/makefile

@@ -0,0 +1,40 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+T=../test_suite
+
+all: chfilt.pl
+
+chfilt.pl: 
+	chmod +x chfilt.pl
+
+setTests:
+	./chfilt.pl -m -l english $(T)/testmss.in -- - > $(T)/testmss.stm
+	./chfilt.pl -C numbers -m -l english $(T)/testmss.in -- - > $(T)/testmss.numbers.stm
+	./chfilt.pl -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.-optdel.stm
+	./chfilt.pl -d -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.stm
+	./chfilt.pl -c -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.-contexp.stm
+	./chfilt.pl -c -l arabic -i -b 0 -e 999999 -d -k $(T)/test.arb2004.txt -- - > $(T)/test.arb2004.txt.stm
+	./chfilt.pl -c -l mandarin -i -b 0 -e 999999 -d -k $(T)/test.man2004.txt -- - > $(T)/test.man2004.txt.stm
+
+check: all
+	@echo "Testing chfilt.pl"
+	./chfilt.pl -m -l english $(T)/testmss.in -- - | diff - $(T)/testmss.stm
+	./chfilt.pl -C numbers -m -l english $(T)/testmss.in -- - | diff - $(T)/testmss.numbers.stm
+	./chfilt.pl -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.-optdel.stm
+	./chfilt.pl -d -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.stm
+	./chfilt.pl -c -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.-contexp.stm	
+	./chfilt.pl -c -l arabic -i -b 0 -e 999999 -d -k $(T)/test.arb2004.txt -- - | diff - $(T)/test.arb2004.txt.stm
+	./chfilt.pl -c -l mandarin -i -b 0 -e 999999 -d -k $(T)/test.man2004.txt -- - | diff - $(T)/test.man2004.txt.stm
+	@echo "   chfilt.pl tests successfully completed"
+
+install: all
+	cp chfilt.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/chfilt/makefile.in

@@ -0,0 +1,40 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=
+bindir = $(PREFIX)/bin
+T=../test_suite
+
+all: chfilt.pl
+
+chfilt.pl: 
+	chmod +x chfilt.pl
+
+setTests:
+	./chfilt.pl -m -l english $(T)/testmss.in -- - > $(T)/testmss.stm
+	./chfilt.pl -C numbers -m -l english $(T)/testmss.in -- - > $(T)/testmss.numbers.stm
+	./chfilt.pl -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.-optdel.stm
+	./chfilt.pl -d -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.stm
+	./chfilt.pl -c -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - > $(T)/test.txt.-contexp.stm
+	./chfilt.pl -c -l arabic -i -b 0 -e 999999 -d -k $(T)/test.arb2004.txt -- - > $(T)/test.arb2004.txt.stm
+	./chfilt.pl -c -l mandarin -i -b 0 -e 999999 -d -k $(T)/test.man2004.txt -- - > $(T)/test.man2004.txt.stm
+
+check: all
+	@echo "Testing chfilt.pl"
+	./chfilt.pl -m -l english $(T)/testmss.in -- - | diff - $(T)/testmss.stm
+	./chfilt.pl -C numbers -m -l english $(T)/testmss.in -- - | diff - $(T)/testmss.numbers.stm
+	./chfilt.pl -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.-optdel.stm
+	./chfilt.pl -d -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.stm
+	./chfilt.pl -c -l english -i -b 0 -e 999999 -k $(T)/test.txt -- - | diff - $(T)/test.txt.-contexp.stm	
+	./chfilt.pl -c -l arabic -i -b 0 -e 999999 -d -k $(T)/test.arb2004.txt -- - | diff - $(T)/test.arb2004.txt.stm
+	./chfilt.pl -c -l mandarin -i -b 0 -e 999999 -d -k $(T)/test.man2004.txt -- - | diff - $(T)/test.man2004.txt.stm
+	@echo "   chfilt.pl tests successfully completed"
+
+install: all
+	cp chfilt.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/ctmValidator/makefile

@@ -0,0 +1,19 @@
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+
+all: ctmValidator.pl
+
+rttmValidator.pl:
+	chmod +x ctmValidator.pl
+
+check: all
+	(cd test_suite; $(SHELL) RunTest.sh ../ctmValidator.pl)
+
+install: all
+	cp ctmValidator.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/hamzaNorm/hamzaNorm.pl

@@ -0,0 +1,95 @@
+#!/usr/bin/env perl
+
+use strict;
+
+my $Version="1.0";
+
+#####
+#  Version 1.0  Released August 25, 2004
+#        - Initial Release
+
+my $Usage="Usage: hamzaNorm.pl [ -i fmt ] Infile|- OutFile|-\n".
+"Version: $Version\n".
+"Desc: hamzaNorm normalizes the hamza in the initial position of the word to be a\n".
+"      consistent form.  In the Buckwalter normalization scheme, the initial letters\n".
+"      'A', '<', '>' and '|' are all translated to 'A'".
+"Options:\n".
+"      -i fmt   Set the input file formant to 'fmt'.  The possible choices are:\n".
+"                  txt -> plain text, the default\n".
+"                  ctm -> CTM format, ignores all but the 5th column, and if\n".
+"                         a division occurs and a confidence score is present,\n".
+"                         the confidence score is copied to all parts.\n".
+"                  stm -> STM format, change only the text field of the stm record\n".
+"\n";
+
+use Getopt::Long;
+my ($InFmt) = undef;
+my $result = GetOptions ("i:s" => \$InFmt);
+die "Aborting:\n$Usage\n:" if (!$result);
+
+if (defined($InFmt)) {
+    die("$Usage\n\nError: Undefined input format '$InFmt'") 
+	if ($InFmt !~ /^(txt|ctm|stm)$/);
+} else {
+    $InFmt = "txt";
+} 
+
+#### The main functions arguments:
+die "$Usage\nToo many arguments" if ($#ARGV > 1);
+die "$Usage\nOutput Not Specified" if ($#ARGV == 0);
+die "$Usage\nInput and Output Not Specified" if ($#ARGV == -1);
+
+my $InFile=$ARGV[0];
+my $OutFile=$ARGV[1];
+die("$Usage\nError: Input file $InFile does not exist\n")
+    if ($InFile ne "-" && ! -r $InFile);
+
+open(IN, "$InFile") || die "Unable to open trans-file $InFile";
+open(OUT, ">$OutFile") || die "Unable to open new-trans-file $OutFile";
+
+while (<IN>){
+    chomp;
+    if ($InFmt eq "txt"){
+	print OUT normalize($_)."\n";
+    } elsif ($InFmt eq "ctm"){
+	if ($_ =~ /^(\;\;|\#)/){
+	    print OUT $_."\n";
+	    next;
+	}	     
+	s/^(\s+)//;
+	my $prefix = (defined($1) ? $1 : "");
+	my @ctm = split(/(\s+)/,$_);
+	$ctm[8] = normalize($ctm[8]);
+	print OUT $prefix.join("", @ctm)."\n";
+    } elsif ($InFmt eq "stm"){
+	if ($_ =~ /^(\;\;|\#)/){
+	    print OUT $_."\n";
+	    next;
+	}	     
+	s/^(\s+)//;
+	my $prefix = (defined($1) ? $1 : "");
+	my @stm = split(/(\s+)/,$_, 7);
+	if ($stm[10] =~ /^<[^<>]*>$/){
+	    $stm[12] = normalize($stm[12]);
+	} else {
+	    $stm[10] .= join("",splice(@stm,11,2));
+	    $stm[10] = normalize($stm[10]);
+	}
+	print OUT $prefix.join("", @stm)."\n";
+    } else {
+	die "Error: unknown input format '$InFmt'\n$Usage\n";
+    }    
+}
+
+close IN; close OUT;
+exit 0;
+
+sub normalize{
+    my ($text) = @_;
+    $text = " ".$text;    
+    $text =~ s/ (\330\242|\330\243|\330\245|\330\247)/ \330\247/g;
+    $text =~ s/^ //;
+    $text;
+}
+
+

SCTK/src/hamzaNorm/makefile

@@ -0,0 +1,33 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+T=../test_suite
+
+all: hamzaNorm.pl
+
+hamzaNorm.pl: 
+	chmod +x hamzaNorm.pl
+
+setTests:
+	cat $(T)/test.arb2004.txt | ./hamzaNorm.pl - $(T)/test.arb2004.hamzaNorm.txt
+	cat $(T)/test.arb2004.txt.stm | sed 's/<O>//' | ./hamzaNorm.pl -i stm - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.ctm | ./hamzaNorm.pl -i ctm - - | sed 's/<O>//' > $(T)/test.arb2004.txt.ctm.hamza
+
+check: all
+	@echo "Testing hamzaNorm.pl"
+	cat $(T)/test.arb2004.txt | ./hamzaNorm.pl -- - - | diff - $(T)/test.arb2004.hamzaNorm.txt
+	cat $(T)/test.arb2004.txt.stm | sed 's/<O>//' | ./hamzaNorm.pl -i stm -- - - | diff - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.stm | ./hamzaNorm.pl -i stm -- - - | sed 's/<O>//' | diff - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.ctm | ./hamzaNorm.pl -i ctm -- - - | sed 's/<O>//' | diff - $(T)/test.arb2004.txt.ctm.hamza
+	@echo "   hamzaNorm.pl tests successfully completed"
+
+install: all
+	cp hamzaNorm.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/hamzaNorm/makefile.in

@@ -0,0 +1,33 @@
+# **********************************************************
+# Makefile for rfilter1 Directory
+# **********************************************************
+SHELL = /bin/sh
+PREFIX=
+bindir = $(PREFIX)/bin
+T=../test_suite
+
+all: hamzaNorm.pl
+
+hamzaNorm.pl: 
+	chmod +x hamzaNorm.pl
+
+setTests:
+	cat $(T)/test.arb2004.txt | ./hamzaNorm.pl - $(T)/test.arb2004.hamzaNorm.txt
+	cat $(T)/test.arb2004.txt.stm | sed 's/<O>//' | ./hamzaNorm.pl -i stm - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.ctm | ./hamzaNorm.pl -i ctm - - | sed 's/<O>//' > $(T)/test.arb2004.txt.ctm.hamza
+
+check: all
+	@echo "Testing hamzaNorm.pl"
+	cat $(T)/test.arb2004.txt | ./hamzaNorm.pl -- - - | diff - $(T)/test.arb2004.hamzaNorm.txt
+	cat $(T)/test.arb2004.txt.stm | sed 's/<O>//' | ./hamzaNorm.pl -i stm -- - - | diff - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.stm | ./hamzaNorm.pl -i stm -- - - | sed 's/<O>//' | diff - $(T)/test.arb2004.txt.stm.hamza
+	cat $(T)/test.arb2004.txt.ctm | ./hamzaNorm.pl -i ctm -- - - | sed 's/<O>//' | diff - $(T)/test.arb2004.txt.ctm.hamza
+	@echo "   hamzaNorm.pl tests successfully completed"
+
+install: all
+	cp hamzaNorm.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/rttmValidator/makefile

@@ -0,0 +1,19 @@
+SHELL = /bin/sh
+PREFIX=/home/userube/mozilla/zz__temp_5_rover/SCTK
+bindir = $(PREFIX)/bin
+
+all: rttmValidator.pl
+
+rttmValidator.pl:
+	chmod +x rttmValidator.pl
+
+check: all
+	(cd test_suite; $(SHELL) RunTest.sh ../rttmValidator.pl)
+
+install: all
+	cp rttmValidator.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/rttmValidator/makefile.in

@@ -0,0 +1,19 @@
+SHELL = /bin/sh
+PREFIX=
+bindir = $(PREFIX)/bin
+
+all: rttmValidator.pl
+
+rttmValidator.pl:
+	chmod +x rttmValidator.pl
+
+check: all
+	(cd test_suite; $(SHELL) RunTest.sh ../rttmValidator.pl)
+
+install: all
+	cp rttmValidator.pl $(DESTDIR)$(bindir)
+
+clean:
+
+distclean: clean
+	$(RM) -r *~ makefile

SCTK/src/rttmValidator/rttmValidator.pl

@@ -0,0 +1,917 @@
+#!/usr/bin/env perl
+
+# RTTMVALIDATOR
+# Author: Jon Fiscus, Jerome Ajot
+#
+# This software was developed at the National Institute of Standards and Technology by 
+# employees of the Federal Government in the course of their official duties. Pursuant
+# to title 17 Section 105 of the United States Code this software is not subject to
+# copyright protection and is in the public domain. RTTMVALIDATOR is an experimental system.
+# NIST assumes no responsibility whatsoever for its use by other parties, and makes no
+# guarantees, expressed or implied, about its quality, reliability, or any other
+# characteristic. We would appreciate acknowledgement if the software is used.
+#
+# THIS SOFTWARE IS PROVIDED "AS IS."  With regard to this software, NIST MAKES NO EXPRESS
+# OR IMPLIED WARRANTY AS TO ANY MATTER WHATSOEVER, INCLUDING MERCHANTABILITY,
+# OR FITNESS FOR A PARTICULAR PURPOSE.
+
+use warnings;
+use strict;
+use Getopt::Std;
+use Data::Dumper;
+
+my $debug = 0;
+
+my $VERSION = "v13";
+
+my $USAGE = "\n\n$0 [-useh] -i <RTTM file>\n\n".
+    "Version: $VERSION\n".
+    "Description: This Perl program (version $VERSION) validates a given RTTM file.\n".
+    "\tNote that the program will exit after it finds all the syntax errors or\n".
+    "\twhen it finds the first logic error.\n".
+    "Options:\n".
+    "  -u            : disable check that ensures all LEXEMEs belong to some SU object\n".
+    "  -s            : disable check that ensures all LEXEMEs belong to some SPEAKER object\n".
+    "  -p            : disable check that ensures all speakers defined in SPKR-INFO object\n".
+    "  -e            : disable check that ensure there is an IP for each EDIT and each FILLER object\n".
+    "  -f            : disable check that ensures the file columc matches the filename\n".
+    "  -S            : allow SCTK enhancements to allow multiple itme notations in LEXEME tags\n".
+	"  -h            : print this help message\n".
+    "Input:\n".
+    "  -i <RTTM file>: an RTTM file\n\n";
+
+#my $NUM_FIELDS = 9; # count from 1
+#number of fields can be 9 or 10 (with the SLAT)
+
+my $ROUNDING_THRESHOLD = 0.000999999;
+
+my $checkFileName = 1;   ### Check the filename against the file column
+
+my %SORT_ORDER = ("NOSCORE"         =>  0,
+		  "NO_RT_METADATA"  =>  1,
+		  "SEGMENT"         =>  2,
+		  "SPEAKER"         =>  3,
+		  "SU"              =>  4,
+		  "A/P"             =>  5,
+		  "CB"              =>  6,
+		  "IP"              =>  7,
+		  "EDIT"            =>  8,
+		  "FILLER"          =>  9,
+		  "NON-SPEECH"      => 10,
+		  "NON-LEX"         => 11,
+		  "LEXEME"          => 12,
+		  "SPKR-INFO"       => 13);
+
+########################################################
+# Main
+#
+########################################################
+{
+    my ($date, $time) = date_time_stamp();
+    my $commandline = join(" ", @ARGV);
+
+    use vars qw ($opt_i $opt_u $opt_s $opt_e $opt_h $opt_f $opt_t $opt_p $opt_S);
+    getopts('i:usevhftpS');
+    die ("$USAGE") if ($opt_h) || (! $opt_i);
+
+    my @mde_types = ();
+    push (@mde_types, "SU") if ! $opt_u;
+    push (@mde_types, "SPEAKER") if ! $opt_s;
+    push (@mde_types, "EDIT") if ! $opt_e;
+    $checkFileName = ! $opt_f if $opt_f;
+    my $allowSCTKExtensions = 1 if $opt_S;
+
+	if (! $opt_t)
+    {
+    	print "$0 (version $VERSION) run on $date at $time\n";
+   		print "command line:  ", $0, " (version ", $VERSION, ") ", $commandline, "\n";
+	}
+	
+    my (%rttm_data, $data_domain);
+    get_rttm_data(\%rttm_data, \$data_domain, $opt_i, $allowSCTKExtensions);
+
+    # debug
+    #
+    my $count;
+    if ($debug) {
+#	print Dumper(%rttm_data);
+	foreach my $src (sort keys %rttm_data) {
+	    foreach my $chnl (sort keys %{$rttm_data{$src}}) {
+		foreach my $spkr (sort keys %{$rttm_data{$src}{$chnl}}) {
+		    foreach my $type (sort keys %{$rttm_data{$src}{$chnl}{$spkr}}) {
+			$count = 0;
+			foreach my $obj (@{$rttm_data{$src}{$chnl}{$spkr}{$type}}) {
+			    print "[$src $chnl $spkr $type $count]\t$obj->{TYPE}\t$obj->{SRC}\t$obj->{CHNL}\t$obj->{TBEG}\t$obj->{TDUR}\t$obj->{ORTHO}\t$obj->{STYPE}\t$obj->{SPKR}\t$obj->{CONF}\n";
+			    $count++;
+			}
+		    }
+		}
+	    }
+	}
+    }
+
+    if (check_syntax_errors(\%rttm_data, $data_domain, $opt_i, $allowSCTKExtensions) &&
+	check_logic_errors(\%rttm_data, \@mde_types)){
+	exit 0;
+    }
+    exit 1;
+}
+
+########################################################
+# Subroutines
+#
+########################################################
+sub get_rttm_data {
+
+    # read the RTTM file
+    #
+    my ($data, $domain, $file) = @_;
+
+    my (@fields, $obj);
+
+    my $line_num = 0;
+
+    ${$domain} = "";
+
+    open DATA, "$file" or die ("ERROR: Unable to open file $file\n");
+    while (<DATA>) {
+	$line_num++;
+	chomp;
+	if (/^\s*;;\s*EXP-ID:\s*(.*)/) {
+	    my $exp_id = $1;
+	    if ($exp_id =~ /bnews/i) {
+		${$domain} = "bn";
+	    } elsif ($exp_id =~ /cts/i) {
+		${$domain} = "cts";
+	    }
+	} elsif (! /^\s*;;/ && ! /^\s*$/) {
+	    @fields = split;
+	    undef $obj;
+	    $obj->{FIELD_COUNT} = scalar (@fields);
+            $obj->{LOC} = "line $line_num";
+	    $obj->{TYPE} = shift @fields;
+	    $obj->{SRC} = shift @fields;
+	    $obj->{CHNL} = shift @fields;
+	    $obj->{TBEG} = shift @fields;
+	    $obj->{TDUR} = shift @fields;
+	    $obj->{ORTHO} = shift @fields;
+	    $obj->{STYPE} = shift @fields;
+	    $obj->{SPKR} = shift @fields;
+	    $obj->{CONF} = shift @fields;
+	    $obj->{SLAT} = shift @fields;
+	    push (@{$data->{$obj->{SRC}}{$obj->{CHNL}}{$obj->{SPKR}}{$obj->{TYPE}}}, $obj);
+	}
+    }
+    close DATA;
+}
+
+sub check_syntax_errors {
+    # check for syntax errors such as missing fields,
+    # invalid field values, etc.
+    # 
+    my ($data, $domain, $file, $allowSCTKExtensions) = @_;
+
+    my $pass = 1;
+
+    $file =~ s/\.[^\.]*$//;
+    $file =~ s/.*\///;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			# make sure that we have the correct number of fields
+			#
+#			if ($obj->{FIELD_COUNT} != $NUM_FIELDS) {
+			if( ($obj->{FIELD_COUNT} != 9) && ($obj->{FIELD_COUNT} != 10) ){
+			    print "ERROR: This record has $obj->{FIELD_COUNT} fields instead of the required 9 or 10 fields; see $obj->{LOC}\n";
+			    $pass = 0;
+			}
+			
+			# for the following checks to work, all the fields must be defined
+			#
+			if (defined ($obj->{TYPE}) && defined ($obj->{SRC}) &&
+			    defined ($obj->{CHNL}) && defined ($obj->{TBEG}) &&
+			    defined ($obj->{TDUR}) && defined ($obj->{ORTHO}) &&
+			    defined ($obj->{STYPE}) && defined ($obj->{SPKR}) &&
+			    defined ($obj->{CONF})) {
+
+			    # make sure the type field is a valid RTTM type
+			    #
+			    if ($obj->{TYPE} !~ /^(SEGMENT|NOSCORE|NO_RT_METADATA|LEXEME|NON-LEX|NON-SPEECH|FILLER|EDIT|SU|IP|CB|A\/P|SPEAKER|SPKR-INFO)$/i) {
+				print "ERROR: Invalid RTTM type '$obj->{TYPE}'; see field (0) in $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			    
+			    # make sure the source field matches the base filename.
+			    # This won't work if people concat all the files into one.
+			    # Make it a warning instead
+			    #
+			    if ($checkFileName && $obj->{SRC} !~ /^$file$/i) {
+				print "WARNING: Source field '$obj->{SRC}' doesn't match the input file's base filename '$file'; see field (2) in $obj->{LOC}\n";
+			    }
+			    
+			    # make sure the channel ID has a value of 1 or 2 but 1 for BN data
+			    #
+			    if ($obj->{CHNL} !~ /^(1|2)$/) {
+				print "ERROR: Invalid channel ID; see field (3) in $obj->{LOC}\n";
+				$pass = 0;
+			    } elsif ($domain =~ /bn/i && $obj->{CHNL} != 1) {
+				print "ERROR: Expected channel ID to be 1 for BN data; see field (3) in $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			    
+			    # make sure that if it's a SPKR-INFO the start time is <NA>
+			    # else the start time must be a number and is non-negative
+			    #
+			    if ($obj->{TYPE} =~ /SPKR-INFO/i) {
+				if ($obj->{TBEG} !~ /<NA>/i) {
+				    print "ERROR: $obj->{TYPE} should not have any start time; see field (4) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } else {
+				if ($obj->{TBEG} !~ /^(\-{0,1}\d+\.?\d*|\-{0,1}\.\d+)\**$/) {
+				    print "ERROR: Expected start time to be a number; see field (4) in $obj->{LOC}\n";
+				    $pass = 0;
+				} else {
+				    my $tbeg = $obj->{TBEG};
+				    $tbeg =~ s/\*//;
+				    if ($tbeg < 0) {
+					print "ERROR: Negative start time; see field (4) in $obj->{LOC}\n";
+					$pass = 0;
+				    }
+				}
+			    }
+			    
+			    # make sure that if it's a SPKR-INFO, an IP, or a CB the duration is <NA>
+			    # else the duration must be a number and is non-negative
+			    #
+			    if ($obj->{TYPE} =~ /(SPKR-INFO|IP|CB)/i) {
+				if ($obj->{TDUR} !~ /^<NA>$/i) {
+				    print "ERROR: $obj->{TYPE} should not have any duration; see field (5) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } else {
+				if ($obj->{TDUR} !~ /^(\-{0,1}\d+\.?\d*|\-{0,1}\.\d+)\**$/) {
+				    print "ERROR: Expected duration to be a number; see field (5) in $obj->{LOC}\n";
+				    $pass = 0;
+				} else {
+				    my $tdur = $obj->{TDUR};
+				    $tdur =~ s/\*//;
+				    if ($tdur < 0) {
+					print "ERROR: Negative duration; see field (5) in $obj->{LOC}\n";
+					$pass = 0;
+				    }
+				}
+			    }
+			    
+			    # make sure that if it's not a LEXEME or a NON-LEX it doesn't have any orthography
+			    #
+			    if ($obj->{TYPE} =~ /(SEGMENT|NOSCORE|NO_RT_METADATA|NON-SPEECH|FILLER|EDIT|SU|IP|CB|A\/P|SPEAKER|SPKR-INFO)/i) {
+				if ($obj->{ORTHO} !~ /^<NA>$/i) {
+				    print "ERROR: Value for the orthography field for $obj->{TYPE} should be <NA>; see field (6) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    }
+			    
+			    # make sure the subtype for various type is valid
+			    #
+			    if ($obj->{TYPE} =~ /SEGMENT/i) {
+				if ($obj->{STYPE} !~ /^(eval|<NA>)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /(NOSCORE|NO_RT_METADATA|SPEAKER)/i) {
+				if ($obj->{STYPE} !~ /^<NA>$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /A\/P/i) {
+				if ($obj->{STYPE} !~ /^<NA>$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /LEXEME/i) {
+				if ($obj->{STYPE} !~ /^(lex|fp|frag|un-lex|for-lex|alpha|acronym|interjection|propernoun|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+				if ($obj->{STYPE} =~ /alpha/i && $obj->{ORTHO} !~ /^([A-Z]\.|[A-Z]\.\'*s|[\x80-\xff]{2}\.|<NA>)$/i) {
+				    print "ERROR: Invalid orthography for alpha $obj->{TYPE}; see field (6) '$obj->{ORTHO}' in $obj->{LOC}\n";
+				    $pass = 0;
+			        } 
+				my $ww = '[\[\]A-Z\.\-\'\x80-\xff]+';
+				if (! $allowSCTKExtensions){
+				    if ($obj->{ORTHO} !~ /^(${ww}|<NA>|%?[A-Z\.\x80-\xff]+-?)$/i) {
+					print "WARNING: Invalid orthography for $obj->{TYPE}; see field (6) '$obj->{ORTHO}' in $obj->{LOC}\n";
+				    }
+				} else {
+				    ### Destructively handle recursive alternations
+				    my $_t = $obj->{ORTHO};
+				    my $altExp = "\\{((_${ww})+_\\/)*(_${ww})+_\\}";
+				    while ($_t =~ s/_${altExp}_/_/g){ ; }
+				    if ($_t !~ /^(${ww}|(${ww}_)*$ww|$altExp|<NA>|%?[A-Z\.\x80-\xff]+-?)$/i) {
+					print "WARNING: Invalid orthography with SCTK Extensions for $obj->{TYPE}; see field (6) '$obj->{ORTHO}' in $obj->{LOC}\n";
+				    }
+				}
+			    } elsif ($obj->{TYPE} =~ /NON-LEX/i) {
+				#### fix lipsmack!!!!
+				if ($obj->{STYPE} !~ /^(laugh|breath|lipsmack|cough|sneeze|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype $obj->{STYPE}; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /NON-SPEECH/i) {
+				if ($obj->{STYPE} !~ /^(noise|music|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /FILLER/i) {
+				if ($obj->{STYPE} !~ /^(filled_pause|discourse_marker|explicit_editing_term|discourse_response|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /EDIT/i) {
+				if ($obj->{STYPE} !~ /^(repetition|restart|revision|simple|complex|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /SU/i) {
+				if ($obj->{STYPE} !~ /^(statement|question|incomplete|backchannel|unannotated|discourse_response|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /IP/i) {
+				if ($obj->{STYPE} !~ /^(edit|filler|edit\&filler|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /CB/i) {
+				if ($obj->{STYPE} !~ /^(coordinating|clausal|other)$/i) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    } elsif ($obj->{TYPE} =~ /SPKR-INFO/i) {
+				if ($obj->{STYPE} !~ /^(adult_male|adult_female|child|unknown)$/) {
+				    print "ERROR: Invalid $obj->{TYPE} subtype; see field (7) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    }
+			    
+			    # make sure that the confidence field is a number it's from 0-1 else it's <NA>
+			    #
+			    if ($obj->{CONF} !~ /<NA>/) {
+				if ($obj->{CONF} !~ /^(\-{0,1}\d+\.?\d*|\-{0,1}\.\d+)$/) {
+				    print "ERROR: Expected confidence value to be a number or <NA>; see field (9) in $obj->{LOC}\n";
+				    $pass = 0;
+				} elsif ($obj->{CONF} < 0 || $obj->{CONF} > 1) {
+				    print "ERROR: Expected confidence value to be [0,1]; see field (9) in $obj->{LOC}\n";
+				    $pass = 0;
+				}
+			    }
+			    
+			    # make sure that the speaker id field for certain types match the speaker id
+			    # given in the SPKR-INFO object
+			    #
+			    if( ! $opt_p )
+			    {
+					if ($type !~ /(SPKR-INFO|NOSCORE|NONSPEECH)/i && ! find_speaker($obj->{SPKR}, $data) ) {
+					print "ERROR: Speaker $obj->{SPKR} doesn't match any of the speaker IDs in SPKR-INFO objects; see $obj->{LOC}\n";
+					$pass = 0;
+				}
+			    }
+			}
+                    }
+                }
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub check_logic_errors {
+    # check for logic errors such as overlapping words from the same speaker, etc.
+    #
+    my ($data, $mde_types) = @_;
+
+    # sort the data 
+    #
+    foreach my $src (sort keys %{$data}) {
+        foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach  my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type !~ /SPKR-INFO/i) {
+			@{$data->{$src}{$chnl}{$spkr}{$type}} = 
+			    sort {$a->{TBEG} <=> $b->{TBEG}} @{$data->{$src}{$chnl}{$spkr}{$type}};
+		    }
+		}
+	    }
+	}
+    }
+
+    if (check_metadata_overlap($data) &&
+	check_metadata_content($data) &&
+	check_partial_word_coverage($data) &&
+	check_noscore_overlap($data)) {
+	foreach my $type (@$mde_types) {
+	    if ($type =~ /(SU|SPEAKER)/i) {
+		if (! ensure_word_covered_by_metadata_of_type($data, $type)) {
+		    return 0;
+		}
+	    }
+	    if ($type =~ /(EDIT|FILLER)/i) {
+		if (! ensure_ip_existed_for_edit_and_filler($data)) {
+		    return 0;
+		}
+	    }
+	}
+	return 1;
+    } else {
+	return 0;
+    }
+}
+
+sub check_metadata_overlap {
+    # check if two metadata objects of the same type
+    # from the same speaker overlap
+    #
+    my ($data) = @_;
+
+    my $pass = 1;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type !~ /SPKR-INFO/i) {
+			my $prev_etime = 0;
+			foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			    if (($prev_etime - $obj->{TBEG}) > $ROUNDING_THRESHOLD) {
+				print "ERROR: Speaker $spkr has ${type}s ending at $prev_etime and starting at $obj->{TBEG}; ${type}s overlap; see $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			    if ($type =~ /(CB|IP)/i) {
+				$prev_etime = $obj->{TBEG};
+			    } else {
+				$prev_etime = $obj->{TBEG} + $obj->{TDUR};
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub check_metadata_content {
+    # check if a metadata object of type SU, EDIT, or FILLER contains any words.
+    # The words could be missing or simply didn't get transcribed
+    #
+    my ($data) = @_;
+
+    my $pass = 1;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type =~ /(SU|EDIT|FILLER)/i) {
+			foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			    if (! find_word($data->{$src}{$chnl}{$spkr}{LEXEME}, $obj->{TBEG}, $obj->{TBEG} + $obj->{TDUR})) {
+				print "ERROR: $type at $obj->{TBEG} contains no words; see $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub check_partial_word_coverage {
+    # check if a given word partially overlaps with
+    # any metadata object of type SU, EDIT, or FILLER
+    #
+    my ($data) = @_;
+    
+    my $pass = 1;
+    
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type =~ /LEXEME/i) {
+			foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			    foreach my $mde_type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+				if ($mde_type =~ /(SU|EDIT|FILLER)/i) {
+				    if (find_partial_coverage($data->{$src}{$chnl}{$spkr}{$mde_type}, $obj->{TBEG}, $obj->{TBEG} + $obj->{TDUR})) {
+					print "ERROR: Word at $obj->{TBEG} is partially covered by $mde_type object; see $obj->{LOC}\n";
+					$pass = 0;
+				    }
+				}
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub check_noscore_overlap {
+    # check if a metadata object of type
+    # SU, EDIT, FILLER, SPEAKER, LEXEME, NONLEX, NONSPEECH
+    # partially overlap with noscore regions
+    #
+    my ($data) = @_;
+
+    my $pass = 1;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    next if (! defined ($data->{$src}{$chnl}{"<NA>"}{NOSCORE}));
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type =~ /(SU|EDIT|FILLER|SPEAKER|LEXEME|NON-LEX|NON-SPEECH)/i) {
+			foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			    if (find_partial_coverage($data->{$src}{$chnl}{"<NA>"}{NOSCORE}, $obj->{TBEG}, $obj->{TBEG} + $obj->{TDUR})) {
+				print "ERROR: Speaker $spkr has $type partially overlapping with NOSCORE tag at $obj->{TBEG}; see $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub ensure_word_covered_by_metadata_of_type {
+    # make sure that all words belong to some metadata object of the given type.
+    #
+    my ($data, $mde_type) = @_;
+
+    my $pass = 1;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $type (sort keys %{$data->{$src}{$chnl}{$spkr}}) {
+		    if ($type =~ /LEXEME/i) {
+			foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{$type}}) {
+			    if (! find_type($data->{$src}{$chnl}{$spkr}{$mde_type}, $obj->{TBEG}, $obj->{TBEG} + $obj->{TDUR})) {
+				print "ERROR: Word at $obj->{TBEG} doesn't belong to any $mde_type object; see $obj->{LOC}\n";
+				$pass = 0;
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub ensure_ip_existed_for_edit_and_filler {
+    # make sure that for each EDIT or FILLER there is
+    # a valid IP that follows or precedes it
+    #
+    my ($data) = @_;
+
+    my $pass = 1;
+
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		if (exists($data->{$src}{$chnl}{$spkr}{SU})) {
+		    my @sus = @{$data->{$src}{$chnl}{$spkr}{SU}};
+		    for (my $i = 0; $i < @sus; $i++) {
+#			print "SU $sus[$i]->{TBEG}\n";
+			my @edits = ();
+			my @fillers = ();
+			my @ips = ();
+			if (exists($data->{$src}{$chnl}{$spkr}{EDIT})) {
+			    @edits = extract_mde_of_type($sus[$i]->{TBEG},
+							 $sus[$i]->{TBEG} + $sus[$i]->{TDUR},
+							 $i,
+							 "EDIT",
+							 $data->{$src}{$chnl}{$spkr}{EDIT},
+							 \@sus);
+			}
+			if (exists($data->{$src}{$chnl}{$spkr}{FILLER})) {
+			    @fillers = extract_mde_of_type($sus[$i]->{TBEG},
+							   $sus[$i]->{TBEG} + $sus[$i]->{TDUR},
+							   $i,
+							   "FILLER",
+							   $data->{$src}{$chnl}{$spkr}{FILLER},
+							   \@sus);
+			}
+			if (exists($data->{$src}{$chnl}{$spkr}{IP})) {
+			    @ips = extract_mde_of_type($sus[$i]->{TBEG},
+						       $sus[$i]->{TBEG} + $sus[$i]->{TDUR},
+						       $i,
+						       "IP",
+						       $data->{$src}{$chnl}{$spkr}{IP},
+						       \@sus);
+			}
+#			print "edits\n";
+#			print Dumper(@edits);
+#			print "fillers\n";
+#			print Dumper(@fillers);
+#			print "ips\n";
+#			print Dumper(@ips);
+			
+			foreach my $obj (@edits) {
+			    my ($status, $ip_stype, $ip_pos) = find_object($obj->[0], $obj->[1], \@ips);
+			    if ($status == 1) {
+#				print "$status, edit_type=$obj->[1] ip_stype=$ip_stype $ip_pos\n";
+				if ($ip_stype =~ /^EDIT$/i) {
+#				    print "case 1\n";
+				    $obj->[4] = 1;
+				    $ips[$ip_pos]->[4] = 1;
+				} elsif ($ip_stype =~ /^EDIT&FILLER$/i) {
+				    foreach my $obj2 (@fillers) {
+					my ($status2, $filler_type, $filler_pos) = find_object($obj->[0], "FILLER", \@fillers);
+					if ($status2 == 1) {
+#					    print "$status2, filler_type=FILLER filler_type=$filler_type $filler_pos\n";
+#					    print "case 2\n";
+					    $obj->[4] = 1;
+					    $ips[$ip_pos]->[4] = 1;
+                                            $fillers[$filler_pos]->[4] = 1;
+					}
+				    }
+				}
+			    }
+			}
+			
+			foreach my $obj (@fillers) {
+			    my ($status, $ip_stype, $ip_pos) = find_object($obj->[0], $obj->[1], \@ips);
+			    if ($status == 1) {
+#				print "$status, filler type=$obj->[1] ip stype=$ip_stype, $ip_pos\n";
+#				print "case 3\n";
+
+				if ($ip_stype =~ /^FILLER$/i) {
+				    $obj->[4] = 1;
+				    $ips[$ip_pos]->[4] = 1;
+				}
+			    }
+			}
+
+			foreach my $obj (@edits) {
+			    if ($obj->[4] == 0) {
+				print "ERROR: EDIT at $obj->[2] doesn't have a matching IP; see $obj->[3]\n";
+				$pass = 0;
+			    }
+			}
+			foreach my $obj (@fillers) {
+			    if ($obj->[4] == 0) {
+				print "ERROR: FILLER at $obj->[2] doesn't have a matching IP; see $obj->[3]\n";
+				$pass = 0;
+			    }
+			}
+			foreach my $obj (@ips) {
+			    if ($obj->[4] == 0) {
+				print "ERROR: IP at $obj->[2] doesn't have a matching EDIT and/or FILLER; see line $obj->[3]\n";
+				$pass = 0;
+			    }
+			}
+		    }
+		}
+	    }
+	}
+    }
+    if (! $pass) {
+	print "Validation aborted\n";
+    }
+    return $pass;
+}
+
+sub find_object {
+    my ($time, $type, $data1) = @_;
+#    print "find object ------------\n";
+#    print "  $time $type\n";
+    for (my $i=0; $i < @$data1; $i++) {
+#	print "      ${$data1}[$i]->[0]\n";
+	if (abs(${$data1}[$i]->[0] - $time) < $ROUNDING_THRESHOLD &&
+	    ${$data1}[$i]->[4] != 1) {
+#            print "      found\n";
+	    return (1, ${$data1}[$i]->[1], $i);
+	}
+    }
+    return (0, undef, undef);
+}
+
+sub find_partial_coverage {
+    # determine if a given object (indicated by its start and end times)
+    # partially overlap with any object of the given type
+    #
+    my ($data, $start, $end) = @_;
+
+    my $obj;
+
+    foreach $obj (@{$data}) {
+	my $curr_beg = $obj->{TBEG};
+	my $curr_end = $obj->{TBEG} + $obj->{TDUR};
+
+	# case 1       #########
+	#        ####
+        #
+	# case 2 ########
+	#                 ####
+	#
+	# case 3 ########
+	#          ####
+	#
+	# everything else is partially covered
+	#
+	if (less_than($end, $curr_beg) || equal_to($end, $curr_beg)) {
+	    # case 1
+	    next;
+	} elsif (greater_than($start, $curr_end) || equal_to($start, $curr_end)) {
+	    # case 2
+	    next;
+	} elsif ( ( greater_than($start, $curr_beg) || equal_to($start, $curr_beg) ) &&
+		  ( less_than($end, $curr_end) || equal_to($end, $curr_end) ) ) {
+	    # case 3
+	    next;
+	} else {
+	    # others
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+sub find_type {
+    # determine if a given word (indicated by its start and end times)
+    # belongs to any metadata object of the given type
+    #
+    my ($data, $token_start, $token_end) = @_;
+
+    foreach my $mde_obj (@{$data}) {
+	if ($token_start + $ROUNDING_THRESHOLD >= $mde_obj->{TBEG} &&
+	    $token_end - $ROUNDING_THRESHOLD <= $mde_obj->{TBEG} + $mde_obj->{TDUR}) {
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+sub find_word {
+    # determine if a given metadata object (indicated by its start and end times)
+    # contains any word
+    #
+    my ($data, $mde_obj_start, $mde_obj_end) = @_;
+
+    foreach my $token (@{$data}) {
+	if ($token->{TBEG} + $ROUNDING_THRESHOLD >= $mde_obj_start &&
+	    $token->{TBEG} + $token->{TDUR} - $ROUNDING_THRESHOLD <= $mde_obj_end) {
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+sub find_ip {
+    # determine if an IP is at the time given
+    #
+    my ($data, $tbeg, $tend, $target) = @_;
+    if ($tbeg && $tend && $target) {
+	foreach my $ip (@{$data}) {
+	    if ( $tbeg <= $ip->{TBEG} &&
+		 $ip->{TBEG} <= $tend &&
+		 abs($ip->{TBEG} - $target) < $ROUNDING_THRESHOLD ) {
+		return $ip;
+	    }
+	}
+    }
+    return undef;
+}
+
+sub find_speaker {
+    # determine if a given speaker is in the data
+    #
+    my ($src_spkr, $data) = @_;
+
+    my ($curr_spkr);
+    foreach my $src (sort keys %{$data}) {
+	foreach my $chnl (sort keys %{$data->{$src}}) {
+	    foreach my $spkr (sort keys %{$data->{$src}{$chnl}}) {
+		foreach my $obj (@{$data->{$src}{$chnl}{$spkr}{'SPKR-INFO'}}) {
+		    $curr_spkr = $obj->{SPKR};
+		    if ($src_spkr =~ /^$curr_spkr$/i) {
+			return 1;
+		    }
+		}
+	    }
+	}
+    }
+    return 0;
+}
+
+sub extract_mde_of_type {
+    # for a given SU (indicated by its begin and end times)
+    # return all the MDE objects of the given type in that SU
+    #
+    my ($su_tbeg, $su_tend, $su_pos, $mde_type, $data, $sus) = @_;
+    my @ip_time = ();
+    if ($mde_type =~ /EDIT/i) {
+	foreach my $obj (@$data) {
+	    if ( (greater_than($obj->{TBEG}, $su_tbeg) || equal_to ($obj->{TBEG}, $su_tbeg)) &&
+		 (less_than($obj->{TBEG} + $obj->{TDUR}, $su_tend) || equal_to ($obj->{TBEG} + $obj->{TDUR}, $su_tend)) ) {
+		push (@ip_time, [ $obj->{TBEG} + $obj->{TDUR}, uc($obj->{TYPE}), $obj->{TBEG}, $obj->{LOC}, 0 ]);
+	    }
+	}
+    } elsif ($mde_type =~ /FILLER/i) {
+	foreach my $obj (@$data) {
+	    if ( (greater_than($obj->{TBEG}, $su_tbeg) || equal_to ($obj->{TBEG}, $su_tbeg)) &&
+		 (less_than($obj->{TBEG} + $obj->{TDUR}, $su_tend) || equal_to ($obj->{TBEG} + $obj->{TDUR}, $su_tend)) ) {
+		push (@ip_time, [ $obj->{TBEG}, uc($obj->{TYPE}), $obj->{TBEG}, $obj->{LOC}, 0 ]);
+	    }
+	}
+    } elsif ($mde_type =~ /IP/i) {
+	foreach my $obj (@$data) {
+	    if ( (greater_than($obj->{TBEG}, $su_tbeg) && less_than($obj->{TBEG}, $su_tend) ) ||
+		 equal_to($obj->{TBEG}, $su_tbeg) ) {
+		push (@ip_time, [ $obj->{TBEG}, uc($obj->{STYPE}), $obj->{TBEG}, $obj->{LOC}, 0 ]);
+	    } elsif ( equal_to($obj->{TBEG}, $su_tend) ) {
+		if ($su_pos + 1 <= scalar($sus)) {
+		    if ( equal_to($su_tend, $sus->[$su_pos + 1]->{TBEG}) &&
+			 $obj->{STYPE} !~ /filler/i ) {
+			push (@ip_time, [ $obj->{TBEG}, uc($obj->{STYPE}), $obj->{TBEG}, $obj->{LOC}, 0 ]);
+		    }
+		}
+	    }
+	}
+    }
+    return @ip_time;
+}
+
+sub sort_data {
+   return ($a->{TBEG} < $b->{TBEG} - $ROUNDING_THRESHOLD ? -1 :
+	   ($a->{TBEG} > $b->{TBEG} + $ROUNDING_THRESHOLD ?  1 :
+	    $SORT_ORDER{$a->{TYPE}} <=> $SORT_ORDER{$b->{TYPE}}));
+}
+
+sub less_than {
+    # if a < b return 1
+    #
+    my ($a, $b) = @_;
+    if ($a + $ROUNDING_THRESHOLD < $b) {
+	return 1;
+    }
+    return 0;
+}
+
+sub greater_than {
+    # if a > b return 1
+    #
+    my ($a, $b) = @_;
+    if ($a > $b + $ROUNDING_THRESHOLD) {
+	return 1;
+    }
+    return 0;
+}
+
+sub equal_to {
+    # if a == b return 1
+    #
+    my ($a, $b) = @_;
+    if ( abs($a - $b) < $ROUNDING_THRESHOLD) {
+	return 1;
+    }
+    return 0;
+}
+
+sub date_time_stamp {
+    my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime();
+    my @months = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
+    my ($date, $time);
+
+    $time = sprintf "%2.2d:%2.2d:%2.2d", $hour, $min, $sec;
+    $date = sprintf "%4.4s %3.3s %s", 1900+$year, $months[$mon], $mday;
+    return ($date, $time);
+}

SCTK/src/sclite/alex.h

@@ -0,0 +1,23 @@
+/* File: alex.h                                                 */
+
+typedef struct AUTO_LEX_struct{
+    int max,                            /* total possible tokens */
+        num;                            /* current number of stored tokens */
+    TEXT **str;                         /* pointer to the list of texts */
+    int *field_a;                       /* pointer to a list if intergers */
+                                        /* used for any purpose what-so-ever, */
+                                        /* Intitializd to ZERO */
+    int *field_b;                       /* Same usage as field_a . . . */
+    double *field_c;                       /* Same usage as field_a . . . */
+    int *sort;                          /* Sorted list of the texts */
+} AUTO_LEX;
+
+void AUTO_LEX_init(AUTO_LEX *alex, int size);
+void AUTO_LEX_free(AUTO_LEX *alex);
+int AUTO_LEX_insert(AUTO_LEX *alex, TEXT *new);
+void AUTO_LEX_dump(AUTO_LEX *alex, FILE *fp);
+TEXT *AUTO_LEX_get(AUTO_LEX *alex, int ind);
+void AUTO_LEX_printout(AUTO_LEX *alex, FILE *fp, char *title, int threshhold);
+double AUTO_LEX_get_c(AUTO_LEX *alex, int ind);
+int AUTO_LEX_find(AUTO_LEX *alex, TEXT *str);
+

SCTK/src/sclite/align.c

@@ -0,0 +1,1443 @@
+#include <unistd.h>
+#include "sctk.h"
+
+
+void remove_id(TEXT *utt, TEXT **id, int *len);
+int extract_speaker(TEXT *hyp_id, TEXT *sname, enum id_types id);
+void load_refs(SCORES *sc, char *hyp_file, char *ref_file, TEXT ***rset, TEXT ***refid, int *refcnt, int case_sense);
+void free_refs(TEXT ***rset, TEXT ***ref_id, int refcnt);
+void convert_text_to_word_list(char *file, char *words,int case_sense);
+void expand_words_to_chars(ARC *arc, void *ptr);
+void decode_opt_del(ARC *arc, void *ptr);
+void decode_fragment(ARC *arc, void *ptr);
+void lookup_word_weight(ARC *arc, void *ptr);
+void set_word_duration(ARC *arc, void *ptr);
+void set_word_start(ARC *arc, void *ptr);
+void set_word_conf(ARC *arc, void *ptr);
+void set_word_opt_del(ARC *arc, void *ptr);
+
+
+#define LOAD_BUFF_LEN 1000
+
+void remove_id(TEXT *utt, TEXT **id, int *id_len){
+    TEXT *L_paren, *R_paren;
+    
+    R_paren = TEXT_strrchr(utt,')');
+    L_paren = TEXT_strrchr(utt,'(');
+
+    **id = NULL_TEXT;
+
+    if (R_paren == NULL && L_paren == NULL)	
+	return;
+
+    if (R_paren == NULL || L_paren == NULL){
+	fprintf(stderr,"Error: Unparsable utterance id %s\n",utt);
+	exit(1);
+    }
+
+    /* make sure there is enough space */
+    if (R_paren - L_paren + 1 + 1 > *id_len){
+	*id_len = R_paren - L_paren + 1 + 1 + 10; 
+	free_singarr(*id,TEXT);
+	alloc_singZ((*id),*id_len,TEXT,'\0');
+    }
+    TEXT_strBcpy(*id,L_paren,R_paren - L_paren + 1);
+    *(*id + (R_paren - L_paren + 1)) = NULL_TEXT;
+    *L_paren = NULL_TEXT;
+}
+
+int extract_speaker(TEXT *id, TEXT *sname, enum id_types idt){
+    char *proc = "extract_speaker";
+    TEXT *p;
+    switch (idt){
+      case SP:
+	TEXT_strcpy(sname, id);
+	return(0);
+      case ATIS:
+      case WSJ:
+	TEXT_strBcpy(sname, id + 1, 3);
+	sname[3] = NULL_TEXT;
+	return(0);
+      case RM:
+      case SPUID:
+      case SWB:
+	if (((p = TEXT_strchr(id,'-')) == NULL) &&
+	    ((p = TEXT_strchr(id,'_')) == NULL)){
+	    fprintf(stderr,"Error: %s can't locate ",proc);
+	    switch (idt){
+	      case RM: 	   fprintf(stderr,"%s id %s\n","RM",id); break;
+	      case SPUID:  fprintf(stderr,"%s id %s\n","SPU_ID",id); break;
+	      case SWB:	   fprintf(stderr,"%s id %s\n","SWB",id); break;
+	      case WSJ:
+	      case ATIS:
+	      default:     fprintf(stderr,"*** internal error\n");
+	    }
+	    return(1);
+        }
+	TEXT_strBcpy(sname, id + 1, p - (id+1)); 
+	sname[p - (id+1)] = NULL_TEXT; 
+	return(0);
+      default:
+	fprintf(stderr,"Error: %s unknown id type\n",proc);
+	return(1);
+    }
+}
+
+/* free the reference transcripts */
+void free_refs(TEXT ***rset, TEXT ***ref_id, int refcnt){
+    free_2dimarr((*rset),refcnt,TEXT); 
+    free_2dimarr((*ref_id),refcnt,TEXT);
+}
+
+void load_refs(SCORES *sc, char *hyp_file, char *ref_file, TEXT ***rset, TEXT ***refid, int *refcnt, int case_sense){
+    TEXT *in_buf, *id_buf;
+    int in_buf_len=LOAD_BUFF_LEN, id_buf_len=LOAD_BUFF_LEN;
+    FILE *fp;
+    TEXT **idset, **tidset, **refset, **refids;
+    int max_ids, num_ids, i, num_ref, failure=0;
+
+    max_ids = 3; num_ids = 0; num_ref = 0;
+    alloc_singZ(idset,max_ids,TEXT *,(TEXT *)0);
+    alloc_singZ(in_buf,in_buf_len,TEXT,'\0');
+    alloc_singZ(id_buf,id_buf_len,TEXT,'\0');
+
+    /* Pre-Read the entire hypothesis file */
+    if ((hyp_file == NULL) || (*hyp_file == '\0') || 
+        ((fp = fopen(hyp_file,"r")) == NULL)){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+    while (!feof(fp)){
+        *in_buf  = *(in_buf+1) = COMMENT_CHAR; 
+        *(in_buf+2) = NULL_TEXT;
+        while (!feof(fp) && (parse_input_comment_line(sc,in_buf) ||
+			     TEXT_is_comment_info(in_buf) ||
+			     TEXT_is_empty(in_buf))){
+            if (TEXT_ensure_fgets(&in_buf,&in_buf_len,fp) == NULL)
+               *in_buf = NULL_TEXT;
+	}
+	if (!feof(fp)){
+	    if (!case_sense)
+		TEXT_str_case_change_with_mem_expand(&in_buf, &in_buf_len, 1);
+	    remove_id(in_buf,&id_buf,&id_buf_len);
+	    if (num_ids + 1 >= max_ids) {
+		/* expanding the array */
+		alloc_singZ(tidset,max_ids*2,TEXT *, (TEXT *)0);
+		memcpy(tidset,idset,sizeof(TEXT *) * num_ids);
+		free_singarr(idset,TEXT *);
+		idset = tidset;
+		max_ids *= 2;
+	    }
+	    idset[num_ids++] = TEXT_strdup(id_buf);
+	}
+    }
+    fclose(fp);
+
+    /* sort the id's in-place */
+    qsort((void *)idset,num_ids,sizeof(TEXT *),qsort_TEXT_strcmp);
+
+    /* space for the reference transcripts allocate */
+    alloc_singZ(refset,num_ids,TEXT *,(TEXT *)0);  
+    
+    /* Read the reference file, saving the needed texts */
+    if ((ref_file == NULL) || (*ref_file == '\0') || 
+        ((fp = fopen(ref_file,"r")) == NULL)){
+        fprintf(stderr,"Can't open input Reference file %s\n",ref_file);
+        exit(1);
+    }
+    while (!feof(fp)){
+	TEXT **ind;
+	int rind;
+
+        *in_buf  = *(in_buf+1) = COMMENT_CHAR; 
+        *(in_buf+2) = NULL_TEXT;
+        while (!feof(fp) && (parse_input_comment_line(sc,in_buf) ||
+			     TEXT_is_comment_info(in_buf) ||
+			     TEXT_is_empty(in_buf))){
+            if (TEXT_ensure_fgets(&in_buf,&in_buf_len,fp) == NULL)
+               *in_buf = '\0';
+	}
+        if (feof(fp))
+            break;
+
+	if (!case_sense)
+	    TEXT_str_case_change_with_mem_expand(&in_buf, &in_buf_len, 1);
+
+	remove_id(in_buf,&id_buf,&id_buf_len);
+	/* Search for the id in idset */
+	if ((ind = (TEXT **)bsearch(id_buf,idset,num_ids,sizeof(TEXT *),
+				    bsearch_TEXT_strcmp)) != NULL){
+	    rind = ind - idset;
+	    if (refset[rind] != (TEXT *)0){
+		if (TEXT_strBcmp(in_buf,refset[rind],TEXT_strlen(in_buf))!= 0){
+		    fprintf(stderr,"Error: double reference text for id '%s'\n",
+			    id_buf);
+		    failure=1;
+		} 
+	    } else {
+		refset[rind] = TEXT_strdup(in_buf);
+		num_ref++;
+	    }
+	}
+    }
+    fclose(fp);
+    
+    if (num_ref != num_ids){
+	failure=1;
+	fprintf(stderr,"Error: Not enough Reference files loaded\nMissing:\n");
+	for (i=0; i<num_ref; i++)
+	    if (refset[i] == (TEXT *)0)
+	      fprintf(stderr,"    %s\n",idset[i]);
+    }
+
+    /* space for the reference id's */
+    alloc_singZ(refids,num_ref,TEXT *,(TEXT *)0);  
+    /* transfer the id's to the new array */
+    memcpy(refids,idset,sizeof(TEXT *) * num_ref);
+    /* free the idset array */
+    free_singarr(idset, TEXT *)
+
+    free_singarr(in_buf,TEXT);
+    free_singarr(id_buf,TEXT);
+
+    *rset = refset;
+    *refcnt = num_ref;
+    *refid = refids;
+
+    if (failure)
+	exit(1);
+}
+
+PATH *infer_word_seg_algo2(TEXT *ref, TEXT *hyp, NETWORK *hnet, int case_sense,char *id, char *lex_fname, int fcorr, int opt_del, int flags)
+{
+    char *proc="infer_word_seg_algo2";
+    NETWORK *refnet, *hypnet;
+    PATH *path;
+    static TEXT_LIST *tl = (TEXT_LIST *)0;
+    static int max_word_len=1;
+    int char_align = (flags == INF_ASCII_TOO) ? CALI_ON : CALI_ON+CALI_NOASCII;
+
+    if (hyp != (TEXT *)0 && hnet != (NETWORK *)0){
+	fprintf(scfp,"Error: %s passed ambiguous arguments "
+		"for 'hyp' and 'hnet'\n",proc);
+	exit(1);
+    }
+
+    if (tl == (TEXT_LIST *)0){
+	int i, s;
+	if ((tl = load_TEXT_LIST(lex_fname,0)) == (TEXT_LIST *)0) {
+	    fprintf(scfp,"Error: Can't load lexicon file '%s'\n",lex_fname);
+	    exit(0);
+	}
+	/* compute the largest word, (in characters) in the TEXT_LIST */
+	for (i=0; i<tl->num; i++)
+	    if ((s = TEXT_chrlen(tl->elem[i])) > max_word_len)
+		max_word_len = s;
+    }
+
+    {
+        TEXT *refCC = (case_sense ? TEXT_strdup(TEXT_str_to_master(ref, 1)) : ref);
+        if ((refnet = Network_create_from_TEXT(ref, "Reference Net",
+					   print_WORD,
+					   equal_WORD2,
+					   release_WORD, null_alt_WORD,
+					   opt_del_WORD,
+					   copy_WORD, make_empty_WORD,
+					   use_count_WORD))
+	== NULL_NETWORK){
+            fprintf(stderr,"Network_create_from_TEXT failed\n");
+	    exit(1);
+        }
+        if (case_sense){
+            free_singarr(refCC, TEXT);
+        }
+    }
+    
+    if (hyp != (TEXT *)0){
+	TEXT *hypCC = (case_sense ? TEXT_strdup(TEXT_str_to_master(hyp, 1)) : hyp);
+	if ((hypnet = Network_create_from_TEXT(hyp,"Hypothesis Net",
+					       print_WORD,
+					       equal_WORD2,
+					       release_WORD, null_alt_WORD,
+					       opt_del_WORD,
+					       copy_WORD, make_empty_WORD,
+					       use_count_WORD))
+	    == NULL_NETWORK){
+    fprintf(stderr,"Network_create_from_TEXT failed\n");
+	    exit(1);
+	}
+        if (case_sense){
+           free_singarr(hypCC, TEXT);
+        }
+    } else 
+	hypnet = hnet;
+
+    if (opt_del){
+      Network_traverse(hypnet,NULL,0,decode_opt_del,0,0);
+      Network_traverse(refnet,NULL,0,decode_opt_del,0,0);
+    }
+
+    if (fcorr){
+      Network_traverse(hypnet,NULL,0,decode_fragment,0,0);
+      Network_traverse(refnet,NULL,0,decode_fragment,0,0);
+    }
+
+    Network_traverse(refnet,NULL,0,expand_words_to_chars,&char_align,0);
+
+    Network_fully_connect_cond(refnet, max_word_len, append_WORD_no_NULL,
+			       WORD_in_TEXT_LIST, tl);
+
+    /* printf("-----------------\n");
+       Network_traverse(hypnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);
+       Network_traverse(refnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);  */
+
+    Network_dpalign(refnet,hypnet,wwd_WORD,&path,FALSE);
+    PATH_add_utt_id(path,(char *)id);
+    PATH_set_sequence(path);
+    if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+    
+    Network_destroy(refnet);
+    Network_destroy(hypnet);
+    
+    /* free_TEXT_LIST (&tl); */
+    return(path);
+}
+
+PATH *infer_word_seg_algo1(TEXT *ref, TEXT *hyp, NETWORK *hnet, int case_sense,char *id, char *lex_fname, int fcorr, int opt_del, int flags)
+{
+    char *proc="infer_word_seg_algo1";
+    NETWORK *refnet, *hypnet;
+    PATH *path;
+    static TEXT_LIST *tl = (TEXT_LIST *)0;
+    static int max_word_len=1;
+    int char_align = (flags == INF_ASCII_TOO) ? CALI_ON : CALI_ON+CALI_NOASCII;
+
+    if (hyp != (TEXT *)0 && hnet != (NETWORK *)0){
+	fprintf(scfp,"Error: %s passed ambiguous arguments "
+		"for 'hyp' and 'hnet'\n",proc);
+	exit(1);
+    }
+
+    if (tl == (TEXT_LIST *)0){
+	int i, s;
+	if ((tl = load_TEXT_LIST(lex_fname,0)) == (TEXT_LIST *)0) {
+	    fprintf(scfp,"Error: Can't load lexicon file '%s'\n",lex_fname);
+	    exit(0);
+	}
+	/* compute the largest word, (in characters) in the TEXT_LIST */
+	for (i=0; i<tl->num; i++)
+	    if ((s = TEXT_chrlen(tl->elem[i])) > max_word_len)
+		max_word_len = s;
+    }
+
+    TEXT *refCC = (! case_sense ? TEXT_strdup(TEXT_str_to_master(ref, 1)) : ref);
+//db=20;
+    if ((refnet = Network_create_from_TEXT(refCC, "Reference Net",
+					   print_WORD,
+					   equal_WORD2,
+					   release_WORD, null_alt_WORD,
+					   opt_del_WORD,
+					   copy_WORD, make_empty_WORD,
+					   use_count_WORD))
+	== NULL_NETWORK){
+	fprintf(stderr,"Network_create_from_TEXT failed\n");
+	exit(1);
+    }
+
+    if (! case_sense){
+        free_singarr(refCC, TEXT);
+    }    
+    if (hyp != (TEXT *)0){
+	TEXT *hypCC = (! case_sense ? TEXT_strdup(TEXT_str_to_master(hyp, 1)) : hyp);
+	if ((hypnet = Network_create_from_TEXT(hypCC,"Hypothesis Net",
+					       print_WORD,
+					       equal_WORD2,
+					       release_WORD, null_alt_WORD,
+					       opt_del_WORD,
+					       copy_WORD, make_empty_WORD,
+					       use_count_WORD))
+	    == NULL_NETWORK){
+    fprintf(stderr,"Network_create_from_TEXT failed\n");
+	    exit(1);
+	}
+        if (! case_sense){
+            free_singarr(hypCC, TEXT);
+        }
+    } else 
+	hypnet = hnet;
+
+//       Network_traverse(hypnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);
+
+    if (opt_del){
+      Network_traverse(hypnet,NULL,0,decode_opt_del,0,0);
+      Network_traverse(refnet,NULL,0,decode_opt_del,0,0);
+    }
+
+    if (fcorr){
+      Network_traverse(hypnet,NULL,0,decode_fragment,0,0);
+      Network_traverse(refnet,NULL,0,decode_fragment,0,0);
+    }
+
+//    Network_traverse(hypnet,NULL,0,0c,0,NT_CA_For+NT_Verbose);
+    Network_traverse(hypnet,NULL,0,expand_words_to_chars,&char_align,0);
+//    Network_traverse(hypnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);
+
+    Network_fully_connect_cond(hypnet, max_word_len, append_WORD_no_NULL,
+			       WORD_in_TEXT_LIST, tl);
+
+//     printf("-----------------\n");
+//       Network_traverse(hypnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);
+//       Network_traverse(refnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);  
+
+    Network_dpalign(refnet,hypnet,wwd_WORD,&path,FALSE);
+    PATH_add_utt_id(path,(char *)id);
+    PATH_set_sequence(path);
+    if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+    
+    Network_destroy(refnet);
+    Network_destroy(hypnet);
+    
+    /* free_TEXT_LIST (&tl); */
+    return(path);
+}
+
+PATH *network_dp_align_texts(TEXT *ref, NETWORK *rnet, TEXT *hyp, NETWORK *hnet, int char_align, int case_sense, char *id, int fcorr, int opt_del, int time_align, WWL *wwl, char *lm_file)
+{
+    char *proc="network_dp_align_texts";
+    NETWORK *refnet, *hypnet;
+    PATH *path;
+
+    if (hyp != (TEXT *)0 && hnet != (NETWORK *)0){
+	fprintf(scfp,"Error: %s passed ambiguous arguments "
+		"for 'hyp' and 'hnet'\n",proc);
+	exit(1);
+    }
+
+    if (ref != (TEXT *)0 && rnet != (NETWORK *)0){
+	fprintf(scfp,"Error: %s passed ambiguous arguments "
+		"for 'ref' and 'rnet'\n",proc);
+	exit(1);
+    }
+//    printf("\nREF -> %s\n",ref);
+//    printf("HYP -> %s\n",hyp);
+    if (ref != (TEXT *)0){
+      TEXT *refCC = (! case_sense ? TEXT_strdup(TEXT_str_to_master(ref, 1)) : ref);
+      
+      if ((refnet = Network_create_from_TEXT(refCC, "Reference Net",print_WORD,
+					     equal_WORD2,
+					     release_WORD, null_alt_WORD,
+					     opt_del_WORD,
+					     copy_WORD, make_empty_WORD,
+					     use_count_WORD))
+	  == NULL_NETWORK){
+	fprintf(stderr,"Network_create_from_TEXT failed\n");
+	exit(1);
+      }
+      if (! case_sense){
+        free_singarr(refCC, TEXT);
+      }
+    } else {
+      refnet = rnet;
+    }
+
+    if (hyp != (TEXT *)0){
+        TEXT *hypCC = (!case_sense ? TEXT_strdup(TEXT_str_to_master(hyp, 1)) : hyp);
+
+	if ((hypnet = Network_create_from_TEXT(hypCC,"Hypothesis Net",
+					       print_WORD,
+					       equal_WORD2,
+					       release_WORD, null_alt_WORD,
+					       opt_del_WORD,
+					       copy_WORD, make_empty_WORD,
+					       use_count_WORD))
+	    == NULL_NETWORK){
+	    fprintf(stderr,"Network_create_from_TEXT failed\n");
+	    exit(1);
+	}
+      if (! case_sense){
+        free_singarr(hypCC, TEXT);
+      }
+    } else 
+	hypnet = hnet;
+
+    if (opt_del){
+      Network_traverse(hypnet,NULL,0,decode_opt_del,0,0);
+      Network_traverse(refnet,NULL,0,decode_opt_del,0,0);
+    }
+
+    if (fcorr){
+      Network_traverse(hypnet,NULL,0,decode_fragment,0,0);
+      Network_traverse(refnet,NULL,0,decode_fragment,0,0);
+    }
+
+    if (char_align){
+      Network_traverse(hypnet,NULL,0,expand_words_to_chars,&char_align,0);
+      Network_traverse(refnet,NULL,0,expand_words_to_chars,&char_align,0);
+    }
+
+    if (wwl != (WWL *)0){
+      Network_traverse(hypnet,NULL,0,lookup_word_weight,wwl,0);
+      Network_traverse(refnet,NULL,0,lookup_word_weight,wwl,0);
+    }
+
+    if (lm_file != (char *)0){
+      Network_traverse(hypnet,NULL,0,lookup_lm_word_weight,lm_file,0);
+      Network_traverse(refnet,NULL,0,lookup_lm_word_weight,lm_file,0);
+    }
+
+//        Network_traverse(hypnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);
+//        Network_traverse(refnet,NULL,0,print_arc,0,NT_CA_For+NT_Verbose);  
+    Network_dpalign(refnet,hypnet,
+		    ( (time_align) ? wwd_time_WORD : 
+		      ( ((wwl != (WWL *)0) || (lm_file != (char *)0)) ?
+			 wwd_weight_WORD : wwd_WORD) ),
+		    &path,FALSE);
+    PATH_add_utt_id(path,(char *)id);
+    PATH_set_sequence(path);
+    if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+    if (char_align) BF_SET(path->attrib,PA_CHAR_ALIGN);
+    if ((wwl != (WWL *)0) || (lm_file != (char *)0)) {
+      BF_SET(path->attrib,PA_HYP_WEIGHT);
+      BF_SET(path->attrib,PA_REF_WEIGHT);
+    }
+
+    Network_destroy(refnet);
+    Network_destroy(hypnet);
+    
+    return(path);
+}
+
+SCORES *align_trans_mode_dp(char *ref_file, char *hyp_file, char *set_title, int keep_path, int case_sense, int feedback, int char_align, enum id_types idt, int infer_word_seg, char *lexicon, int fcorr, int opt_del, int inf_no_ascii, WWL *wwl, char *lm_file){
+    FILE *fp_hyp;
+    TEXT *hyp_buff, *hyp_id, *spkr_id;
+    int hyp_buff_len=LOAD_BUFF_LEN, hyp_id_len=LOAD_BUFF_LEN;
+    int spkr_id_len=LOAD_BUFF_LEN;
+    PATH *path;
+    SCORES *scor;
+    int spk, rind, last_spk=0;
+    TEXT **reftran, **refid, **ind;
+    int refcnt;
+
+    alloc_singZ(hyp_buff,hyp_buff_len,TEXT,'\0');
+    alloc_singZ(hyp_id,hyp_id_len,TEXT,'\0');
+    alloc_singZ(spkr_id,spkr_id_len,TEXT,'\0');
+
+    if ((hyp_file == NULL) || (*hyp_file == '\0') || 
+        ((fp_hyp = fopen(hyp_file,"r")) == NULL)){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+
+    scor = SCORES_init(set_title,20);
+    scor->ref_fname = (char *)TEXT_strdup((TEXT *)ref_file);
+    scor->hyp_fname = (char *)TEXT_strdup((TEXT *)hyp_file);
+    scor->creation_date = get_date();
+    scor->frag_corr = fcorr;
+    scor->opt_del = opt_del;
+    if (wwl != (WWL*)0) {
+      scor->weight_ali = 1;
+      scor->weight_file =
+	(char *)TEXT_strdup((TEXT*)rsprintf("%s Column \"%s\"",wwl->filename,
+					    wwl->weight_desc[0]));
+    } else if (lm_file != (char *)0) {
+      scor->weight_ali = 1;
+      scor->weight_file = (char *)TEXT_strdup((TEXT*)lm_file);
+    } else
+      scor->weight_ali = 0;
+
+    load_refs(scor, hyp_file, ref_file, &reftran, &refid, &refcnt, case_sense); 
+
+    while (!feof(fp_hyp)){
+        *hyp_buff  = *(hyp_buff+1) = COMMENT_CHAR;
+        *(hyp_buff+2) = NULL_TEXT;
+        while (!feof(fp_hyp) && (parse_input_comment_line(scor,hyp_buff) ||
+                                 TEXT_is_comment_info(hyp_buff) ||
+                                 TEXT_is_empty(hyp_buff)))
+            if (TEXT_ensure_fgets(&hyp_buff,&hyp_buff_len,fp_hyp) == NULL)
+               *hyp_buff = '\0';
+        if (feof(fp_hyp))
+            break;
+
+	remove_id(hyp_buff,&hyp_id,&hyp_id_len);
+	if (!case_sense)  TEXT_str_case_change_with_mem_expand(&hyp_id, &hyp_id_len, 1);
+
+	extract_speaker(hyp_id,spkr_id,idt);
+	spk = SCORES_get_grp(scor,(char *)spkr_id);
+	
+	if (feedback >= 1){
+	    if (spk != last_spk) {  printf("\n"); last_spk = spk; }
+	    printf("\r    Alignment# %d for speaker %s          ",
+		   scor->grp[spk].nsent+1,spkr_id);
+	    fflush(stdout);
+	}
+
+	if ((ind = (TEXT **)bsearch(hyp_id,refid,refcnt,sizeof(TEXT *),
+				    bsearch_TEXT_strcmp)) == NULL){
+	    fprintf(stderr,"Error: Unable to locate Ref transcript for '%s'\n",
+		    hyp_id);
+	    exit(1);
+	}
+	rind = ind - refid;
+
+	if (feedback >= 100){
+            printf("\n        HYP -> %s\n",hyp_buff);
+            printf("        REF -> %s\n",reftran[rind]);
+	    fflush(stdout);
+        }
+	if (infer_word_seg == 0)
+	    path = network_dp_align_texts(reftran[rind], (NETWORK *)0,
+					  hyp_buff, (NETWORK *)0,
+					  char_align, case_sense, 
+					  (char *)hyp_id, fcorr, opt_del, FALSE, wwl, lm_file);
+	else if (infer_word_seg == INF_SEG_ALGO1)
+	    path = infer_word_seg_algo1(reftran[rind], hyp_buff, (NETWORK *)0,
+					case_sense, (char *)hyp_id, lexicon,
+					fcorr, opt_del, inf_no_ascii);
+	else if (infer_word_seg == INF_SEG_ALGO2)
+	    path = infer_word_seg_algo2(reftran[rind], hyp_buff, (NETWORK *)0,
+					case_sense, (char *)hyp_id, lexicon,
+					fcorr, opt_del, inf_no_ascii);
+	else{
+	    fprintf(scfp,"Error: unknown alignment procedure\n");
+	    exit(1);
+	}
+
+	if (feedback >= 2){
+	    printf("\n"); 
+	    PATH_print(path,stdout,199);
+	    printf("\n");
+	}
+
+	add_PATH_score(scor,path,spk, keep_path);
+
+	if (!keep_path)
+	    PATH_free(path);
+    }
+    if (feedback >= 1)
+	printf("\n\n");
+
+    free_singarr(hyp_buff,TEXT);
+    free_singarr(hyp_id,TEXT);
+    free_singarr(spkr_id,TEXT);
+    free_refs(&reftran, &refid, refcnt);
+    fclose(fp_hyp);
+
+    return(scor);
+}
+
+
+/*  This function, called by the network travers function, will set the */
+/*  duration of the word to *(double *)ptr */
+void set_word_duration(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    tw->T_dur = *(double *)ptr;
+}
+
+/*  This function, called by the network traverse function, will set the */
+/*  words start to *(double *)ptr, then set T2 to be T1+dur, then */
+/*  increment *(double *)ptr by dur. */
+void set_word_start(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    tw->T1 = *(double *)ptr;
+    tw->T2 = tw->T_dur + tw->T1;
+    *(double *)ptr += tw->T_dur;
+}
+
+/*  This function, called by the network traverse function, will set the */
+/*  word's confidence score to *(double *)ptr */
+void set_word_conf(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    tw->conf = *(double *)ptr;
+}
+
+/*  This function, called by the network traverse function, will set the */
+/*  word's optionally deletable flag to *(int *)ptr */
+void set_word_opt_del(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    TEXT *ot;
+
+    if (*(int *)ptr){
+      /* re-make the optionally deletable text */
+      ot = tw->value;
+      tw->value = TEXT_strdup((TEXT *)rsprintf(WORD_OPT_DEL_PRE_STR "%s"
+					       WORD_OPT_DEL_POST_STR,ot));
+      TEXT_free(ot);
+    }
+}
+
+void expand_words_to_chars(ARC *arc, void *ptr){
+    char *proc = "expand_words_to_chars";
+    WORD *tw = (WORD *)(arc->data);
+    static TEXT *chars=(TEXT *)0, *charsEsc;
+    static int chars_len=100;
+    NETWORK *subnet;
+    char buf[20];
+    static int alt=1;
+
+    if (chars == (TEXT *)0) alloc_singZ(chars,chars_len,TEXT,'\0');
+    /* it the string is already a single letter, return */
+    if (TEXT_strlen(tw->value) <= 1)
+	return;
+
+    if (db >= 5) {
+	printf("%s: String length > 1\n",proc);
+	arc->net->arc_func.print(arc->data);
+    }
+//    arc->net->arc_func.print(arc->data);
+    sprintf(buf,"expand-net %d",alt);
+    TEXT_separate_chars((!tw->opt_del) ? tw->value : tw->intern_value,
+			&chars,&chars_len,*((int *)ptr));
+			
+//    printf("   from /%s/ comes /%s/\n",(!tw->opt_del) ? tw->value : tw->intern_value, chars);
+    
+    // Escape the semicolon
+    alloc_singZ(charsEsc, chars_len*2+1, TEXT, NULL_TEXT);
+    TEXT_strcpy_escaped(charsEsc, chars, ';');
+//    printf("   and escaped /%s/\n", charsEsc);
+    
+    /* Fix, it wasn't enough check to see if the there were any spaces */
+    if (TEXT_strcmp(chars,tw->value) != 0){
+	subnet = Network_create_from_TEXT(charsEsc, buf, arc->net->arc_func.print,
+					  arc->net->arc_func.equal,
+					  arc->net->arc_func.destroy,
+					  arc->net->arc_func.is_null_alt,
+					  arc->net->arc_func.is_opt_del,
+					  arc->net->arc_func.copy,
+					  arc->net->arc_func.make_empty,
+					  arc->net->arc_func.use_count);
+	if (subnet == NULL_NETWORK){
+	    fprintf(scfp,"proc %s: Internal error.  Unable to expand words"
+		    " to characters\n",proc);
+	    return;
+	}
+	/* traverse the network, setting the character times to equal */
+	/* durations based on the original word */
+	{  
+	  double dur = tw->T_dur/subnet->arc_count, t1 = tw->T1, conf = tw->conf;
+	  int od = tw->opt_del;
+	  Network_traverse(subnet,NULL,0,set_word_duration,&dur,0);
+	  Network_traverse(subnet,NULL,0,set_word_start,&t1,0);
+	  Network_traverse(subnet,NULL,0,set_word_conf,&conf,0);
+	  if (od){
+	    Network_traverse(subnet,NULL,0,set_word_opt_del,&od,0);
+	    Network_traverse(subnet,NULL,0,decode_opt_del,0,0);
+	  }
+	}
+	Network_merge_network(arc->from_node,arc->to_node,subnet);
+	Network_delete_arc(arc);
+	alt++;
+    }
+    free_singarr(charsEsc, TEXT);
+}
+
+
+      
+void lookup_word_weight(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    WWL *wwl = (WWL *)ptr;
+
+    if (tw->value == (TEXT *)0)
+      tw->weight = 0.0;
+    else if (TEXT_strcmp(tw->value,(TEXT *)"@") == 0) 
+      tw->weight = 0.0;
+    else if (tw->opt_del) 
+      tw->weight = 0.0;
+    else
+      tw->weight = Weight_wwl((tw->opt_del ? tw->intern_value : tw->value), wwl);
+
+    return;
+}
+
+
+void decode_opt_del(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    int len;
+
+    if (tw->value == (TEXT *)0) return;
+    len = TEXT_strlen(tw->value);
+
+    if ((*(tw->value + len - 1) == WORD_OPT_DEL_POST_CHAR) &&
+	(*(tw->value) == WORD_OPT_DEL_PRE_CHAR)){
+      tw->opt_del = TRUE;
+      tw->intern_value = TEXT_strBdup(tw->value + 1,len - 2);
+    }
+    return;
+}
+
+void decode_fragment(ARC *arc, void *ptr){
+    WORD *tw = (WORD *)(arc->data);
+    int len;
+
+    if (tw->value == (TEXT *)0) return;
+    len = TEXT_strlen(tw->value);
+    if (((*(tw->value + len - 1) == WORD_FRAG_CHAR) || 
+	 (*(tw->value          ) == WORD_FRAG_CHAR))    ||
+	(tw->opt_del && 
+	 (*(tw->value + len - 2) == WORD_FRAG_CHAR)))
+      tw->frag_corr = TRUE;
+
+    return;
+}
+
+
+/**********************************************************************/
+/************      USE GNU-DIFF to compute the alignments *************/
+/**********************************************************************/
+
+void create_word_lists(SCORES *sc, TEXT **reftran, TEXT **refid, int refcnt, char *hyp_file, char *out_ref, char *out_hyp, int *num_hyp, int case_sense);
+void process_diff_line(TEXT *diff_out, PATH *path, TEXT **firsttoken);
+void set_temp_files(char *refwords, char *hypwords);
+
+void create_word_lists(SCORES *sc, TEXT **reftran, TEXT **refid, int refcnt, char *hyp_file, char *out_ref, char *out_hyp, int *num_hyp, int case_sense){
+    FILE *fp_hyp, *fp_out_ref, *fp_out_hyp;
+    TEXT *hyp_buff, *hyp_id, *ctext, *tbuf;
+    int hyp_buff_len=LOAD_BUFF_LEN, hyp_id_len=LOAD_BUFF_LEN;
+    int tbuf_len=100;
+    int rind;
+    TEXT **ind;
+
+    alloc_singZ(tbuf,tbuf_len,TEXT,'\0');
+    alloc_singZ(hyp_buff,hyp_buff_len,TEXT,'\0');
+    alloc_singZ(hyp_id,hyp_id_len,TEXT,'\0');
+
+    if ((fp_hyp = fopen(hyp_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+
+    if ((fp_out_hyp=fopen(out_hyp,"w")) == NULL){
+	fprintf(stderr,"Error: Unable to open Hypothesis word file '%s'\n",
+		out_hyp);
+	exit(1);
+    }
+    if ((fp_out_ref=fopen(out_ref,"w")) == NULL){
+	fprintf(stderr,"Error: Unable to open Reference word file '%s'\n",
+		out_ref);
+	exit(1);
+    }
+
+    while (!feof(fp_hyp)){
+        *hyp_buff  = *(hyp_buff+1) = COMMENT_CHAR;
+        *(hyp_buff+2) = NULL_TEXT;
+ 
+        while (!feof(fp_hyp) && (parse_input_comment_line(sc,hyp_buff) ||
+                                 TEXT_is_comment_info(hyp_buff) ||
+                                 TEXT_is_empty(hyp_buff)))
+            if (TEXT_ensure_fgets(&hyp_buff,&hyp_buff_len,fp_hyp) == NULL)
+               *hyp_buff = '\0';
+
+        if (feof(fp_hyp))
+            break;
+
+	(*num_hyp) ++;
+
+	if (!case_sense)
+	     TEXT_str_case_change_with_mem_expand(&hyp_buff, &hyp_buff_len, 1);
+
+	remove_id(hyp_buff,&hyp_id,&hyp_id_len);
+	if ((ind = (TEXT **)bsearch(hyp_id,refid,refcnt,sizeof(TEXT *),
+				    bsearch_TEXT_strcmp)) == NULL){
+	    fprintf(stderr,"Error: Unable to locate Ref transcript for '%s'\n",
+		    hyp_id);
+	    exit(1);
+	}
+	rind = ind - refid;
+
+	/* Write each individual word to a file */
+	if (TEXT_strlen(reftran[rind]) > tbuf_len+1){
+	    free_singarr(tbuf,TEXT);
+	    tbuf = TEXT_strdup(reftran[rind]);
+	    tbuf_len = TEXT_strlen(tbuf);
+	} else
+	    TEXT_strcpy(tbuf,reftran[rind]);
+
+	ctext = tokenize_TEXT_first_alt(tbuf,(TEXT *)" \t\n");
+	while (ctext != NULL) {
+	    fprintf(fp_out_ref,"%s\n",ctext);
+	    ctext = tokenize_TEXT_first_alt(NULL,(TEXT *)" \t\n");
+	}
+	
+	ctext = TEXT_strtok(hyp_buff,(TEXT *)" \t\n");
+	while (ctext != NULL) {
+	    fprintf(fp_out_hyp,"%s\n",ctext);
+	    ctext = TEXT_strtok(NULL,(TEXT *)" \t\n");
+	}
+    }
+
+    fclose(fp_out_ref);
+    fclose(fp_out_hyp);
+    fclose(fp_hyp);
+    free_singarr(hyp_buff,TEXT);
+    free_singarr(hyp_id,TEXT);
+    free_singarr(tbuf,TEXT);
+}
+
+void process_diff_line(TEXT *diff_out, PATH *path, TEXT **firsttoken){
+    static TEXT s1[100],s2[100],s3[100];
+    int iret;
+    WORD *rwd, *hwd;
+
+    check_space_in_PATH(path);
+    *s1 = *s2 = *s3 = '\0';
+    iret = sscanf((char *)diff_out,
+		  "%s %s %s",(char *)s1,(char *)s2,(char *)s3);
+    *firsttoken = s1;
+
+    if (iret == 2 && TEXT_strcmp(s1,s2) == 0)
+	path->pset[path->num].eval = P_CORR;
+    else if (iret == 2 && TEXT_strcmp(s1,(TEXT *)">") == 0)
+	path->pset[path->num].eval = P_INS;
+    else if (iret == 3)
+	path->pset[path->num].eval = P_SUB;
+    else
+	path->pset[path->num].eval = P_DEL;
+
+    path->pset[path->num].b_ptr = path->pset[path->num].a_ptr = (void *)0;
+
+    switch (path->pset[path->num].eval){
+      case P_CORR:
+      case P_DEL:
+      case P_SUB:
+	rwd = new_WORD(s1, -1, 0.0, 0.0, 0.0, (TEXT *)0, (TEXT *)0, 0, 0, -1.0);
+	path->pset[path->num].a_ptr = rwd;
+      case P_INS:
+	;
+    }
+    switch (path->pset[path->num].eval){
+      case P_CORR:
+      case P_INS:
+      case P_SUB:
+	hwd = (WORD *)0;
+	switch (path->pset[path->num].eval){
+	  case P_CORR:  
+	  case P_INS:  
+	    hwd = new_WORD(s2, -1, 0.0, 0.0, 0.0,(TEXT *)0, (TEXT *)0, 0, 0, -1.0); break;
+	  case P_SUB:
+	    hwd = new_WORD(s3, -1, 0.0, 0.0, 0.0,(TEXT *)0, (TEXT *)0, 0, 0, -1.0); break;
+	  case P_DEL:
+	    hwd = new_WORD((TEXT *)0, -1, 0.0, 0.0, 0.0,(TEXT *)0, (TEXT *)0, 0, 0, -1.0);
+	    break;
+	}
+	if (hwd == (WORD *)0){
+	  fprintf(stderr,"Error: Failed to alloc word in process_diff_line\n");
+	  exit(1);
+	}
+	path->pset[path->num].b_ptr = hwd;		    
+      case P_DEL:
+	;
+    }
+
+    path->num++;
+}
+
+void set_temp_files(char *refwords, char *hypwords){
+    int pid = getpid();
+    int max=30, n=0, fail;
+    struct stat fileinfo;
+
+    do {
+	sprintf(refwords,"/tmp/ref.%d",pid+n);
+	sprintf(hypwords,"/tmp/hyp.%d",pid+n);
+	n++;
+	fail = (stat(refwords,&fileinfo) == 0) ||
+	       (stat(hypwords,&fileinfo) == 0);
+    } while (fail && (n < max));
+    if (fail){
+	fprintf(stderr,"Can't locate temp file names. Searched /tmp/ref.[%d-%d]\n",
+		pid,pid+n);
+	exit(1);
+    }
+}
+
+SCORES *align_trans_mode_diff(char *ref_file, char *hyp_file, char *set_title, int keep_path, int case_sense, int feedback, enum id_types idt){
+    FILE *fp_hyp, *fp_diff;
+    TEXT *hyp_buff, *hyp_id, *spkr_id, *diff_out;
+    int hyp_buff_len=LOAD_BUFF_LEN, hyp_id_len=LOAD_BUFF_LEN;
+    int spkr_id_len=LOAD_BUFF_LEN, diff_out_len=LOAD_BUFF_LEN;
+    PATH *path;
+    SCORES *scor;
+    int spk, rind;
+    TEXT **reftran, **refid, **ind;
+    TEXT *firsttoken;
+    int refcnt, num_hyp=0;
+    TEXT *ctext = hyp_buff;               /* current position is text string */
+    char refwords[100], hypwords[100];
+
+    alloc_singZ(diff_out,diff_out_len,TEXT,'\0');
+    alloc_singZ(hyp_buff,hyp_buff_len,TEXT,'\0');
+    alloc_singZ(hyp_id,hyp_id_len,TEXT,'\0');
+    alloc_singZ(spkr_id,spkr_id_len,TEXT,'\0');
+
+    set_temp_files(refwords, hypwords);
+
+    /* initialize the score structure */
+    scor = SCORES_init(set_title,20);
+    scor->ref_fname = (char *)TEXT_strdup((TEXT *)ref_file);
+    scor->hyp_fname = (char *)TEXT_strdup((TEXT *)hyp_file);
+    scor->creation_date = get_date();
+
+    /* load the required reference sentences */
+    if (feedback >= 1)	printf("    Loading transcripts\n");
+    load_refs(scor, hyp_file, ref_file, &reftran, &refid, &refcnt, case_sense); 
+
+    /* create a file of words for both the ref and hyp */
+    create_word_lists(scor, reftran, refid, refcnt, hyp_file,
+		      refwords, hypwords, &num_hyp, case_sense);
+
+    /* use readpipe to open a process without creating a temporary file */
+    if (feedback >= 1)	printf("    Executing diff\n");
+    if ((fp_diff = readpipe (DIFF_PROGRAM, "-y",refwords,
+			     hypwords, NULL)) ==NULL){
+	fprintf(stderr,"Error: Execution of 'diff' failed\n");
+	exit(1);
+    }
+    
+    if ((fp_hyp = fopen(hyp_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+    while (!feof(fp_hyp)){
+        *hyp_buff  = *(hyp_buff+1) = COMMENT_CHAR;
+        *(hyp_buff+2) = NULL_TEXT;
+ 
+        while (!feof(fp_hyp) && (parse_input_comment_line(scor,hyp_buff) ||
+                                 TEXT_is_comment_info(hyp_buff) ||
+                                 TEXT_is_empty(hyp_buff)))
+            if (TEXT_ensure_fgets(&hyp_buff,&hyp_buff_len,fp_hyp) == NULL)
+               *hyp_buff = '\0';
+
+        if (feof(fp_hyp))  break;
+	if (!case_sense)    TEXT_str_case_change_with_mem_expand(&hyp_buff, &hyp_buff_len, 1);
+	remove_id(hyp_buff,&hyp_id,&hyp_id_len);
+	extract_speaker(hyp_id,spkr_id,idt);
+	spk = SCORES_get_grp(scor,(char *)spkr_id);
+
+	if ((ind = (TEXT **)bsearch(hyp_id,refid,refcnt,sizeof(TEXT *),
+				    bsearch_TEXT_strcmp)) == NULL){
+	    fprintf(stderr,"Error: Unable to locate Ref transcript for '%s'\n",
+		    hyp_id);
+	    exit(1);
+	}
+	rind = ind - refid;
+	--num_hyp;
+
+	if (feedback >= 3)
+	    printf("REF: %s\nHYP: %s\n",reftran[rind],hyp_buff);
+
+	path = PATH_alloc(100);
+
+	ctext = tokenize_TEXT_first_alt(reftran[rind],(TEXT *)" \t\n");
+	while (ctext != NULL) {
+	    do {
+		check_space_in_PATH(path);
+		if (TEXT_ensure_fgets(&diff_out,&diff_out_len,fp_diff) == 
+			NULL)
+		    *diff_out = '\0';
+		if (feof(fp_diff)) break;
+		process_diff_line(diff_out,path,&firsttoken);
+	    } while (!feof(fp_diff) && 
+		     ((TEXT_strcmp(firsttoken,ctext) != 0) || num_hyp==0));
+	    ctext = tokenize_TEXT_first_alt(NULL,(TEXT *)" \t\n");
+	}
+
+	PATH_add_utt_id(path,(char *)hyp_id);
+	PATH_set_sequence(path);
+	if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+	
+	if (feedback >= 2){
+	    printf("\n");
+	    PATH_print(path,stdout,199);
+	    printf("\n");
+	}
+
+	add_PATH_score(scor,path,spk, keep_path);
+
+	if (!keep_path)  PATH_free(path);
+    }
+    fclose(fp_diff);
+    fclose(fp_hyp);
+
+    if (num_hyp != 0){
+	fprintf(stderr,"Error: Unexpected number of output utts.  Missing %d\n",num_hyp);
+	exit(1);
+    }
+
+    free_singarr(diff_out,TEXT);
+    free_singarr(hyp_buff,TEXT);
+    free_singarr(hyp_id,TEXT);
+    free_singarr(spkr_id,TEXT);
+    free_refs(&reftran, &refid, refcnt);
+
+    unlink(refwords);
+    unlink(hypwords);
+    return(scor);
+}
+
+SCORES *align_ctm_to_stm_diff(char *ref_file, char *hyp_file, char *set_title, int keep_path, int case_sense, int feedback, enum id_types idt){
+    char *proc="align_ctm_to_stm";
+    FILE *fp_ref, *fp_hyp;
+    WTOKE_STR1 *hyp_segs;
+    int ref_eof, ref_err, hyp_eof, hyp_err;
+    int hyp_end_chan1, ref_end_chan1;
+    int hyp_end_chan2, ref_end_chan2;
+    int just_read, hyp_file_end, ref_file_end;
+    int number_of_channels;
+    int i;
+    PATH *path;
+    STM *stm;
+    SCORES *scor;
+    int spkr, r;
+    char refwords[100], hypwords[100];
+    TEXT *diff_out, *firsttoken, *ctext;
+    static int diff_out_len=100;
+    FILE *fp_diff;
+    int pdb = 0;
+
+    /* allocate memory */
+    hyp_segs = WTOKE_STR1_init(hyp_file);
+    scor = SCORES_init(set_title,20);
+    scor->ref_fname = (char *)TEXT_strdup((TEXT *)ref_file);
+    scor->hyp_fname = (char *)TEXT_strdup((TEXT *)hyp_file);
+    scor->creation_date = get_date();
+
+    alloc_singZ(diff_out,diff_out_len,TEXT,'\0');
+    
+    set_temp_files(refwords, hypwords);
+
+    if ((fp_hyp = fopen(hyp_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+    if ((fp_ref = fopen(ref_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input reference file %s\n",ref_file);
+        exit(1);
+    }
+
+    if ((stm = alloc_STM(200)) == (STM *)0) {
+	fprintf(stderr,"%s: Failed to alloc STM struct\n",proc);
+	exit(1);
+    }
+    hyp_eof = ref_eof = 0;
+    do {
+        do {
+	    locate_WTOKE_boundary(hyp_segs, hyp_segs->s, 1, 0, &hyp_file_end);
+            just_read = 0;
+            if ((hyp_file_end == hyp_segs->n) && !hyp_eof){
+                fill_mark_struct(fp_hyp,hyp_segs,hyp_file,&hyp_eof,&hyp_err, 
+				 case_sense);
+                if (hyp_err != 0){
+                    fprintf(stdout,"; *Err: Error detected in hyp file '%s'\n",
+                            hyp_file);
+                    exit(1);
+                }
+                just_read = 1;
+            }
+        } while (just_read);
+	just_read = 0;
+        do {
+	    locate_STM_boundary(stm, stm->s, 1, 0, &ref_file_end);
+            if ((ref_file_end == stm->num) && !ref_eof){
+		if (just_read == 1){
+		    just_read = 0;
+		    expand_STM(stm);
+		}
+		fill_STM(fp_ref, stm, ref_file, &ref_eof, case_sense,&ref_err);
+                if (hyp_err != 0){
+                    fprintf(stdout,"; *Err: Error detected in hyp file '%s'\n",
+                            hyp_file);
+                    exit(1);
+                }
+                just_read = 1;
+            } else 
+		just_read = 0;
+        } while (just_read);
+
+	/* find the end of the second channel for the hyp file */
+	locate_WTOKE_boundary(hyp_segs, hyp_segs->s, 1, 1, &hyp_end_chan1);
+	if (hyp_end_chan1+1 < hyp_segs->n &&
+            strcmp(hyp_segs->word[hyp_segs->s].conv,
+		   hyp_segs->word[hyp_end_chan1+1].conv)==0){
+            locate_WTOKE_boundary(hyp_segs,hyp_end_chan1+1,1,1,&hyp_end_chan2);
+            number_of_channels = 2;
+        } else {
+            /* There was no second channel */
+            hyp_end_chan2 = hyp_end_chan1;
+            number_of_channels = 1;
+        }
+
+	/* find the end of the second channel for the ref file */
+	locate_STM_boundary(stm, stm->s, 1, 1, &ref_end_chan1);
+	if (ref_end_chan1+1 < stm->num &&
+            TEXT_strcmp(stm->seg[stm->s].file,
+			stm->seg[ref_end_chan1+1].file)==0){
+	    if (number_of_channels != 2){
+		fprintf(stderr,"%s: Channel Mis-match, 1 in Hyp, 2 in Ref\n",
+			proc);
+		return((SCORES *)0);
+	    }
+	    locate_STM_boundary(stm, ref_end_chan1+1, 1, 1, &ref_end_chan2);
+        } else {
+            /* There was no second channel */
+	    if (number_of_channels != 1){
+		fprintf(stderr,"%s: Channel Mis-match, 2 in Hyp, 1 in Ref\n",
+			proc);
+		return((SCORES *)0);
+	    }
+            ref_end_chan2 = ref_end_chan1;
+        }
+
+	/* give some user feedback */
+	if (feedback >= 1)
+	    printf("    Performing alignments for file '%s'.\n",
+		   stm->seg[stm->s].file);
+	if (db+pdb >= 5){
+	    printf("%s: hyp CTM File Range [%d,%d]",
+		   proc,hyp_segs->s,hyp_file_end);
+	    printf("   Chan1 Range[%d,%d] ",hyp_segs->s,hyp_end_chan1);
+	    printf("  Chan2 Range[%d,%d]\n",hyp_end_chan1+1,hyp_end_chan2);
+	    if (db+pdb >= 10){
+		dump_word_tokens2(hyp_segs, hyp_segs->s, hyp_file_end);
+		printf("\n");
+	    }
+	    printf("%s: ref STM File Range [%d,%d]",proc,stm->s,ref_file_end);
+	    printf("   Chan1 Range[%d,%d] ",stm->s, ref_end_chan1);
+	    printf("  Chan2 Range[%d,%d]\n",ref_end_chan1+1,ref_end_chan2);
+	    if (db+pdb >= 10) dump_STM(stm, stm->s, ref_file_end);
+	    printf("\n");
+	}
+	/* Align each channel using diff */
+	for (i=0; i<number_of_channels; i++){
+	    int h_st, h_end, r_st, r_end;
+
+	    if (db+pdb >= 5) printf("\n%s: Starting Channel %d\n",proc,i);
+	    if (i==0){
+		h_st = hyp_segs->s;       h_end = hyp_end_chan1;
+		r_st = stm->s;            r_end = ref_end_chan1;
+	    } else {
+		h_st = hyp_end_chan1+1;   h_end = hyp_end_chan2;
+		r_st = ref_end_chan1+1;   r_end = ref_end_chan2;
+	    }
+
+	    if (strcmp(hyp_segs->word[h_st].conv,
+		       (char *)stm->seg[r_st].file) != 0){
+		fprintf(stderr,"%s: File identifiers do not match,\nhyp ",proc);
+		fprintf(stderr,"file '%s' and ref file '%s' not synchronized\n",
+			hyp_segs->word[h_st].conv,stm->seg[r_st].file);
+		return((SCORES *)0);
+	    }
+	    if (strcmp(hyp_segs->word[h_st].turn,
+		       (char *)stm->seg[r_st].chan) != 0){
+		fprintf(stderr,"%s: channel identifiers do not match, ",proc);
+		fprintf(stderr,"hyp file '%s' and ref file '%s' not",
+			hyp_segs->word[h_st].conv,stm->seg[r_st].file);
+		fprintf(stderr," synchronized\n");
+		return((SCORES *)0);
+	    }
+
+	    /* write out the words in the STM file */
+	    dump_STM_words(stm,r_st,r_end+((r_end==stm->num)?-1:0),refwords);
+	    dump_WTOKE_words(hyp_segs, h_st, h_end, hypwords);
+
+	    if ((fp_diff = readpipe (DIFF_PROGRAM, "-y",refwords, hypwords, NULL)) == NULL){
+		fprintf(stderr,"Error: Execution of 'diff' failed\n");
+		exit(1);
+	    }
+
+	    for (r=r_st; r<=r_end + ((r_end==stm->num)?-1:0); r++){
+		path = PATH_alloc(100);
+		path->num = 0;
+
+		/* for each reference word, located it's matching diff output*/
+		ctext = tokenize_TEXT_first_alt(stm->seg[r].text,
+						(TEXT*)" \t\n");
+		while (ctext != NULL){
+ 		    do {
+			if (TEXT_ensure_fgets(&diff_out,&diff_out_len,fp_diff)
+			    == NULL)
+			    *diff_out = '\0';
+			if (feof(fp_diff)) break;
+			process_diff_line(diff_out,path,&firsttoken);
+		    } while (!feof(fp_diff) && 
+			     ((TEXT_strcmp(firsttoken,ctext) != 0) ||
+			      r== r_end + (r_end==stm->num)?-1:0));
+		    ctext = tokenize_TEXT_first_alt(NULL,(TEXT *)" \t\n");
+		}
+		spkr = SCORES_get_grp(scor,(char *)stm->seg[r].spkr);
+		PATH_add_utt_id(path,rsprintf("(%s-%03d)",stm->seg[r].spkr,
+					      scor->grp[spkr].nsent));
+		PATH_set_sequence(path);
+		PATH_add_label(path,(char *)stm->seg[r].labels);
+		PATH_add_file(path,(char *)stm->seg[r].chan);
+		PATH_add_channel(path,(char *)stm->seg[r].chan);
+		if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+		add_PATH_score(scor,path,spkr, keep_path);
+		
+		if (feedback >= 2){
+		    printf("\n");
+		    PATH_print(path,stdout,199);
+		    printf("\n");
+		}
+		if (!keep_path)  PATH_free(path);
+	    }
+	    fclose(fp_diff);
+
+    
+	    stm->s = r_end+1;
+	} /* for each channel */
+
+	/* increment the start pointers */
+	hyp_segs->s = hyp_end_chan2+1;
+	stm->s = ref_end_chan2+1;
+    } while (hyp_segs->s <= hyp_segs->n);
+
+    fclose(fp_hyp);
+    fclose(fp_ref);
+
+    unlink(refwords);
+    unlink(hypwords);
+    free_singarr(diff_out,TEXT);
+
+    free_mark_file(hyp_segs);
+    free_STM(stm);
+
+    return(scor);
+}
+
+void convert_text_to_word_list(char *file, char *words,int case_sense){
+    FILE *fp, *fp_out;
+    TEXT *buf, *ctext;
+    int buf_len=100;
+
+    if ((fp=fopen(file,"r")) == NULL){
+	fprintf(stderr,"Error: Unable to open input STM file '%s'\n",file);
+	exit(1);
+    }
+
+    if ((fp_out=fopen(words,"w")) == NULL){
+	fprintf(stderr,"Error: Unable to open STM word file '%s'\n",words);
+	exit(1);
+    }
+
+    alloc_singZ(buf,buf_len,TEXT,'\0');
+
+    /* for each stm, load the aligned text and create a path as before */
+    while (!feof(fp)){
+	/* read the next ref line */
+	*buf = '\0';
+        while (!feof(fp) && (TEXT_is_comment(buf) ||
+			     TEXT_is_comment_info(buf) ||
+			     TEXT_is_empty(buf)))
+	    if (TEXT_ensure_fgets(&buf,&buf_len,fp) == NULL)
+		*buf = '\0';
+	if (*buf == '\0' && feof(fp)) break;
+	if (!case_sense)  TEXT_str_case_change_with_mem_expand(&buf, &buf_len, 1);
+
+	/* for each reference word, located it's matching diff output */
+	ctext = TEXT_strtok(buf,(TEXT *)" \t\n");
+	while (ctext != NULL){
+	    fprintf(fp_out,"%s\n",ctext);
+	    ctext = TEXT_strtok(NULL,(TEXT *)" \t\n");
+	}
+    }
+    fclose(fp);
+    fclose(fp_out);
+    free_singarr(buf,TEXT);
+}
+
+SCORES *align_text_to_stm(char *ref_file, char *hyp_file, char *set_title, int keep_path, int case_sense, int feedback, enum id_types idt){
+    /* char *proc="align_text_to_stm"; */
+    FILE *fp_ref;
+    PATH *path;
+    SCORES *scor;
+    int spkr, num_ref;
+    char refwords[100], hypwords[100];
+    TEXT *diff_out, *ref_buf, *firsttoken, *ctext;
+    int diff_out_len=100, ref_buf_len=100;
+    FILE *fp_diff;
+    STM_SEG ref_seg;
+
+    /* allocate memory */
+    scor = SCORES_init(set_title,20);
+    scor->ref_fname = (char *)TEXT_strdup((TEXT *)ref_file);
+    scor->hyp_fname = (char *)TEXT_strdup((TEXT *)hyp_file);
+    scor->creation_date = get_date();
+
+    alloc_singZ(diff_out,diff_out_len,TEXT,'\0');
+    alloc_singZ(ref_buf,ref_buf_len,TEXT,'\0');
+
+    set_temp_files(refwords, hypwords);
+
+    /* stm to word texts */
+    convert_stm_to_word_list(ref_file,refwords,case_sense,&num_ref);
+
+    /* text to word separeted texts */
+    convert_text_to_word_list(hyp_file,hypwords,case_sense);
+
+    /* start up diff */
+    if ((fp_diff = readpipe (DIFF_PROGRAM,"-y",refwords, hypwords, NULL)) ==
+	NULL){
+	fprintf(stderr,"Error: Execution of 'diff' failed\n");
+	exit(1);
+    }
+
+    /* again, open the input reference file */
+    if ((fp_ref = fopen(ref_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input reference file %s\n",ref_file);
+        exit(1);
+    }
+
+    /* for each stm, load the aligned text and create a path as before */
+    while (!feof(fp_ref)){
+	/* read the next ref line */
+	read_stm_line(&ref_buf,&ref_buf_len,fp_ref);
+	if (feof(fp_ref)) break;
+
+	/* parse the reference transcript */
+	parse_stm_line(&ref_seg, &ref_buf, &ref_buf_len, case_sense, 0);
+	num_ref --;
+	
+	/* allocate a path structure */
+	path = PATH_alloc(100);	path->num = 0;
+
+	/* for each reference word, located it's matching diff output */
+	ctext = tokenize_TEXT_first_alt(ref_seg.text,(TEXT *)" \t\n");
+	while (ctext != NULL){
+	    do {
+		if (TEXT_ensure_fgets(&diff_out,&diff_out_len,fp_diff) == NULL)
+		    *diff_out = '\0';
+		if (feof(fp_diff)) break;
+		process_diff_line(diff_out,path,&firsttoken);
+	    } while (!feof(fp_diff) && 
+		     ((TEXT_strcmp(firsttoken,ctext) != 0) || num_ref==0));
+	    ctext = tokenize_TEXT_first_alt(NULL,(TEXT *)" \t\n");
+	}
+
+	spkr = SCORES_get_grp(scor,(char *)ref_seg.spkr);
+	PATH_add_utt_id(path,rsprintf("(%s-%03d)",ref_seg.spkr,
+				      scor->grp[spkr].num_path));
+	PATH_set_sequence(path);
+	PATH_add_label(path,(char *)ref_seg.labels);
+	PATH_add_file(path,(char *)ref_seg.file);
+	PATH_add_channel(path,(char *)ref_seg.chan);
+	if (case_sense) BF_SET(path->attrib,PA_CASE_SENSE);
+	add_PATH_score(scor,path,spkr, keep_path);
+	
+	if (feedback >= 2){
+	    printf("\n");
+	    PATH_print(path,stdout,199);
+	    printf("\n");
+	}
+	if (!keep_path)  PATH_free(path);
+	free_STM_SEG(&ref_seg);
+    }
+    fclose(fp_diff);
+    fclose(fp_ref);
+
+    free_singarr(diff_out,TEXT);
+    free_singarr(ref_buf,TEXT);
+
+    unlink(refwords);
+    unlink(hypwords);
+
+    return(scor);
+}

SCTK/src/sclite/anovar.c

@@ -0,0 +1,485 @@
+/**********************************************************************/
+/*                                                                    */
+/*           FILE: anovar.c                                           */
+/*           WRITTEN BY: Jonathan G. Fiscus                           */
+/*           DATE: April 14 1989                                      */
+/*                  NATIONAL INSTITUTE OF STANDARDS AND TECHNOLOGY    */
+/*                  SPEECH RECOGNITION GROUP                          */
+/*                                                                    */
+/*           USAGE: This uses the rank structure to perform           */
+/*                  a Friedman two-way analysis of variance           */
+/*                  by ranks and generates a report.                  */
+/*                                                                    */
+/*           SOURCE:Applied Non Parametric Statistics by Daniel       */
+/*                                                                    */
+/**********************************************************************/
+
+#include "sctk.h"
+
+#define FRI_TITLE	"Friedman Two-way Analysis of Variance by Ranks"
+#define RANK_TITLEA	"COMPARISON MATRIX: Comparing All Systems"
+#define RANK_TITLEB	"COMPARISON MATRIX: Comparing All Speakers"
+#define RANK_TITLE_1	"Using a Multiple Comparison Test"
+#define ADJUST_THRESH	0.50000
+#define MULTI_COMP_ALPHA	0.05
+
+
+static int analyze_rank_sum(RANK *, int, int, double *, int, int, FILE *);
+static void print_result_of_analyze_rank_sum(int, double, double, char *,FILE *);
+static double dsum_sqr(double *, int);
+static double calc_adjust_for_trt(RANK *);
+static double calc_adjust_for_blks(RANK *);
+static void do_multi_comp_for_anovar(int, int, char **, double *, double *, char *, int *, int **, int, FILE *);
+static void calc_stat_ranks(int, int, int **, double*);
+static double calc_comp_value(int, int);
+static void print_ANOVAR_comp_matrix(int, int **, char **, char *, int *, FILE *);
+
+/****************************************************************/
+/*   main procedure to perform a two-way analysis of variance by*/
+/*   ranks.  first the sum of the treatment and block ranks     */
+/*   are summed then if there is a difference between at least  */
+/*   one of the blocks or treatments, then a multiple comparison*/
+/*   is performed                                               */
+/****************************************************************/
+void compute_anovar(RANK *rank, int verbose, int report, int ***out_sys_winner, char *outroot, int feedback, double ***out_conf)
+{
+    double *sum_trt_rank, *sum_blk_rank;
+    char pad[FULL_SCREEN];
+    int i,j, **trt_winner, **blk_winner;
+    double **conf;
+    FILE *fp = stdout;
+    
+    if (report || verbose){
+	char *f = rsprintf("%s.anovar",outroot);
+	if ((fp=(strcmp(outroot,"-") == 0) ? stdout : fopen(f,"w")) ==
+	    (FILE *)0){
+	    fprintf(stderr,"Warning: Open of %s for write failed.  "
+		           "Using stdout instead.\n",f);
+	    fp = stdout;
+	} else
+	    if (feedback >= 1) printf("        Output written to '%s'\n",f);
+    }
+
+    alloc_2dimZ(conf,rank->n_trt,rank->n_trt,double,0.0);
+    alloc_2dimZ(trt_winner,rank->n_trt,rank->n_trt,int,NO_DIFF);
+    alloc_2dimZ(blk_winner,rank->n_blk,rank->n_blk,int,NO_DIFF);
+    *out_sys_winner = trt_winner;
+    *out_conf = conf;
+
+    alloc_singZ(sum_trt_rank,rnk_trt(rank),double,0.0);
+    alloc_singZ(sum_blk_rank,rnk_blks(rank),double,0.0);
+
+    /*  sum the ranks */
+    for (i=0;i<rnk_blks(rank);i++)
+        for (j=0;j<rnk_trt(rank);j++){
+            sum_trt_rank[j] += Vrnk_t_rank(rank,i,j);
+            sum_blk_rank[i] += Vrnk_b_rank(rank,i,j);
+	}
+
+    if (verbose){
+        set_pad(pad,FRI_TITLE, FULL_SCREEN);
+        fprintf(fp,"\n\n\n%s%s\n",pad,FRI_TITLE);
+        fprintf(fp,"%s----------------------------------------------\n\n",pad);
+        set_pad_cent_n(pad,SCREEN_WIDTH, FULL_SCREEN);
+    fprintf(fp,"%s\tHo: Testing the hypothesis that all recognizers are the same",
+                 pad);
+    }
+    /* if at least one difference, do multiple comparison */
+    if (analyze_rank_sum(rank,rnk_trt(rank),rnk_blks(rank),
+			 sum_trt_rank,
+                         FOR_TREATMENTS,verbose,fp) == TEST_DIFF)
+	clear_2dim(trt_winner,rank->n_trt,rank->n_trt,NO_DIFF);
+        do_multi_comp_for_anovar(rnk_trt(rank),rnk_blks(rank),
+                                 rnk_t_name(rank),sum_trt_rank,
+                                 ovr_t_rank(rank),RANK_TITLEA,
+                                 srt_t_rank(rank),trt_winner,verbose,fp);
+    if (verbose) form_feed(fp);
+
+    if (verbose){
+        set_pad(pad,FRI_TITLE, FULL_SCREEN);
+        fprintf(fp,"\n\n\n%s%s\n",pad,FRI_TITLE);
+        fprintf(fp,"%s----------------------------------------------\n\n",pad);
+        set_pad_cent_n(pad,SCREEN_WIDTH, FULL_SCREEN);
+       fprintf(fp,"%s\tHo: Testing the hypothesis that all speakers are the same",
+                 pad);
+    }
+    /* if at least one difference, do multiple comparison */
+    if (analyze_rank_sum(rank,rnk_blks(rank),rnk_trt(rank),sum_blk_rank,
+                         FOR_BLOCKS,verbose,fp) == TEST_DIFF)
+	clear_2dim(blk_winner,rank->n_blk,rank->n_blk,NO_DIFF);
+        do_multi_comp_for_anovar(rnk_blks(rank),rnk_trt(rank),
+                                 rnk_b_name(rank),sum_blk_rank,
+                                 ovr_b_rank(rank),RANK_TITLEB,
+                                 srt_b_rank(rank),blk_winner, verbose,fp);
+    if (verbose && (fp == stdout)) form_feed(fp);
+    free_2dimarr(blk_winner,rank->n_blk,int); 
+    free_singarr(sum_trt_rank,double);
+    free_singarr(sum_blk_rank,double);
+    if (fp != stdout) fclose(fp);
+}
+
+/*****************************************************************/
+/*        X2_r formula:                                          */
+/*                       k                                       */
+/*             12        --.  2                                  */
+/*          -------  *   \   R   - 3b(t+1)                       */
+/*         bt(t+1)       /    j                                  */
+/*                       --'                                     */
+/*                      j = 1                                    */
+/*                                                               */
+/*   b = num_blk                                                 */
+/*   t = num_trt                                                 */
+/*   R = Sum of the ranks for that column                        */
+/*                                                               */
+/*  this is the initial test to see if the treatment or ranks    */
+/*  are different                                                */
+/*****************************************************************/
+static int analyze_rank_sum(RANK *rank, int trt_num, int blk_num, double *sum_arr, int orient, int verbose,FILE *fp)
+{
+    int df;
+    double X2_r, adjust;
+    char *subject_blk = "speaker";
+    char *subject_trt = "recognition system";
+  
+    /* calculate the test  statistic */
+    df = trt_num - 2;     /* #trt is one based so subtract 2 */
+    /* make sure df isn't negative */;
+    if (df < 0) df = 0;
+    /* make sure df isn't greater than MAX_DF */
+    if (df > MAX_DF) df = MAX_DF;
+
+    X2_r = (12.00000 /
+                (double)(blk_num*trt_num*(trt_num+1))) *
+           (dsum_sqr(sum_arr,trt_num)) -
+           3.00000 * (double)blk_num * (double)(trt_num+1);
+    if (orient == FOR_TREATMENTS)
+        adjust = calc_adjust_for_trt(rank);
+    else
+        adjust = calc_adjust_for_blks(rank);
+
+    /* if two identical systems are entered, adjust will be 0.0 and core*/
+    /* dump.  this avoids this */
+    if (adjust == 0.0)
+       X2_r = 0.0;
+    else
+       X2_r /= adjust;
+
+    if (verbose) {
+        if (orient == FOR_TREATMENTS)
+            print_result_of_analyze_rank_sum(df, X2_r, adjust, subject_trt,fp);
+	else
+            print_result_of_analyze_rank_sum(df, X2_r, adjust, subject_blk,fp);
+    }
+    /* return the result */
+    if (X2_r > X2.df[df].level[GEN_X2_PER])
+        return(TEST_DIFF);
+    else
+        return(NO_DIFF);
+}
+
+/*****************************************************************/
+/*   print to stdout the results of the rank_sum test            */
+/*****************************************************************/
+static void print_result_of_analyze_rank_sum(int df, double X2_r, double adjust, char *subject, FILE *fp)
+{
+    char pad[FULL_SCREEN];
+    int i;
+
+    set_pad_cent_n(pad,SCREEN_WIDTH, FULL_SCREEN);
+    fprintf(fp,"\n\n\n%s%35sReject if\n",pad,"");
+    fprintf(fp,"%s%26sX2_r > X2 of %s %s (%2.3f)\n", pad, "", 
+                          X2_pct_str(GEN_X2_PER),
+                          X2_df_str(df),
+                          X2_score(df,GEN_X2_PER));
+    fprintf(fp,"\n"); 
+    fprintf(fp,"%s%26sadjustment = %2.3f\n",pad,"",adjust);
+    fprintf(fp,"%s%30s  X2_r = %2.3f\n",pad,"",X2_r);
+    if (adjust < ADJUST_THRESH){
+        fprintf(fp,"\n\n%s\t\t*** Warning:  ties adjustment may have severely\n",
+                                                                   pad);
+        fprintf(fp,"%s\t\t              exagerated the X2_r value\n\n",pad);
+    }
+    fprintf(fp,"%sANALYSIS:\n%s--------\n",pad,pad);
+    if (X2_r > X2_score(df,GEN_X2_PER)){
+     fprintf(fp,"%s\tThe test statistic X2_r shows, with %s confidence, that at\n"
+                                            ,pad,X2.neg_per_str[GEN_X2_PER]),
+     fprintf(fp,"%s\tleast one %s is significantly different.\n",pad,subject);
+        fprintf(fp,"\n");
+        fprintf(fp,"%s\tFurther, the probablity of there being a difference is\n",
+               pad);
+        for (i=GEN_X2_PER;i<MAX_X2_PER;i++)
+            if (X2_r < X2_score(df,i+1))
+                break;
+        if (i==MAX_X2_PER)
+            fprintf(fp,"%s\tgreater that %s.\n",pad,X2_neg_pct_str(i));
+        else
+            fprintf(fp,"%s\tbetween %s to %s.\n",pad,X2_neg_pct_str(i),
+                                             X2_neg_pct_str(i+1));
+        fprintf(fp,"\n\n");
+    }
+    else{
+       fprintf(fp,"%s\tThe test statistic X2_r shows that at the %s confidence\n"
+                          ,pad,X2_neg_pct_str(GEN_X2_PER)),
+       fprintf(fp,"%s\tinterval, the %ss are not significantly\n",pad,subject);
+        fprintf(fp,"%s\tdifferent.\n\n",pad);
+        fprintf(fp,"%s\tFurther, the probablity of there being a difference is\n"
+                          ,pad);
+        for (i=GEN_X2_PER;i>MIN_X2_PER;i--)
+            if (X2_r > X2_score(df,i-1))
+                break;
+        if (i==MIN_X2_PER)
+            fprintf(fp,"%s\tless than %s.\n",pad,X2_neg_pct_str(i));
+        else
+            fprintf(fp,"%s\tbetween %s to %s.\n",pad,X2_neg_pct_str(i-1),
+                                         X2_neg_pct_str(i));
+        fprintf(fp,"\n\n");
+    }
+}
+
+/********************************************************************/
+/*  return the doubleing point value of the sum of the squares       */
+/********************************************************************/
+static double dsum_sqr(double *arr, int len)
+{
+    int i;
+    double tot=0.000000;
+
+    for (i=0;i<len;i++)
+        tot += (arr[i]*arr[i]);
+    return(tot);
+}
+
+/********************************************************************/
+/*     Ties adjustments                                             */
+/*              b                                                   */
+/*             --.                                                  */
+/*    1   -    \   T                                                */
+/*             /    i                                               */
+/*             --'                                                  */
+/*          -----------                                             */
+/*                2                                                 */
+/*            bt(t -1)                                              */
+/*                                                                  */
+/*               3                                                  */
+/*     T =  Sum t   - Sum t        where t = num ties in ith block  */
+/*      i        i         i                                        */
+/*                                                                  */
+/*   to adjust for ties, the test statistic must be taken down      */
+/*   by a certain factor, this function returns it                  */
+/********************************************************************/
+static double calc_adjust_for_trt(RANK *rank)
+{
+    double *sort_arr;
+    int sort_num, *sort_count, i, j, sum_T=0;
+
+    alloc_singarr(sort_arr,rnk_trt(rank),double);
+    alloc_singarr(sort_count,rnk_trt(rank),int);
+
+    /* find the ties ans sum Ti */
+    for (i=0;i<rnk_blks(rank);i++){
+        for (j=0;j<rnk_trt(rank); j++){
+            sort_arr[j]=0.0;
+            sort_count[j]=0;
+	}
+        for (j=0;j<rnk_trt(rank); j++){
+            sort_num = 0;
+            while ((sort_arr[sort_num] != 0.0) && 
+                   (Vrnk_t_rank(rank,i,j) != sort_arr[sort_num]))
+                sort_num++;
+            sort_arr[sort_num] = Vrnk_t_rank(rank,i,j);
+            sort_count[sort_num]++;
+	}
+        for (j=0;j<rnk_trt(rank); j++)
+            if (sort_count[j] > 0)
+                sum_T += (sort_count[j] * sort_count[j] * sort_count[j])
+                           - sort_count[j];
+    }
+    free_singarr(sort_arr,double);
+    free_singarr(sort_count,int);
+
+    /* return the adjustment number */
+    return( 1.000000 - (sum_T/
+                        (double)(rnk_blks(rank)*
+                                rnk_trt(rank)*
+                                ((rnk_trt(rank) * rnk_trt(rank)) - 1))));
+}
+
+/********************************************************************/
+/*  see calc_adjust_for_trt but substitute B for t                  */
+/********************************************************************/
+static double calc_adjust_for_blks(RANK *rank)
+{
+    double *sort_arr;
+    int sort_num, *sort_count, i, j, sum_T=0;
+
+    alloc_singarr(sort_arr,rnk_blks(rank),double);
+    alloc_singarr(sort_count,rnk_blks(rank),int);
+
+    /* find the ties ans sum Ti */
+    for (i=0;i<rnk_trt(rank);i++){
+        for (j=0;j<rnk_blks(rank); j++){
+            sort_arr[j]=0.0;
+            sort_count[j]=0;
+	}
+        for (j=0;j<rnk_blks(rank); j++){
+            sort_num = 0;
+            while ((sort_arr[sort_num] != 0.0) && 
+                   (Vrnk_b_rank(rank,j,i) != sort_arr[sort_num]))
+                sort_num++;
+            sort_arr[sort_num] = Vrnk_b_rank(rank,j,i);
+            sort_count[sort_num]++;
+	}
+        for (j=0;j<rnk_trt(rank); j++)
+            if (sort_count[j] > 0)
+                sum_T += (sort_count[j] * sort_count[j] * sort_count[j])
+                           - sort_count[j];
+    }
+    free_singarr(sort_arr,double);
+    free_singarr(sort_count,int);
+    /* return the adjustment number */
+    return( 1.000000 - (sum_T/
+                        (double)(rnk_trt(rank)*
+                                rnk_blks(rank)*
+                                ((rnk_blks(rank) * rnk_blks(rank)) - 1))));
+}
+
+/********************************************************************/
+/*  for the treatments, calculate and do the MULTIPLE comparison    */
+/*  print everything out if requested                               */
+/********************************************************************/
+static void do_multi_comp_for_anovar(int trt_num, int blk_num, char **trt_names, double *sum_arr, double *ovr_rank_arr, char *title, int *srt_ptr, int **stat_sum, int verbose, FILE *fp)
+{
+    calc_stat_ranks(trt_num,blk_num,stat_sum,sum_arr);
+
+    if (verbose)
+        print_ANOVAR_comp_matrix(trt_num,stat_sum,trt_names,title,srt_ptr,fp);
+}
+
+/*********************************************************************/
+/*      Multiple comparison test to rank the systems                 */
+/*                             ________                              */
+/*                            / bt(t+1)                              */
+/*     | R  - R  |    >=   z / --------                              */
+/*        j    i            V     6                                  */ 
+/*                                                                   */
+/*  R = the Sum of the nth  column ranks                             */
+/*  z = Z score the alpha = 0.05                                     */
+/*                          ----                                     */
+/*                         t(t-1)                                    */
+/*                                                                   */
+/*  go through all possible comparisons and rank the treatments for  */
+/*  according to the number of treatments beaten or beaten by        */
+/*                                                                   */
+/*********************************************************************/
+static void calc_stat_ranks(int trt_num, int blk_num, int **stat_sum, double *sum_arr)
+{
+    int comp1, comp2, result;
+    double Zcomp_value, compare_value;
+    double sqr_root;
+
+    /* calculate the comparison value for the confidence interval */
+    Zcomp_value = calc_comp_value(trt_num,blk_num);
+    sqr_root = sqrt(blk_num * trt_num * (trt_num - 1) / 6.000000);
+    compare_value = (double)Zcomp_value * (double)sqr_root;
+
+    /* go through the comparison and build an array to rank */
+    for (comp1=0; comp1 <(trt_num-1); comp1++)
+        for (comp2=comp1+1; comp2<(trt_num); comp2++){
+            if (fabs(sum_arr[comp1] - sum_arr[comp2]) >= compare_value)
+                if (sum_arr[comp1] > sum_arr[comp2])
+                    result = 1;
+                else
+                    result = -1;
+            else
+                result = 0;
+            stat_sum[comp1][comp2] += result;
+            stat_sum[comp2][comp1] += result * (-1);
+	}
+}
+
+/*********************************************************************/
+/*   Given the alpha, find the closest approximate Z index to that   */
+/*   percentage                                                      */
+/*********************************************************************/
+static double calc_comp_value(int trt_num, int blk_num)
+{
+    double Z_area;
+    int i;
+
+    Z_area = MULTI_COMP_ALPHA / (double)(trt_num * (trt_num-1));
+
+    if (Z_area < Z_exter(Z1tail,MAX_Z_PER))
+        return(Z_score(Z1tail,MAX_Z_PER));
+
+    for (i=MAX_Z_PER+1; i< MIN_Z_PER ; i++)
+        if (Z_area < Z_exter(Z1tail,i))
+            return(Z_score(Z1tail,i));
+
+    return(Z_score(Z1tail,MIN_Z_PER));
+}
+
+
+static void print_ANOVAR_comp_matrix(int trt_num, int **stat_sum, char **trt_names, char *title, int *srt_ptr, FILE *fp)
+{
+    char pad[FULL_SCREEN],name_format[50],*hyphens="--------------------";
+    char *spaces="                    ";
+    int t, t2;
+    int max_trt_name_len=4, hy_l=20;
+
+    set_pad(pad,title, FULL_SCREEN);
+    fprintf(fp,"\n\n%s%s\n",pad,title);
+    set_pad(pad,RANK_TITLE_1, FULL_SCREEN);
+    fprintf(fp,"%s%s\n\n\n\n",pad,RANK_TITLE_1);
+
+    for (t=0; t<trt_num; t++)
+        if (max_trt_name_len < strlen(trt_names[t]))
+            max_trt_name_len = strlen(trt_names[t]);
+    set_pad_cent_n(pad,(trt_num+1) * (max_trt_name_len+2+1), FULL_SCREEN);
+    sprintf(name_format,"| %%-%ds ",max_trt_name_len);
+
+    /* first line */
+    fprintf(fp,"%s|%s",pad,(hyphens+(hy_l-(max_trt_name_len+2))));
+    for (t=0; t<trt_num; t++)
+       fprintf(fp,"%s",(hyphens+(hy_l-(max_trt_name_len+3))));
+    fprintf(fp,"|\n");
+
+    /* systems on top */
+    fprintf(fp,"%s|%s",pad,(spaces+(hy_l-(max_trt_name_len+2))));
+    for (t=0; t<trt_num; t++)
+       fprintf(fp,name_format,trt_names[srt_ptr[t]]);
+    fprintf(fp,"|\n"); 
+
+    /* separation line */
+    fprintf(fp,"%s|%s+",pad,(hyphens+(hy_l-(max_trt_name_len+2))));
+    for (t=0; t<trt_num-1; t++)
+       fprintf(fp,"%s",(hyphens+(hy_l-(max_trt_name_len+3))));
+    fprintf(fp,"%s|\n",(hyphens+(hy_l-(max_trt_name_len+2))));
+
+    /* systems */
+    for (t=0; t<trt_num; t++){
+       fprintf(fp,"%s",pad);
+       fprintf(fp,name_format,trt_names[srt_ptr[t]]);
+       for (t2=0; t2<trt_num; t2++){
+           if (t < t2)
+               switch (stat_sum[srt_ptr[t]][srt_ptr[t2]]) {
+                   case -1: fprintf(fp,name_format,trt_names[srt_ptr[t]]);
+                            break;
+                   case 1:  fprintf(fp,name_format,trt_names[srt_ptr[t2]]);
+                            break;
+                   default: fprintf(fp,name_format,"same");
+                            break;
+	       }
+           else
+               fprintf(fp,name_format,"");
+       }
+       fprintf(fp,"|\n");
+    }
+
+    /* last line */
+    fprintf(fp,"%s|%s",pad,(hyphens+(hy_l-(max_trt_name_len+2))));
+    for (t=0; t<trt_num; t++)
+       fprintf(fp,"%s",(hyphens+(hy_l-(max_trt_name_len+3))));
+    fprintf(fp,"|\n");
+}
+
+

SCTK/src/sclite/arcseq1.c

@@ -0,0 +1,13 @@
+/* file arcseq1.c */
+
+#include "sctk.h"
+
+boolean arcs_equal(ARC *arc1, ARC *arc2)                  
+/***************************************************/
+/* Determines if two arcs of a network are equal.  */
+/***************************************************/
+{if ((arc1->from_node == arc2->from_node) &&
+/*      (streq(arc1->symbol,arc2->symbol)) && */
+     (arc1->to_node   == arc2->to_node)) return T;
+ else  return F;
+}

SCTK/src/sclite/config.in

@@ -0,0 +1,151 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(sclite.c)
+
+dnl Checks for programs.
+AC_PROG_CC
+dnl AC_PROG_INSTALL
+AC_PROG_MAKE_SET
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_STDC
+AC_CHECK_HEADERS(unistd.h)
+
+dnl If gcc is used, check the system.  If it's any of the below,
+dnl add OPTIONS to CFLAGS
+if test "$CC" = "cc" ; then
+    echo "Setting default CFLAGS for Native cc"
+    case "`uname -s`" in
+	OSF1)   CFLAGS="$CFLAGS -std1"; export CFLAGS;;
+	HP-UX) CFLAGS="$CFLAGS -Ae"; export CFLAGS;;
+    esac
+fi
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_C_CONST
+AC_TYPE_SIZE_T
+
+dnl Checks for library functions.
+dnl AC_FUNC_VPRINTF
+dnl AC_CHECK_FUNCS(strdup)
+dnl AC_CHECK_FUNC(readpipe)
+
+AC_PATH_PROGS(INSTALL, install cp copy)
+
+echo ""
+echo "!!!!!!!!!!!!!   USER INFORMATION NEEDED   !!!!!!!!!!!!!"
+echo ""
+echo "Sclite has the ability to use GNU's 'diff' program as means to"
+echo "align reference and hypothesis strings.  Currently, no evaluations"
+echo "have used GNU-DIFF for alignments, so it may be unnecessary for"
+echo "you to enable its use.  Also, some versions of GNU's diff, V2.8.1,"
+echo "will not work with GB encoded Mandarin characters and test6 of the"
+echo "test suite will fail."
+echo ""
+echo "    Do you want to enable alignments via GNU's 'diff'?  yes or no"
+echo ""
+ans=""
+while test "$ans" = "" ; do 
+	read ans
+	ans=`echo $ans | tr 'A-Z' 'a-z'`
+	if test "$ans" = "no" ; then
+		echo "Disabling GNU-DIFF alignments"
+	elif test "$ans" = "yes" ; then
+		echo "Enabling GNU-DIFF alignments"
+	else
+		echo "Error: You must respond either yes or no"
+		ans=""
+	fi
+done
+
+CFLAGS_BEFORE_DIFF=$CFLAGS
+if test "$ans" = "yes" ; then
+  AC_PATH_PROGS(DIFF, gnudiff diffgnu gdiff diff)
+  case "$DIFF" in
+    *gdiff)
+      # Carefully avoid gdiff for X as found on SGI systems.
+      DISPLAY= gdiff /dev/null /dev/null > /dev/null 2>&1 || DIFF=""
+      if test -z "$DIFF" ; then
+	  echo "    Not using $DIFF, it's an X window executable"
+	  AC_PATH_PROGS(NDIFF, diff)
+	  DIFF=$NDIFF
+      fi
+      ;;
+  esac
+  if test -z "$DIFF" ; then
+    echo "Error: Unable to locate 'diff'"
+    exit 1
+  fi
+  echo "    Making sure diff supports '-y'"
+  if test ! -z "`$DIFF -y /dev/null /dev/null 2>&1`" ; then
+    echo "Error: $DIFF does not support '-y'.  Install GNU's diff"
+    exit 1
+  fi
+  CFLAGS="$CFLAGS -DDIFF_EXE=\\\"\$(GNUDIFF)\\\""
+else
+  DIFF=""
+fi
+
+echo ""
+echo "!!!!!!!!!!!!!   USER INFORMATION NEEDED   !!!!!!!!!!!!!"
+echo ""
+echo "In support of the LVCSR/HUB-5 evaluations, a newly defined DP"
+echo "alignment cost function has been incorporated into SCTK.  The cost"
+echo "function is defined to be a function of language model probabilities."
+echo "In order to compute the LM probabilities, the CMU-Cambridge SLM"
+echo "Toolkit - V2 has been included in SCTK.  The compilation including"
+echo "the SLM toolkit is optional, depending on your needs."
+echo ""
+echo "    Do you want to compile in the CMU-Cambridge SLM toolkit?  yes or no"
+echo ""
+ans=""
+SLM_DEFS=""
+while test "$ans" = "" ; do 
+	read ans
+	ans=`echo $ans | tr 'A-Z' 'a-z'`
+	if test "$ans" = "no" ; then
+		echo "Disabling Compilation of CMU-Cambridge SLM "
+		SLM_TARGETS=""
+	elif test "$ans" = "yes" ; then
+		echo "Enabling Compilation of CMU-Cambridge SLM "
+                echo "    Checking for Un-expanded tar archive of SLM"
+                if test -f slm_v2.tar ; then
+                        echo "        Expanding tar file"
+                        tar xf slm_v2.tar
+                        rm slm_v2.tar
+                fi
+		echo "    Checking Endian-ness for SLM toolkit"
+		echo 'main(){short s=1; char *b=(char *)&s; printf("%s",(*b==0)?"Big":"Little");}' > x.c
+		$CC $CFLAGS_BEFORE_DIFF x.c -o a.out
+		endian=`./a.out`
+		rm -f a.out x.c
+
+		if test "$endian" = "Little" ; then
+			SLM_DEFS="-DSLM_SWAP_BYTES"
+			echo "        Little-endian machine"
+		else
+			echo "        Big-endian machine"
+		fi
+		CFLAGS="$CFLAGS -DWITH_SLM"
+		SLM_TARGETS="slm_v2/lib/SLM2.a"
+		cat slm_v2/src/makefile.in | \
+			sed "s/@CC@/$CC/" | \
+			sed "s/@SCTK_FLAGS@/$CFLAGS_BEFORE_DIFF/" | \
+			sed "s/@SLM_DEFS@/$SLM_DEFS/"  > slm_v2/src/makefile
+	else
+		echo "Error: You must respond either yes or no"
+		ans=""
+	fi
+done
+
+AC_SUBST(SLM_DEFS)		
+AC_SUBST(SLM_TARGETS)		
+
+dnl Check to see if nist.local exists. If it does, define AT_NIST.
+echo "Checking installation site"
+test -f nist.local && CFLAGS="$CFLAGS -DAT_NIST -DPEDANTIC -ansi -pedantic -pedantic-errors -Wall -Wstrict-prototypes -Wmissing-prototypes -DWARN_ZERO_MALLOC"
+
+AC_OUTPUT(Makefile)

SCTK/src/sclite/ctm2ctm.c

@@ -0,0 +1,518 @@
+#include "sctk.h"
+
+int chop_WTOKE_2(WTOKE_STR1 *ref, WTOKE_STR1 *hyp, int Rstart, int Hstart, int Rendconv, int Hendconv, int max_words, int *Rret_end, int *Hret_end);
+
+
+SCORES *align_ctm_to_ctm(char *hyp_file, char *ref_file, char *set_title, int feedback, int fcorr, int opt_del, int case_sense, int time_align, int left_to_right, WWL *wwl, char *lm_file){
+
+    char *proc = "align_ctm_to_ctm";
+    WTOKE_STR1 *hyp;
+    WTOKE_STR1 *ref;
+    FILE *fp_hyp, *fp_ref;
+    int hyp_conv_end, hyp_eof;
+    int ref_conv_end, ref_eof;
+    int hyp_chop_end, ref_chop_end;
+    int i;
+    PATH *path;
+    int ref_end_chan1, hyp_end_chan1, ref_end_chan2, hyp_end_chan2;
+    int ref_end, hyp_end, ref_begin, hyp_begin;
+    int ref_opp_end, hyp_opp_end, ref_opp_begin, hyp_opp_begin;
+    int dbg = TRUE;
+    int chunk;
+    char uttid[100];
+    int spkr;
+    int number_of_channels;
+    SCORES *scor;
+    NETWORK *hnet, *rnet;
+    int keep_path = 1;
+   
+    hyp = WTOKE_STR1_init(hyp_file);
+    ref = WTOKE_STR1_init(ref_file);
+
+    scor = SCORES_init(set_title,20);
+    scor->ref_fname = (char *)TEXT_strdup((TEXT *)ref_file);
+    scor->hyp_fname = (char *)TEXT_strdup((TEXT *)hyp_file);
+    scor->creation_date = get_date();
+    scor->frag_corr = fcorr;
+    scor->opt_del = opt_del;
+    if (wwl != (WWL*)0) {
+      scor->weight_ali = 1;
+      scor->weight_file =
+	(char *)TEXT_strdup((TEXT*)rsprintf("%s Column \"%s\"",wwl->filename,
+					    wwl->weight_desc[0]));
+    } else if (lm_file != (char *)0) {
+      scor->weight_ali = 1;
+      scor->weight_file = (char *)TEXT_strdup((TEXT*)lm_file);
+    } else
+      scor->weight_ali = 0;
+
+    dbg=0;
+
+    if ((fp_hyp = fopen(hyp_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input hypothesis file %s\n",hyp_file);
+        exit(1);
+    }
+    if ((fp_ref = fopen(ref_file,"r")) == NULL){
+        fprintf(stderr,"Can't open input reference file %s\n",ref_file);
+	exit(1);
+    }
+
+    hyp_eof = ref_eof = FALSE;
+    do {
+        if (dbg  > 5) printf("Starting do-while loop on conversations"
+			     "--------------------------\n");
+	fill_WTOKE_structure(hyp, fp_hyp, hyp_file, &hyp_eof, case_sense);
+	fill_WTOKE_structure(ref, fp_ref, ref_file, &ref_eof, case_sense);
+	
+	if ((hyp->n < hyp->s) && (ref->n > ref->s)) {
+	  fprintf(stderr,"Error: premature EOF in hyp file '%s' at conv"
+		  "ersation '%s' of ref file '%s'\n",hyp_file,
+		  ref->word[ref->s].conv,ref_file);
+	  exit(1);
+	}
+	if ((hyp->n > hyp->s) && (ref->n < ref->s)) {
+	  fprintf(stderr,"Error: premature EOF in ref file '%s' at conv"
+		  "ersation '%s' of hyp file '%s'\n",ref_file,
+		  hyp->word[hyp->s].conv,hyp_file);
+	  exit(1);
+	}
+	
+	locate_boundary(ref, ref->s, TRUE, FALSE, &ref_conv_end);
+	locate_boundary(hyp, hyp->s, TRUE, FALSE, &hyp_conv_end);
+
+	/* let's do some checking !!!!!! */
+	/* Are the conversations in sync */
+	if (strcmp(ref->word[ref->s].conv,hyp->word[hyp->s].conv)!=0){
+	  fprintf(stderr,"Error: in ref file '%s' and hyp file '%s'\n",
+		  ref_file, hyp_file);
+	  fprintf(stderr,"       file strings out of "
+		  "synchronization around ref-file='%s'  hyp-file='%s'\n",
+		  ref->word[ref->s].conv,hyp->word[hyp->s].conv);
+	  exit(1);
+	}
+
+	locate_boundary(ref, ref->s, TRUE, TRUE, &ref_end_chan1);
+	locate_boundary(hyp, hyp->s, TRUE, TRUE, &hyp_end_chan1);
+
+	number_of_channels = 1;
+	hyp_end_chan2 = ref_end_chan2 = -1;
+	if (ref_end_chan1+1 < ref->n &&
+	    strcmp(ref->word[ref->s].conv,ref->word[ref_end_chan1+1].conv)==0){
+	    locate_boundary(ref, ref_end_chan1+1, TRUE, TRUE, &ref_end_chan2);
+	    number_of_channels = 2;
+	}
+	if (hyp_end_chan1+1 < hyp->n &&
+	    strcmp(hyp->word[hyp->s].conv,hyp->word[hyp_end_chan1+1].conv)==0){
+	    locate_boundary(hyp, hyp_end_chan1+1, TRUE, TRUE, &hyp_end_chan2);
+	    number_of_channels = 2;
+	}
+
+	if (number_of_channels == 2){
+	  /* either the ref, hyp or both had two channels.  make sure there */
+	  /* are two channels in both */
+	  if (hyp_end_chan2 == -1){
+	    fprintf(stderr,"Error: in ref file '%s' and hyp file '%s'\n   ",
+		    ref_file, hyp_file);
+	    fprintf(stderr,"    Hyp is missing second channel of file '%s'\n",
+		    ref->word[ref->s].conv);
+	    exit(1);
+	  }
+	  if (ref_end_chan2 == -1){
+	    fprintf(stderr,"Error: in ref file '%s' and hyp file '%s'\n   ",
+		    ref_file, hyp_file);
+	    fprintf(stderr,"    ref is missing second channel of file '%s'\n",
+		    hyp->word[hyp->s].conv);
+	    exit(1);
+	  }
+	  
+	  
+	} else {
+	    /* There was no second channel */
+	    ref_end_chan2 = ref_end_chan1;
+	    hyp_end_chan2 = hyp_end_chan1;
+	}
+
+	if (feedback >= 1)
+	    printf("    Performing %s alignments for file '%s'.\n",
+		   (time_align?"Time-mediated":"Word"),
+		   ref->word[ref->s].conv);
+
+	if (dbg > 5){
+	    printf("Located REF boundary: for %s %s -> %d,%d\n",
+		   ref->word[ref->s].conv,
+		   ref->word[ref->s].turn,ref->s,ref_conv_end);
+	    printf("Located HYP boundary: for %s %s -> %d,%d\n",
+		   hyp->word[hyp->s].conv,
+		   hyp->word[hyp->s].turn,hyp->s,hyp_conv_end);
+
+	    printf("Located REF Channel 1 boundary: for %d - %d\n",
+		   ref->s,ref_end_chan1);
+	    printf("Located HYP Channel 1 boundary: for %d - %d\n",
+		   hyp->s,hyp_end_chan1);	
+	    printf("Located REF Channel 2 boundary: for %d - %d\n",
+		   ref_end_chan1+1,ref_end_chan2);
+	    printf("Located HYP Channel 2 boundary: for %d - %d\n",
+		   hyp_end_chan1+1,hyp_end_chan2);
+	}
+
+
+	/* Start the alignment */
+	/* Side A */
+	for (i=0; i<number_of_channels; i++){
+	    int base_len;
+	    chunk = 0;	    
+	    if (i==0){
+		ref_end = ref_end_chan1;
+		hyp_end = hyp_end_chan1;
+		ref_begin = ref->s;
+		hyp_begin = hyp->s;
+
+		ref_opp_end = ref_end_chan2;
+		hyp_opp_end = hyp_end_chan2;
+		ref_opp_begin = ref_end_chan1 + 1;
+		hyp_opp_begin = hyp_end_chan1 + 1;
+	    } else {
+		ref_end = ref_end_chan2;
+		hyp_end = hyp_end_chan2;
+		ref_begin = ref_end_chan1 + 1;
+		hyp_begin = hyp_end_chan1 + 1;
+
+		ref_opp_end = ref_end_chan1;
+		hyp_opp_end = hyp_end_chan1;
+		ref_opp_begin = ref->s;
+		hyp_opp_begin = hyp->s;
+	    }
+	    if (dbg > 5){
+		printf("Starting channel %d\n",i);
+		printf("Ref Range: %d - %d\n",ref_begin,ref_end);
+		printf("Hyp Range: %d - %d\n",hyp_begin,hyp_end);
+	    }
+	    while (ref_begin<=ref_end || hyp_begin<=hyp_end){
+		int removals;
+
+		base_len = 50;
+		while (! chop_WTOKE_2(ref, hyp, ref_begin, hyp_begin,
+				      ref_end, hyp_end, base_len,
+				      &ref_chop_end,&hyp_chop_end)){
+		    /* printf("Expanding len from %d to %d\n",base_len,
+		       base_len*2); */
+		    base_len *= 2;
+		}
+
+		removals = 0;
+
+		/* create the utterance ID */
+		sprintf(uttid,"(%s-%s-%04d)",ref->word[ref_begin].conv,
+			ref->word[ref_begin].turn,chunk);
+
+		spkr = SCORES_get_grp(scor,
+				      rsprintf("%s-%s",
+					       ref->word[ref_begin].conv,
+					       ref->word[ref_begin].turn));
+
+		if (dbg > 5){
+		    printf("Chunk %d: Starting for channel %d  id=%s",
+			   chunk,i,uttid);
+		    printf("    Ref Range: %d - %d",ref_begin,ref_chop_end);
+		    printf("    Hyp Range: %d - %d\n",hyp_begin,hyp_chop_end);
+		}
+		
+
+		/* create the networks to be aligned */
+		if ((hnet=Network_create_from_WTOKE(hyp,hyp_begin,
+					 hyp_chop_end,"Hypothesis net",
+					 print_WORD_wt,
+					 equal_WORD2,
+				         release_WORD, null_alt_WORD,
+						    opt_del_WORD,
+						    copy_WORD,make_empty_WORD,
+						    use_count_WORD,
+						    left_to_right))
+		    == NULL_NETWORK){ 
+		    fprintf(stderr,"%s: Network_create_from_WTOKE failed\n",
+			    proc);
+		    return(0);
+		}
+		if ((rnet=Network_create_from_WTOKE(ref,ref_begin,
+					 ref_chop_end,"Reference net",
+					 print_WORD_wt,
+					 equal_WORD2,
+				         release_WORD, null_alt_WORD,
+						    opt_del_WORD,
+						    copy_WORD,make_empty_WORD,
+						    use_count_WORD,
+						    left_to_right))
+		    == NULL_NETWORK){ 
+		    fprintf(stderr,"%s: Network_create_from_WTOKE failed\n",
+			    proc);
+		    return(0);
+		}
+
+		path = network_dp_align_texts((TEXT *)0, rnet,
+					      (TEXT *)0, hnet,
+					      FALSE, case_sense, 
+					      (char *)uttid, fcorr,
+					      opt_del, time_align, wwl, lm_file);
+		BF_SET(path->attrib,PA_HYP_WTIMES);
+		BF_SET(path->attrib,PA_REF_WTIMES);
+		if (ref->has_conf) BF_SET(path->attrib,PA_REF_CONF);
+		if (hyp->has_conf) BF_SET(path->attrib,PA_HYP_CONF);
+
+		sort_PATH_time_marks(path);
+
+		if (dbg > 10){
+		    printf("After Sorting\n");
+		    PATH_print_wt(path, stdout);
+		}
+
+		add_PATH_score(scor,path,spkr, keep_path);
+		PATH_add_file(path,ref->word[ref_begin].conv);
+		PATH_add_channel(path,ref->word[ref_begin].turn);
+		/* PATH_add_label(path,(char *)stm->seg[rs].labels); */
+	
+		ref_begin = ref_chop_end+1;
+		hyp_begin = hyp_chop_end+1;
+
+		chunk++;
+	    }
+	}
+
+
+	hyp->s = hyp_conv_end+1;
+	ref->s = ref_conv_end+1;
+
+    } while (hyp->s <= hyp->n || ref->s <= ref->n);
+
+    free_mark_file(ref);
+    free_mark_file(hyp);
+
+    fclose(fp_hyp);
+    fclose(fp_ref);
+
+    return (scor);
+}
+
+/*******************************************************************/
+/*  Compute the overlap of two segments in time and return that    */
+/*  time.  Negative times indicate no overlap.  There are 6 cases  */
+/*  Case 1:    S1   t1  t2                                         */
+/*             S2            t1  t2                                */
+/*                                                                 */
+/*  Case 2:    S1            t1  t2                                */
+/*             S2   t1  t2                                         */
+/*                                                                 */
+/*  Case 3a:   S1   t1    t2                                       */
+/*             S2      t1    t2                                    */
+/*                                                                 */
+/*  Case 3b:   S1   t1           t2                                */
+/*             S2      t1    t2                                    */
+/*                                                                 */
+/*  Case 4a:   S1         t1    t2                                 */
+/*             S2      t1           t2                             */
+/*                                                                 */
+/*  Case 4b:   S1         t1        t2                             */
+/*             S2      t1       t2                                 */
+/*                                                                 */
+double overlap(double s1_t1, double s1_t2, double s2_t1, double s2_t2)
+{ 
+    double rval;
+    char *rule;
+    int dbg=0;
+    /* Case 1: */
+    if (s1_t2 < s2_t1)
+	rule="Case 1", rval=(s1_t2 - s2_t1);
+    /* Case 2: */
+    if (s1_t1 > s2_t2)
+	rule="Case 2", rval=(s2_t2 - s1_t1);
+    /* Case 3: */
+    if (s1_t1 < s2_t1){
+	/* Case 3a: */
+	if (s1_t2 < s2_t2)
+	    rule="Case 3a", rval=(s1_t2 - s2_t1);
+	else /* Case 3b: */
+	    rule="Case 3b", rval=(s2_t2 - s2_t1);
+    } else {  /* Case 4: */
+	/* Case 4a: */
+	if (s1_t2 < s2_t2)
+	    rule="Case 4a", rval=(s1_t2 - s1_t1);
+	else /* Case 4b: */
+	    rule="Case 4b", rval=(s2_t2 - s1_t1);
+    }
+    if (dbg){
+	printf("Overlap: s1_t1=%2.2f s1_t2=%2.2f",s1_t1,s1_t2);
+	printf(" s2_t1=%2.2f s2_t2=%2.2f   Rule:%s  Rval=%2.2f\n",
+	       s2_t1,s2_t2,rule,rval);
+    }
+    return(rval);
+}
+
+
+int chop_WTOKE_2(WTOKE_STR1 *ref, WTOKE_STR1 *hyp, int Rstart, int Hstart, int Rendconv, int Hendconv, int max_words, int *Rret_end, int *Hret_end)
+{
+    int Rend, Hend, Rend_sync, Hend_sync;
+    char *proc="chop_WTOKE_2";
+    int dbg=0, skipped_alt;
+    double Rgap_t1, Rgap_t2, Hgap_t1, Hgap_t2;
+    
+    if (dbg)
+	printf("%s:  Start Ref=%d  Start Hyp=%d  Max_words=%d\n",
+	       proc,Rstart,Hstart,max_words);
+
+    /* Find the end of this segment */
+    /*     locate_boundary(ref, Rstart, TRUE, TRUE, &Rendconv);
+	   locate_boundary(hyp, Hstart, TRUE, TRUE, &Hendconv); */
+
+    if (dbg) printf("%s:  Conversation Rendconv=%d  Hendconv=%d\n",
+		    proc,Rendconv,Hendconv);
+
+    if ((Rstart > Rendconv) && (Hstart > Hendconv))
+	return(0);
+
+    /* if we've hit the end of the conversation, send everything */
+    if (((Rendconv - Rstart) <max_words) && ((Hendconv - Hstart) < max_words)){
+	*Rret_end = Rendconv; 
+	*Hret_end = Hendconv; 
+	if (dbg) printf("%s: R1 Success Chop Ref (%d-%d)  Hyp (%d-%d)\n",
+			proc,Rstart,*Rret_end,Hstart,*Hret_end);
+	return(1);
+    }
+
+    /* set the indexes to maximum length */
+    Rend = MIN(Rstart + max_words, Rendconv);
+    Hend = MIN(Hstart + max_words, Hendconv);
+	
+    if (dbg) printf("%s:  Limited Rend=%d  Hend=%d\n",proc,Rend,Hend);
+
+    /* synchronize the ends to permit searching */
+    while (Rend >= Rstart && ref->word[Rend].alternate)
+	Rend--;
+    skipped_alt = TRUE;
+    while (skipped_alt){
+	skipped_alt = FALSE;
+	if (ref->word[Rend].t1 > hyp->word[Hend].t1)
+	    while (!skipped_alt &&
+		   Rend >= Rstart && 
+		   ref->word[Rend].t1 > hyp->word[Hend].t1 && 
+		   overlap(ref->word[Rend].t1,
+			   ref->word[Rend].t1+ref->word[Rend].dur,
+			   hyp->word[Hend].t1,
+			   hyp->word[Hend].t1+hyp->word[Hend].dur) < 0.0)
+		for (Rend--; Rend >= Rstart && ref->word[Rend].alternate; ) {
+		    Rend--;
+		    skipped_alt = TRUE;
+		}
+	else if (ref->word[Rend].t1 < hyp->word[Hend].t1)
+	    while (Hend >= Hstart &&
+		   ref->word[Rend].t1 < hyp->word[Hend].t1 && 
+		   overlap(ref->word[Rend].t1,
+			   ref->word[Rend].t1+ref->word[Rend].dur,
+			   hyp->word[Hend].t1,
+			   hyp->word[Hend].t1+hyp->word[Hend].dur) < 0.0)
+		Hend --;
+    }
+    if (dbg) {
+	printf("%s:  Synchronized Rend=%d  Hend=%d  ",proc,Rend,Hend);
+	printf("Times: %2.2f,%2.2f   %2.2f,%2.2f\n",
+	       ref->word[Rend].t1, ref->word[Rend].t1+ref->word[Rend].dur,
+	       hyp->word[Hend].t1, hyp->word[Hend].t1+hyp->word[Hend].dur);
+    }
+    Rend_sync = Rend;
+    Hend_sync = Hend;
+
+    /* begin the backward search */
+    while (Rend > Rstart && Hend > Hstart){
+	/* first compute t1 and t2 for the gap in each segment */
+	if (Rend >= Rstart){
+	    Rgap_t1 = ref->word[Rend].t1 + ref->word[Rend].dur;
+	    Rgap_t2 = (Rend == Rendconv) ? 999999.99 : ref->word[Rend+1].t1;
+	} else {
+	    Rgap_t1 = 0.0;
+	    Rgap_t2 = ref->word[Rstart].t1;
+	}
+	if (Hend >= Hstart) {
+	    Hgap_t1 = hyp->word[Hend].t1 + hyp->word[Hend].dur;
+	    Hgap_t2 = (Hend == Hendconv) ? 999999.99 : hyp->word[Hend+1].t1;
+	} else {
+	    Hgap_t1 = 0.0;
+	    Hgap_t2 = hyp->word[Hstart].t1;
+	}
+
+	if (dbg) {
+	    printf("\n%s:  Gaps: ref %2.2f,%2.2f  hyp %2.2f,%2.2f\n",
+		   proc,Rgap_t1, Rgap_t2, Hgap_t1, Hgap_t2);
+	    printf("%s:  **** middle word is Rend or Hend\n",proc);
+	    dump_word_tokens2(ref,Rend-1,Rend+1);
+	    dump_word_tokens2(hyp,Hend-1,Hend+1);
+	}
+
+	/* check to see if there is any overlap */
+	/* yes if :    S1   |   |         #          |  |    */
+	/*             S2          |   |  #  |   |           */
+	   
+	if (overlap(Rgap_t1,Rgap_t2,Hgap_t1,Hgap_t2) < 0.0){
+	    /* no overlap decide which to move */
+	
+	    if (dbg) printf("%s: Shifting down\n",proc);
+
+	    skipped_alt = TRUE;
+	    while (skipped_alt){
+		skipped_alt = FALSE;
+		if (ref->word[Rend].t1 > hyp->word[Hend].t1){
+		    if (Rend > Rstart) {
+			Rend --;
+			while (Rend >= Rstart && ref->word[Rend].alternate) {
+			    Rend--;
+			    skipped_alt = TRUE;
+			}
+		    }
+		    if (dbg) printf("%s: Moving Ref\n",proc);
+		    while (!skipped_alt &&
+			   Hend >= Hstart && 
+			   ref->word[Rend].t1 < hyp->word[Hend].t1 &&
+			   overlap(ref->word[Rend].t1,
+				   ref->word[Rend].t1+ref->word[Rend].dur,
+				   hyp->word[Hend].t1,
+				 hyp->word[Hend].t1+hyp->word[Hend].dur)< 0.0){
+			Hend --;
+			if (dbg) printf("%s: Adjusting Hyp\n",proc);
+		    }
+		} else if (ref->word[Rend].t1 <= hyp->word[Hend].t1) {
+		    if (Hend > Hstart) 
+			Hend --;
+		    if (dbg) printf("%s: Moving Hyp\n",proc);
+		    while (!skipped_alt &&
+			   Rend >= Rstart &&
+			   ref->word[Rend].t1 > hyp->word[Hend].t1 && 
+			   overlap(ref->word[Rend].t1,
+				   ref->word[Rend].t1+ref->word[Rend].dur,
+				   hyp->word[Hend].t1,
+				  hyp->word[Hend].t1+hyp->word[Hend].dur)<0.0){
+			for (Rend--;
+			     Rend >= Rstart && ref->word[Rend].alternate; ) {
+			    Rend--;
+			    skipped_alt = TRUE;
+			}
+			if (dbg) printf("%s: Adjusting Ref\n",proc);
+		    }
+		}
+	    }
+	} else { /* Overlap exists !!!!!   Return the values */
+	    *Rret_end = Rend; 
+	    *Hret_end = Hend; 
+	    if (dbg) printf("%s: R2 Success Chop Ref (%d-%d)  Hyp (%d-%d)\n",
+			    proc,Rstart,*Rret_end,Hstart,*Hret_end);
+	    return(1);
+	}
+    }
+
+    if (Rend > Rstart || Hend > Hstart){
+	*Rret_end = Rend; 
+	*Hret_end = Hend; 
+	if (dbg) printf("%s: R3 Success Chop Ref (%d-%d)  Hyp (%d-%d)\n",
+			proc,Rstart,*Rret_end,Hstart,*Hret_end);
+	return(1);
+    }
+    /* fprintf(stdout,"%s: Failure to find chop'd segment, returning 0\n",proc); */
+    return(0);
+
+}

SCTK/src/sclite/llist.c

@@ -0,0 +1,159 @@
+/**  Mark Przybocki
+ **  Filename:  LLIST.C
+ **             Module 1 used to build liblist.a (library).
+ **
+ **/
+
+#include "sctk.h"
+
+static LList *getLList(void *);
+static void freeLList(LList **);
+void recur(LList **t, LList *f);
+
+
+/*
+ *  Function GETLNODE returns a pointer to a lnode which will
+ *  contain the data element taken as input.
+ */
+static LList *getLList (void *pdata)
+{
+    LList *ptemp;  /* temporary lnode being set with data element */
+
+    /* ALLOCATE MEMORY for our TEMP LNODE */
+    ptemp = malloc(sizeof(LList));
+
+    if (ptemp != NULL) {
+	/* we have successfully allocated memory so set the LList */
+	ptemp -> data = pdata;  /** DATA ELEMENT                **/
+	ptemp -> next = NULL;   /** NEXT LLIST NOT ASSIGNED YET  **/
+    }
+
+    return ptemp;  /** RETURNS either a Null LList or a set LList **/
+}
+
+
+
+/*
+ * Function FREELLIST deallocates space occupied by a LList and
+ * clears the LList from the list.
+ */
+static void freeLList (LList **p)
+{
+    free(*p);  /** DEALLOCATE SPACE  **/
+    *p=NULL;   /** Clear value of *p **/
+}
+
+
+
+/*
+ * Function INIT_LIST sets the starting point of a list NULL,
+ * which infact gives existence to a list.
+ */
+void LL_init (LList **s)
+{
+    *s=NULL;  /** creates a NULL list **/
+}
+
+void recur(LList **t, LList *f){ 
+  if (! LL_empty(f)){
+    recur(t,f->next);
+    LL_put_front(t,f->data);
+  }
+}
+
+void LL_copy (LList **t, LList **f)
+{
+    LL_init(t);
+    recur(t,*f);
+}
+
+
+
+/*
+ * Function EMPTY test whether or not LLIST is empty.
+ */
+int LL_empty (LList *s)
+{
+    return ( s==NULL );  /** Test for existing list **/
+}
+
+
+
+/*
+ * Function PUSH places one LList at the head (top) of the list.
+ */
+int LL_put_front (LList **s, void *pdata)
+{
+    LList *ptemp;  /** Temporary LList holder **/
+
+    /* Get space and set a LList */
+    ptemp = getLList(pdata);
+
+    /* Check for error */
+    if (ptemp == NULL)
+	return 0;     /** FAILURE: Couldn't allocate space **/
+
+    /* Add to list */
+    ptemp -> next = *s;
+    
+    /** Set the begining of the list **/
+    *s = ptemp;
+
+    return 1;   /** SUCCESS **/
+}
+
+/*
+ * Function LL_put_tail places one LList at the tail (bottom) of the list.
+ */
+int LL_put_tail (LList **s, void *pdata)
+{
+    LList *ptemp, *pt1 = *s;  /** Temporary LList holder **/
+
+    if (*s == (LList *)0)
+	return LL_put_front(s,pdata);
+
+    /* Get space and set a LList */
+    ptemp = getLList(pdata);
+
+    /* Check for error */
+    if (ptemp == NULL)
+	return 0;     /** FAILURE: Couldn't allocate space **/
+
+    /* traverse to the end of the list */
+    while (pt1->next != NULL)
+	pt1 = pt1->next;
+
+    /* Add to list */
+    pt1 -> next = ptemp;
+    ptemp->next = (LList *)NULL;
+    
+    return 1;   /** SUCCESS **/
+}
+
+
+
+/*
+ *  Function LL_get_first removes a LList for the head (top) of list.
+ */
+int LL_get_first (LList **s, void **apdata)
+{
+    LList *ptemp;  /* Temporary LList holder */
+
+    if (LL_empty(*s) == 1)
+	return 0;  /** list already empty nothing to pop **/
+
+    /* Set ptemp to the first LList */
+    ptemp = *s;
+
+    /* Reset to point to the following LList*/
+    *s = ptemp -> next;
+
+    /* Grab the data element */
+    *apdata = ptemp -> data;
+
+    /* Free the space of the popped LList */
+    freeLList (&ptemp);
+    
+    return 1; /** Success **/
+}
+

SCTK/src/sclite/llist.h

@@ -0,0 +1,32 @@
+/**
+ **  Mark Przybocki
+ **  598 Unix/c
+ **
+ **  Project #7:  PART I  (creating a library)
+ **
+ **  Filename:  LLIST.H
+ **             header file for the liblist.a library
+ **             and other linked list functions.
+ **
+ **/
+
+/** DEFINE THE BASIC NODE STRUCTURE **/
+
+typedef struct lnode LList;
+struct lnode
+{
+ void *data;  /* data element            */
+ void *next;  /* pointer to next element */
+};
+
+/** FUNCTION PROTO-TYPING **/
+
+    /* file: llist.c */
+
+int LL_put_tail(LList **, void *);
+int LL_put_front(LList **, void *);
+int LL_get_first(LList **, void **);
+void LL_init(LList **);
+void LL_copy(LList **, LList **);
+int LL_empty(LList *);
+

SCTK/src/sclite/makefile.in

@@ -0,0 +1,246 @@
+# makefile for SCLITE
+
+#### Start of system configuration section. ####
+
+srcdir = @srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+INSTALL = @INSTALL@
+
+GNUDIFF = @DIFF@
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+CFLAGS = @CFLAGS@
+LDFLAGS = @LDFLAGS@
+SLM_TARGETS = @SLM_TARGETS@
+LIBS = @LIBS@ $(SLM_TARGETS)
+
+# Some System V machines do not come with libPW.
+# If this is true for you, use the GNU alloca.o here.
+
+prefix = @prefix@
+exec_prefix = @exec_prefix@
+edit_program_name = sed '@program_transform_name@'
+
+bindir = $(exec_prefix)/bin
+mandir =$(prefix)/man
+man1dir =$(mandir)/man1
+man5dir = $(mandir)/man5
+
+infodir = $(prefix)/info
+
+DEFAULT_EDITOR_PROGRAM = ed
+NULL_DEVICE = /dev/null
+PR_PROGRAM = /bin/pr
+
+#### End of system configuration section. ####
+
+SHELL = /bin/sh
+
+######################################################################
+# The source files for all of the programs.
+glib_c = order.c rsprintf.c path.c text.c scores.c word.c rpg.c statdist.c \
+	pad.c sgml.c readpipe.c alex.c sc_dtl.c det.c wwscr_f.c corresp.c \
+	slm_intf.c
+
+statlib_c = rank.c range.c anovar.c signtest.c wilcoxon.c sentmcn.c mtchprs.c 
+
+netlib_c = addarc1.c delarc1.c llist.c marc1.c arcseq1.c \
+	mnode1.c boolpr1.c killarc1.c net_adt.c knode2.c net_dp.c 
+
+align_c = stm.c align.c dwtoke2.c fillmrks.c wtokeini.c lur.c stm2ctm.c \
+	ctm2ctm.c mfa.c cores.c
+
+
+######################################################################
+# The Object files for all of the library code.
+glib_o = order.o rsprintf.o path.o text.o scores.o word.o rpg.o statdist.o \
+	pad.o sgml.o readpipe.o alex.o sc_dtl.o det.o wwscr_f.o corresp.o \
+	slm_intf.o
+
+statlib_o = rank.o range.o anovar.o signtest.o wilcoxon.o sentmcn.o mtchprs.o 
+
+netlib_o = addarc1.o delarc1.o llist.o marc1.o arcseq1.o \
+	mnode1.o boolpr1.o killarc1.o net_adt.o knode2.o net_dp.o 
+
+align_o = stm.o align.o dwtoke2.o fillmrks.o wtokeini.o lur.o stm2ctm.o \
+	ctm2ctm.o mfa.o cores.o
+
+######################################################################
+#  Group the libraries into their seperate forms either standalone or
+#  in the tool kit
+
+sclib_c = $(glib_c) $(netlib_c) $(align_c)
+tklib_c = $(sclib_c) $(statlib_c)
+
+sclib_o = $(glib_o) $(netlib_o) $(align_o)
+tklib_o = $(sclib_o) $(statlib_o)
+
+SC_PROGRAMS = sclite
+TK_PROGRAMS = $(SC_PROGRAMS) sc_stats rover sctkUnit
+
+TEST_PROGRAMS =  test test2 sc_conf mtchpath rover_v2 sc_quick mprint
+
+PURIFY_PROGRAMS = $(PROGRAMS:%=%_pure) $(TEST_PROGRAMS:%=%_pure)
+
+PURECOV_PROGRAMS = $(PROGRAMS:%=%_purecov)
+
+DOCS = ../install.txt ../readme.txt ../disclaim.txt ../revision.txt \
+	../makefile ../doc 
+CONFIG = makefile.in config.sh config.in config.txt
+
+tk_distfiles = $(tklib_c) $(TK_PROGRAMS:%=%.c) *.h $(CONFIG) testdata 
+
+######################################################################
+###############   Defines the installation type  #####################
+lib_o = $(tklib_o)
+lib_c = $(tklib_c)
+PROGRAMS = $(TK_PROGRAMS)
+distribution = tk_dist
+
+#lib_o = $(sclib_o)
+#lib_c = $(sclib_c)
+#PROGRAMS = $(SC_PROGRAMS)
+#distribution = sc_dist
+
+######################################################################
+
+
+all: makefile $(SLM_TARGETS) $(PROGRAMS)
+
+
+.c.o:
+	$(CC) -c $(OFLAGS) $(DEFS) -I. $(CFLAGS) $(CPPFLAGS) -o $@ $<
+
+sclite: $(lib_o) sclite.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+sc_stats: $(lib_o) sc_stats.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+rover: $(lib_o) rover.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+mcnemar: $(lib_o) mcnemar.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+sctkUnit: $(lib_o) sctkUnit.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+sign: $(lib_o) sign.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+sclite_tolower: $(lib_o) sclite_tolower.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+$(TEST_PROGRAMS): $(lib_o) $$@.o
+	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ $(LIBS)
+$(PURIFY_PROGRAMS): $(lib_o)
+	purify $(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ `echo $@.c|sed 's/_pure//'` $(LIBS)
+$(PURECOV_PROGRAMS): $(lib_o)
+	purecov $(CC) -o $@ $(CFLAGS) $(LDFLAGS) $^ `echo $@.c|sed 's/_purecov//'` $(LIBS)
+
+### make a rule to compile the SLM toolkit
+slm_v2/lib/SLM2.a:
+	(cd slm_v2/src && make SLM2.a install-lib)
+
+#arpg: rpg.o arpg.o pad.o 
+#	$(CC) -o $@ $(CFLAGS) $(LDFLAGS) rpg.o arpg.o pad.o $(LIBS)
+clean:
+	rm -f *.o $(PROGRAMS) core
+	if test -f slm_v2/src/makefile ; then \
+		(cd slm_v2/src && make spotless) \
+	fi
+
+mostlyclean: clean
+
+distclean: clean
+	rm -rf *~ config.status config.log config.cache config.h makefile
+
+realclean: distclean
+	rm -f TAGS *.info*
+
+install: all testinstalldirs
+	set -e; for p in $(PROGRAMS); do \
+	  $(INSTALL) $$p $(DESTDIR)$(bindir)/$$p; \
+	done
+#REenable this later
+#	set -e; for p in `(cd ../doc ; ls *.1)`; do \
+#	  $(INSTALL) ../doc/$$p $(DESTDIR)$(man1dir)/$$p; \
+#	done
+#	@echo "!!!!  If 'man sclite' does not work, run 'catman -M $(mandir)'"
+#	@echo "!!!!  to make manual pages accessible."
+
+testinstalldirs:
+	@-set -e; for p in $(DESTDIR)$(bindir) $(DESTDIR)$(man1dir) $(DESTDIR)$(man5dir); do \
+	   if test ! -d $$p ; then \
+		echo "Error: directory $$p doesn't exist"; \
+	   fi ; \
+	done
+#REenable this later
+#	   if test ! -w $$p ; then \
+#		echo "Error: directory $$p is not writeable by you"; \
+#	   fi \
+
+# We need more tests.
+check: sctkUnit
+	./sctkUnit
+	cd testdata && ./tsclite.sh
+	cd testdata && ./tsc_stats.sh
+
+uninstall:
+	set -e; for p in $(PROGRAMS); do \
+	  rm -f $(bindir)/`echo $$p | $(edit_program_name)`; \
+	done
+	set -e; for p in `(cd ../doc ; ls *.1)`; do \
+	  rm -f $(man1dir)/$$p; \
+	done
+
+config.sh: config.in
+	cd $(srcdir) && /usr/local/bin/autoconf config.in | sed 's/Makefile/makefile/g' > config.sh
+
+# autoheader might not change config.hin.
+config.hin: stamp-h.in
+stamp-h.in: config.in
+	cd $(srcdir) && autoheader
+	date > $(srcdir)/stamp-h.in
+
+config.status: config.sh
+	./config.status --recheck
+
+# config.status might not change config.h, but it changes stamp-h.
+makefile: makefile.in config.status
+	./config.status
+
+dist: $(distribution)
+
+tk_dist: $(tk_distfiles)
+	echo sctk-`grep define\ TK_VERSION sctk.h | sed -e 's/^[^"]*"//' -e 's/"[^"]*//'` > .fname
+	echo "Distribution file '../`cat .fname`.tar.Z'"
+	(cd ../doc && ./export.sh $(PROGRAMS))
+	rm -rf ../`cat .fname`
+	mkdir ../`cat .fname`
+	(cd ./testdata && tsclite.sh -clean) ;
+	set -e; for file in $(DOCS); do \
+		ln -s $$file ../`cat .fname`; \
+	done
+	mkdir ../`cat .fname`/src
+	set -e; for file in $(tk_distfiles); do \
+		ln -s ../../src/$$file ../`cat .fname`/src/$$file; \
+	done
+	echo "Installing SLM links"
+	mkdir ../`cat .fname`/src/slm_v2
+	mkdir ../`cat .fname`/src/slm_v2/lib
+	mkdir ../`cat .fname`/src/slm_v2/bin
+	mkdir ../`cat .fname`/src/slm_v2/src
+	set -e; for file in readme.txt endian.sh include doc; do \
+		ln -s ../../../src/slm_v2/$$file ../`cat .fname`/src/slm_v2/$$file; \
+	done
+	echo "    Copying SLM source"
+	tar cf - `find slm_v2/src \( -name makefile.in -o -name install-sh -o -name "*.[ch]" \) -print` | (cd ../`cat .fname`/src && tar xf -)
+	echo "Tar-ing SLM to make it ISO-compatable"
+	(cd ../`cat .fname`/src && tar chf slm_v2.tar ./slm_v2 && rm -r slm_v2)
+	(cd .. && tar -chf - --exclude=CVS `cat src/.fname` | compress > `cat src/.fname`.tar.Z)
+	rm -rf ../`cat .fname`
+	echo "Decompressing the release file"
+	(cd .. && zcat `cat src/.fname`.tar.Z | tar xf -)
+	echo "ISO_checking.   . . . Only the top-level makefile should fail"
+	(cd .. && iso_check `cat src/.fname`)
+	rm -rf ../`cat .fname` .fname
+
+# Prevent GNU make v3 from overflowing arg limit on SysV.
+.NOEXPORT:

SCTK/src/sclite/marc1.c

@@ -0,0 +1,25 @@
+/* file marc1.c */
+
+
+#include "sctk.h"
+
+ARC *make_arc(void *dat, NODE *from_node, NODE *to_node, int *perr)
+                                                   
+/*********************************************************/
+/* Makes an arc <from_node/symbol/to_node>, returning a  */
+/* pointer to it.                                        */
+/*********************************************************/
+ {char *proc = "make_arc";
+  ARC *p; 
+/* code */
+  db_enter_msg(proc,1); /* debug only */
+  p = (ARC *)malloc_safe(sizeof(ARC),proc);
+  *perr = 0;
+  p->data = dat;
+  p->weight = 0;
+  p->net = NULL_NETWORK;
+  p->from_node = from_node;
+  p->to_node   = to_node;
+  db_leave_msg(proc,1); /* debug only */
+  return p;
+ } /* end make_arc() */

SCTK/src/sclite/mcnemar.c

@@ -0,0 +1,18 @@
+#define MAIN
+#include "sctk.h"
+
+int main(int argc, char **argv){
+  int **table;
+
+  alloc_2dimarr(table, 2, 2, int);
+  table[0][0] = 100;
+  table[0][1] = 200;
+  table[1][0] = 300;
+  table[1][1] = 400;
+
+  double conf;
+  do_McNemar(table, "Sys1", "Sys2", TRUE, stdout, &conf);
+
+
+
+}

SCTK/src/sclite/mfa.c

@@ -0,0 +1,299 @@
+#include <sctk.h>
+
+void locate_next_file_channel(WTOKE_STR1 **ctms, int nctms, FILE **files, char **hypname, int *eofs, int *conv_end, int case_sense, int feedback);
+void print_linear(NODE *node, void *p);
+int gap_at_times(WTOKE_STR1 *ctm, int end, double *mgap_beg, double *mgap_end, int *mgap_s);
+NETWORK *perform_mfalign(WTOKE_STR1 **ctms, int nctm, int *sil_end, int time_align);
+void populate_tag1(NODE *node, void *p);
+
+int mfdb = 0;
+int glob_ties = 0;
+
+void mfalign_ctm_files(char **hypname, int nhyps, int time_align, int case_sense, int feedback, void (*callback)(NETWORK *, char *, char *), double silence_dur){
+    char *proc = "mfalign_ctm_files";
+    WTOKE_STR1 **ctms;
+    NETWORK *mfnet;
+    FILE **files;
+    int *eofs, *conv_end, *sil_end;
+    int in;
+    int done;
+    NETWORK  *cor;
+    char *file, *chan;
+
+
+    {if (getenv("MFALIGN_DBG") != NULL) {
+	mfdb=(int)atof(getenv("MFALIGN_DBG")); }}
+
+    if (feedback > 1) printf("Beginning: %s\n",proc);
+    /* ALLOCATE MEMORY */
+    alloc_singZ(ctms,nhyps,WTOKE_STR1 *,(WTOKE_STR1 *)0);
+    alloc_singZ(files,nhyps,FILE *,(FILE *)0);
+    alloc_singZ(eofs,nhyps,int,0);
+    alloc_singZ(conv_end,nhyps,int,0);
+    alloc_singZ(sil_end,nhyps,int,0);
+  
+    /* OPEN FILES, INIT WTOKE'S, FILE WTOKE'S */
+    for (in=0; in < nhyps; in++){
+	if ((files[in] = fopen(hypname[in],"r")) == NULL){
+	    fprintf(stderr,"Can't open input hypothesis file %s\n",hypname[in]);
+	    exit(1);
+	}
+	ctms[in] = WTOKE_STR1_init(hypname[in]);
+    }
+    done = 0;
+    while (!done){
+        locate_next_file_channel(ctms, nhyps, files, hypname, eofs, conv_end,
+				 case_sense, feedback);
+	
+	file = ctms[0]->word[ctms[0]->s].conv;
+	chan = ctms[0]->word[ctms[0]->s].turn;
+
+	/* loop though the file/channel section, looking for time breaks */
+	while(find_common_silence(ctms, nhyps, conv_end,
+				  sil_end, silence_dur) == 1){
+	    if (mfdb >= 5){
+		printf("--------- Aligning this chunk -----------\n");
+
+		for (in=0; in<nhyps; in++){
+		    double beg, end;
+		    beg = (ctms[in]->s > conv_end[in]) ? -1.0 : 
+			ctms[in]->word[sil_end[in]].t1 +
+			ctms[in]->word[sil_end[in]].dur;
+		    end = (sil_end[in] < conv_end[in]) ? 
+		    ctms[in]->word[sil_end[in]+1].t1 : 9999999.9;
+		    printf("   CTM %d: start: %d end_word: %d start: %.2f"
+			   " end: %.2f\n",in,ctms[in]->s,sil_end[in],beg,end);
+		}
+	    }
+
+	    mfnet = perform_mfalign(ctms,nhyps,sil_end,time_align);
+	    Network_traverse(mfnet,populate_tag1,(void *)hypname,0,0,0);
+	    cor = Network_WORD_to_CORES(mfnet);
+
+	    callback(cor, file, chan);
+
+	    Network_destroy(cor);
+	    Network_destroy(mfnet);
+
+	    /* skip over this chunk because we're done with it. */
+	    done = 1;
+	    for (in=0; in < nhyps; in++){
+		ctms[in]->s = sil_end[in] + 
+		    ((sil_end[in] <= conv_end[in]) ? 1 : 0);
+		if (!eofs[in] || (ctms[in]->s < ctms[in]->n)) done = 0;
+	    }
+	}
+    }
+    if (glob_ties > 0)
+	fprintf(stderr,"Warning: %d ties were arbitrarily broken\n",glob_ties);
+  
+    /* FREE THE MEMORY */
+    free_singarr(eofs, int);
+    free_singarr(conv_end, int);
+    free_singarr(sil_end, int);
+    for (in=0; in<nhyps; in++) {
+	free_mark_file(ctms[in]);
+	fclose(files[in]);
+    }
+    free_singarr(files, FILE *);
+    free_singarr(ctms,WTOKE_STR1 *);
+    cleanup_NET_ALIGN();
+}
+
+NETWORK *perform_mfalign(WTOKE_STR1 **ctms, int nctm, int *end, int time_align){
+    char *proc = "perform_mfalign";
+    NETWORK **nets, *out_net;
+    WORD *null_alt;
+    int in;
+
+    alloc_singZ(nets,nctm,NETWORK *,(NETWORK *)0);
+    null_alt = new_WORD((TEXT *)"@",-1,0.0,0.0,0.0,(TEXT *)0,(TEXT *)0,0,0,-1.0);
+     
+    /* create the networks */
+    for (in=0; in < nctm; in++){
+	if ((nets[in]=
+	     Network_create_from_WTOKE(ctms[in],ctms[in]->s,end[in],
+				       rsprintf("mfalign net %d",in),
+				       print_WORD_wt,
+				       equal_WORD2,
+				       release_WORD, null_alt_WORD,
+				       opt_del_WORD,
+				       copy_WORD, make_empty_WORD,
+				       use_count_WORD,
+				       1))
+	    == NULL_NETWORK){ 
+	    fprintf(stderr,"%s: Network_create_from_WTOKE failed\n",proc);
+	    exit(1);
+	}
+	/*	Network_traverse(nets[in],0,0,set_tag1,(void *)ctms[in]->id,0); */
+    }
+    
+    /* align the networks */
+    Network_dpalign_n_networks(nets,nctm,(!time_align)?wwd_WORD_rover:wwd_time_WORD,
+			       &out_net,(void *)null_alt);
+    
+    /* delete the networks */
+    for (in=0; in < nctm; in++)
+	Network_destroy(nets[in]);
+
+    free_singarr(nets,NETWORK *);
+    release_WORD(null_alt);
+
+    return(out_net);
+}
+
+    
+void locate_next_file_channel(WTOKE_STR1 **ctms, int nctm, FILE **files, char **hypname, int *eofs, int *conv_end, int case_sense, int feedback){
+    int in;
+
+    /* locate the file boundaries */
+    for (in=0; in < nctm; in++){ 
+	fill_WTOKE_structure(ctms[in], files[in], hypname[in],
+			     &(eofs[in]), case_sense);
+	locate_boundary(ctms[in], ctms[in]->s, TRUE,TRUE,&(conv_end[in]));
+	if (in >= 1 && ctms[in]->s <= ctms[in]->n)
+	    if (strcmp(ctms[0]->word[ctms[0]->s].conv,
+		       ctms[in]->word[ctms[in]->s].conv) != 0){
+		fprintf(scfp,"Error: Conversation mismatches in %s and %s, "
+			"%s and %s respectively",hypname[0],hypname[in],
+			ctms[0]->word[ctms[0]->s].conv,
+			ctms[in]->word[ctms[in]->s].conv);
+		exit(1);
+	    }
+    }
+    if (feedback >= 1) {
+	printf("Working on conversation: %s channel: %s\n",	
+	       ctms[0]->word[ctms[0]->s].conv,
+	       ctms[0]->word[ctms[0]->s].turn);
+	for (in=0; in < nctm; in++)
+	    printf("   File: %s  range %d-%d\n",hypname[in],ctms[in]->s,
+		   conv_end[in]);
+    }
+}
+
+/* This function looks for common periods of silence to all the  WTOKE
+ * structures 
+ */
+int find_common_silence(WTOKE_STR1 **ctms, int nctm, int *ctm_end, int *sil_end, double silence_dur){
+    double mgap_beg, mgap_end;
+    int nc, failed, more_data;
+    int dbg = 0;
+
+    {if (getenv("MFALIGN_COMMON_SIL_DBG") != NULL) {
+      dbg=(int)atof(getenv("MFALIGN_COMMON_SIL_DBG")); }}
+
+    /* locate a gap in the master, ctms[0] */
+    sil_end[0] = ctms[0]->s;
+    while (sil_end[0] <= ctm_end[0]){
+	/* set the gap betwen this word and the next */
+	mgap_beg = ctms[0]->word[sil_end[0]].t1 +ctms[0]->word[sil_end[0]].dur;
+	/* if there is no next word, make it the end of time */
+	mgap_end = (sil_end[0] < ctm_end[0]) ? ctms[0]->word[sil_end[0]+ 1].t1:
+	    9999999.9 ;
+
+	if (mgap_end > mgap_beg + silence_dur){
+	    if (dbg) printf("Found a gap %d (%.2f-%.2f) = %.2f\n",
+			    sil_end[0],mgap_beg,mgap_end,mgap_end-mgap_beg);
+	    for (nc=1, failed=0; nc<nctm && !failed; nc++){
+		if (gap_at_times(ctms[nc],ctm_end[nc],
+			 &mgap_beg,&mgap_end,&sil_end[nc]) == 1){
+		    if (dbg) printf("  Gap in %d\n",nc);
+		} else {
+		    failed = 1;
+		}
+	    }
+	    if (failed != 1){
+		if (dbg) printf("Segment defined at:\n");
+		return(1);
+	    }
+	}
+	sil_end[0] ++;
+    }
+    /* ctm[0] is at the end of the road, move all the other ctms to the 
+     * end as well.  If there is data left in them, return 1 
+     */
+    if (dbg) printf("*** ctm[0] has ran out of data\n");
+    more_data = 0;
+    for (nc=0; nc<nctm; nc++){
+	sil_end[nc] = ctm_end[nc];
+	if (ctms[nc]->s <= ctm_end[nc]) more_data=1;
+    }
+    return (more_data);
+}
+
+void populate_tag1(NODE *node, void *p){
+  char **tags = (char **)p;
+  int Narc, f, *use, arc;
+  ARC_LIST_ATOM *parc;
+
+  if (node != NULL && strcmp(node->name,"STOP") != 0){
+    /* count the arcs */
+    for (Narc=0, parc=node->out_arcs; parc != (ARC_LIST_ATOM *)0;
+	 parc = parc->next)
+      Narc++;
+    alloc_singZ(use,Narc,int,-1);
+    /* map the usage */
+    for (arc = 0, parc=node->out_arcs; parc != (ARC_LIST_ATOM *)0;
+	 parc = parc->next, arc++){
+      /* Eprintf("   %s\n",((WORD*)(parc->arc->data))->value); */
+      if (TEXT_strcmp(((WORD*)(parc->arc->data))->value,(TEXT *)"@") != 0){
+	/* reset that systems use pointer */
+	for (f=0; f<Narc; f++){
+	  if (TEXT_strcmp(((WORD*)(parc->arc->data))->tag1,(TEXT*)tags[f]) == 0)
+	    use[arc] = f;
+	}
+      }
+    }
+    /* dump_singarr(use,Narc," %d",stdout); */
+    for (arc = 0, parc=node->out_arcs; parc != (ARC_LIST_ATOM *)0;
+	 parc = parc->next, arc++)
+      if (use[arc] < 0){
+	int hit = -1, y;
+	/* printf("   Arc %d not tagged\n",arc); */
+	for (f=0; f<Narc && hit < 0; f++){
+	  hit = f;
+	  /* printf("     is %d used? \n",f); */ 
+	  /* does any  of the use elements == f */
+	  for (y=0; y<Narc; y++)
+	    if (use[y] == f)
+	      hit = -1;
+	  /* printf("        %d\n",hit); */
+	}
+	if (hit >= 0){
+	  ((WORD*)(parc->arc->data))->tag1 = TEXT_strdup((TEXT*)tags[hit]);
+	  use[arc] = hit;
+	}
+      }
+    /* printf("   "); dump_singarr(use,Narc," %d",stdout); */
+    
+    free_singarr(use,int);
+  }
+  
+}
+
+int gap_at_times(WTOKE_STR1 *ctm, int end, double *mgap_beg, double *mgap_end, int *mgap_s){
+    int i;
+    double gap_end, gap_beg;
+    
+    /* if there is no more data in this ctm, that answer is YES */
+    if (ctm->s > end) {
+	*mgap_s = ctm->s;
+	return(1);
+    }
+
+    for (i=ctm->s; i<=end; i++){
+	gap_beg = ctm->word[i].t1 + ctm->word[i].dur;
+	gap_end = (i < end) ? ctm->word[i+1].t1 : 9999999.9;
+	if (overlap(*mgap_beg,*mgap_end,gap_beg,gap_end) > 0.0){
+	    *mgap_s = i;
+	    /* printf("   Gap@ %d times %.2f to %.2f\n",
+	       i,gap_beg,gap_end); */
+	    *mgap_beg = MAX(*mgap_beg,gap_beg);
+	    *mgap_end = MIN(*mgap_end,gap_end);
+	    /* printf("   Reset gap times to %.2f-%.2f\n",
+	     *mgap_beg,*mgap_end); */
+	    return (1);
+	}
+    }
+    return(0);
+}

SCTK/src/sclite/mnode1.c

@@ -0,0 +1,55 @@
+/* file mnode1.c */
+
+#include "sctk.h"
+
+NODE *make_node(char *name, NETWORK *net, ARC *from_arc, ARC *to_arc, int *highest_nnode_name, int *perr)
+
+/************************************************************/
+/*  Makes a new node, returns a pointer to it.              */
+/* The new node is initialized with name, from_arc, to_arc. */
+/* Set to be neither start not stop state.                  */
+/* All boolean flags are initialized to F.                  */
+/* If name is NULL, uses a char string version of bumped    */
+/* *highest_nnode_name for the new node name.               */
+/************************************************************/
+ {char *proc = "make_node";
+  char sxx[LINE_LENGTH], *sx = &sxx[0];
+  NODE *p;
+/* code */
+  db_enter_msg(proc,1); /* debug only */
+  p = (NODE *)malloc_safe(sizeof(NODE),proc);
+  *perr = 0;
+  if (name != NULL)
+    {p->name = strdup_safe(name,proc);
+    }
+  else
+    {/* make name from node number */
+     *highest_nnode_name += 1;
+     sprintf(sx,"%d",*highest_nnode_name);
+     p->name = strdup_safe(sx,proc);
+    }
+  p->net      = net ;
+  p->in_arcs  = NULL;
+  p->out_arcs = NULL;
+  p->start_state = F;
+  p->stop_state  = F;
+  p->flag1       = F;
+  p->flag2       = F;
+  if (from_arc != NULL){
+      p->in_arcs  = add_to_arc_list(p->in_arcs,from_arc,perr);
+      if (*perr > 0) {
+	  printf("%s:*ERR: add_to_arc_list() returns %d\n",proc,*perr);
+	  return((NODE *)0);
+      }
+  }
+  if (to_arc != NULL) {
+      p->out_arcs = add_to_arc_list(p->out_arcs,to_arc,perr);
+      if (*perr > 0) {
+	  printf("%s:*ERR: add_to_arc_list() returns %d\n",proc,*perr);
+	  return((NODE *)0);
+      }
+  }
+db_leave_msg(proc,1); /* debug only */
+  return p;
+ } /* end make_node() */
+

SCTK/src/sclite/mtchprs.c

@@ -0,0 +1,833 @@
+/**********************************************************************/
+/*                                                                    */
+/*             FILENAME:  matched_pairs.c                             */
+/*             BY:  Jonathan G. Fiscus                                */
+/*             DATE: April 14 1989                                    */
+/*                   NATIONAL INSTITUTE OF STANDARDS AND TECHNOLOGY   */
+/*                   SPEECH RECOGNITION GROUP                         */
+/*                                                                    */
+/*             DESCRIPTION:  These programs perform the Matched Pairs */
+/*                           tests. which include:                    */
+/*                                  SEGMENTATION ANALYSIS             */
+/*                                  MATCHED PAIRS TEST FOR SYSTEMS    */
+/*                                                                    */
+/**********************************************************************/
+
+
+#include "sctk.h"
+
+/**********************************************************************/
+/*   structures and defines for the sentence segment list             */
+#define MIN_NUM_GOOD		2
+#define MAX_SEG_ANAL		10
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct segment_count_struct{
+    int number_ref;
+    int number_errors_for_hyp1;
+    int number_errors_for_hyp2;
+} SEG;
+
+#ifdef __cplusplus
+}
+#endif
+
+/* alloc a segment list */
+#define alloc_SEG_LIST(_seg,_num) \
+   alloc_2dimarr(_seg,_num,1,SEG);
+
+#define expand_SEG_LIST(_seg_l,_num,_max) \
+{   int _l;\
+    expand_1dim(_seg_l,_num,_max,2.0,SEG *,1); \
+    for (_l=_num; _l < _max; _l++) \
+      alloc_singarr((_seg_l)[_l],1,SEG); \
+}
+
+#define free_SEG_LIST(_seg,_num)   free_2dimarr(_seg,_num,SEG)
+
+
+/*  macros to access the segment list */
+#define seg_n(_sg,_s)		_sg[_s]
+#define seg_ref(_sg,_s)		_sg[_s]->number_ref
+#define seg_hyp1(_sg,_s)	_sg[_s]->number_errors_for_hyp1
+#define seg_hyp2(_sg,_s)	_sg[_s]->number_errors_for_hyp2
+
+
+void evaluate_SEG(char *sys1_str, char *sys2_str, SEG **seg_l, int *num_seg, int sent_cnt, int min_good, double *s1_pct, double *s2_pct, double *seg_per_sent, int verbose, FILE *fp);
+int analyze_Z_score(SCORES *sys1, SCORES *sys2, double mean, double variance, double std_dev, double Z_stat, int verbose);
+void test_comp_sents(PATH *snt1, PATH *snt2, SEG ***seg_l, int *num_seg, int *max_seg, int min_good);
+void count_seg(PATH *snt1, PATH *snt2, int beg1, int beg2, int end1, int end2, SEG ***seg_l, int *num_seg, int *max_seg, int dbg);
+void seg_count(PATH *snt, int from, int to, int *err, int *ref);
+void print_sent_seg_averaged_analysis(SCORES *scor[], int nscor, double ***seg_per_sent, char *tname);
+void print_sent_seg_long_analysis(SCORES *scor[], int nscor, double ***seg_per_sent, char *tname);
+void print_compare_matrix_for_sys(SCORES *scor[], int nscor, int **winner, char *matrix_name, char *tname, char *v_desc, double **sys1_pct_arr, double **sys2_pct_arr, int **num_seg, double **Z_stat, int min_num_good, FILE *fp);
+
+/**********************************************************************/
+/*   this procedure is written to analyze the segmentation for        */
+/*   varying numbers on the minimum number of good words to bound     */
+/*   the beginning and the ends of the segments                       */
+/**********************************************************************/
+void do_mtch_pairs_seg_analysis(SCORES *scor[], int nscor, char *t_name, int seg_ave, int seg_long)
+{
+    int sys1, sys2, min_g;
+    int result;
+    double conf;
+    int **num_seg, **max_seg;
+    double **sys1_pct_arr, **sys2_pct_arr, **Z_stat_arr, ***seg_per_sent;
+
+    /* do all allocations */
+    fprintf(stderr,"Doing segmentation analysis. This is slow so . . . \n\n");
+    alloc_2dimarr(num_seg,nscor,nscor,int);
+    alloc_2dimZ(max_seg,nscor,nscor,int,100);
+    alloc_2dimarr(sys1_pct_arr,nscor,nscor,double);
+    alloc_2dimarr(sys2_pct_arr,nscor,nscor,double);
+    alloc_2dimarr(Z_stat_arr,nscor,nscor,double);
+    alloc_3dimZ(seg_per_sent,MAX_SEG_ANAL,nscor,nscor,double,0.0);
+
+    /* loop for min_good, and compared systems */
+    for (min_g=0; min_g < MAX_SEG_ANAL; min_g++){
+        fprintf(stderr,"   calculating buffer of  %d\n",min_g);
+        for (sys1=0; sys1 <(nscor-1); sys1++)
+            for (sys2=sys1+1; sys2<(nscor); sys2++){
+                result = do_mtch_pairs_on_sys(scor, nscor,sys1,sys2,
+                                              &(sys1_pct_arr[sys1][sys2]),
+                                              &(sys2_pct_arr[sys1][sys2]),
+                                              &(num_seg[sys1][sys2]),
+                                              &(max_seg[sys1][sys2]),
+                                              &(Z_stat_arr[sys1][sys2]),
+                                           &(seg_per_sent[min_g][sys1][sys2]),
+					      min_g+1,FALSE,stdout,&conf);
+        }
+    }
+
+    /* if the flag, print the requested analysis */
+    if (seg_long)
+        print_sent_seg_long_analysis(scor,nscor,seg_per_sent,t_name);
+    if (seg_ave)
+        print_sent_seg_averaged_analysis(scor,nscor,seg_per_sent,t_name);
+
+    free_2dimarr(num_seg,nscor,int);
+    free_2dimarr(max_seg,nscor,int);
+    free_2dimarr(sys1_pct_arr,nscor,double);
+    free_2dimarr(sys2_pct_arr,nscor,double);
+    free_2dimarr(Z_stat_arr,nscor,double);
+    free_3dimarr(seg_per_sent,MAX_SEG_ANAL,nscor,double);
+}
+
+
+/**********************************************************************/
+/*   this procedure performs a single matched paired analysis for the */
+/*   one level of the min_good and returns the results in the winner  */
+/*   array                                                            */
+/**********************************************************************/
+void do_mtch_pairs(SCORES *scor[], int nscor, char *min_num_good_str, char *test_name, int print_report, int verbose, int ***out_winner, char *outroot, int feedback, double ***out_conf)
+{
+    int sys1, sys2;
+    int result, **winner;
+    double **conf;
+    int *sum_winner, **num_seg, min_num_good, **max_seg;
+    double **sys1_pct_arr, **sys2_pct_arr, **Z_stat_arr, **seg_per_sent;
+    char *matrix_name = "COMPARISON MATRIX: FOR THE MATCHED PAIRS TEST";
+    char *mean_desc = "PECENTAGES ARE MEAN PCT ERROR/SEGMENT.  FIRST # IS LEFT SYSTEM";
+    FILE *fp = stdout;
+    
+    if (print_report || verbose){
+	char *f = rsprintf("%s.mapsswe",outroot);
+	if ((fp=(strcmp(outroot,"-") == 0) ? stdout : fopen(f,"w")) ==
+	    (FILE *)0){
+	    fprintf(stderr,"Warning: Open of %s for write failed.  "
+		           "Using stdout instead.\n",f);
+	    fp = stdout;
+	} else
+	    if (feedback >= 1) printf("        Output written to '%s'\n",f);
+    }
+    if ((min_num_good = atoi(min_num_good_str)) <= 0){
+        fprintf(stderr,"Warning: Minimum seperation by correct words is \n");
+        fprintf(stderr,"         too low, setting to %d\n",
+                       DEFAULT_MIN_NUM_GOOD);
+        min_num_good = DEFAULT_MIN_NUM_GOOD;
+    }
+    if (!print_report) verbose = FALSE;
+
+    /* all allocations */
+    alloc_2dimarr(num_seg,nscor,nscor,int);
+    alloc_2dimZ(max_seg,nscor,nscor,int,100);
+    alloc_2dimarr(sys1_pct_arr,nscor,nscor,double);
+    alloc_2dimarr(sys2_pct_arr,nscor,nscor,double);
+    alloc_2dimarr(Z_stat_arr,nscor,nscor,double);
+    alloc_2dimZ(seg_per_sent,nscor,nscor,double,0.0);
+    alloc_singZ(sum_winner,nscor,int,NO_DIFF);
+    alloc_2dimZ(winner,nscor,nscor,int,NO_DIFF);
+    *out_winner = winner;
+    alloc_2dimZ(conf,nscor,nscor,double,0.0);
+    *out_conf = conf;
+
+    /* loop each pair of compared systems */
+    for (sys1=0; sys1 <(nscor-1); sys1++)
+        for (sys2=sys1+1; sys2<(nscor); sys2++){
+            result = do_mtch_pairs_on_sys(scor,nscor,sys1,sys2,
+                                          &(sys1_pct_arr[sys1][sys2]),
+                                          &(sys2_pct_arr[sys1][sys2]),
+                                          &(num_seg[sys1][sys2]),
+                                          &(max_seg[sys1][sys2]),
+                                          &(Z_stat_arr[sys1][sys2]),
+                                          &(seg_per_sent[sys1][sys2]),
+                                          min_num_good,verbose,fp,
+					  &(conf[sys1][sys2]));
+            winner[sys1][sys2] = result;
+            sum_winner[sys2] += result * (-1);
+            sum_winner[sys1] += result;
+	  }
+
+    /* if print_report print the requested analysis */
+    if (print_report){
+	if (verbose) form_feed(fp);
+	print_compare_matrix_for_sys(scor,nscor,winner,matrix_name,test_name,
+				     mean_desc,sys1_pct_arr,sys2_pct_arr,
+				     num_seg,Z_stat_arr,min_num_good,fp);
+	if (fp == stdout) form_feed(fp);
+    }
+
+    free_2dimarr(sys1_pct_arr,nscor,double);
+    free_2dimarr(sys2_pct_arr,nscor,double);
+    free_2dimarr(Z_stat_arr,nscor,double);
+    free_2dimarr(seg_per_sent,nscor,double);
+    free_singarr(sum_winner,int);
+    free_2dimarr(num_seg,nscor,int);
+    free_2dimarr(max_seg,nscor,int);
+    /* Big Error     free_2dimarr(conf,nscor,double); */
+
+    if (fp != stdout) fclose(fp);
+}
+
+/**********************************************************************/
+/*   the guts of the Matched Pairs program,  this procedure compares  */
+/*   2 systems in a SYS_ALIGN_LIST with indexes sys1_ind and sys2_ind */
+/*   then goes through each matching sentence then calls another      */
+/*   procedure to find the segments and store them in the segment     */
+/*   list. after all sentences are compared, the segment list is      */
+/*   analyzed                                                         */
+/**********************************************************************/
+int do_mtch_pairs_on_sys(SCORES *scor[], int nscor, int sys1_ind, int sys2_ind, double *sys1_pct, double *sys2_pct, int *num_seg, int *max_seg, double *Z_stat, double *seg_per_sent, int min_num_good, int verbose,FILE *fp, double *conf)
+{
+    int spk1, spk2, snt1, snt2, i, sent_cnt=0, *err_diff;
+    double mean, std_dev, variance, median;
+    SCORES *sys1, *sys2;
+    SEG **seg_l;
+
+    *num_seg = 0;
+    sys1 = scor[sys1_ind];
+    sys2 = scor[sys2_ind];
+
+    alloc_SEG_LIST(seg_l,*max_seg);
+
+    for (spk1=0;spk1 < sys1->num_grp; spk1++){ /* for all speaker sys1 */
+        /**** find the matching speaker */
+        for (spk2=0;spk2 < sys2->num_grp; spk2++)
+            if (strcmp(sys1->grp[spk1].name, sys2->grp[spk2].name) == 0)
+                break;
+        /**** the the speakers match, start on the sentences */
+        if (spk2 != sys2->num_grp){
+            /**** for all sents in sys1,spkr1 */
+            for (snt1 = 0; snt1 < sys1->grp[spk1].num_path; snt1++){
+                /**** for all sents in sys2,spkr2 */
+                for (snt2 = 0; snt2 < sys2->grp[spk2].num_path; snt2++) 
+                   /**** if the sentences are the same, compare them */
+                   if(strcmp(sys1->grp[spk1].path[snt1]->id,
+                             sys2->grp[spk2].path[snt2]->id) == 0){
+                        test_comp_sents(sys1->grp[spk1].path[snt1],
+					sys2->grp[spk2].path[snt2],
+					&seg_l,num_seg, max_seg, min_num_good);
+                        sent_cnt++;
+                        break;
+		   }
+	    }
+	} else {
+            fprintf(stderr,"Warning: Speaker %s is in system %s but not system %s\n",
+		    sys1->grp[spk1].name,sys1->title,sys2->title);
+        }
+    }
+    /* calculate the percentages of errors for both systems and print */
+    /* a report if verbose == TRUE                                    */
+    evaluate_SEG(sys1->title, sys2->title,seg_l,num_seg,sent_cnt,min_num_good,
+		 sys1_pct,sys2_pct,seg_per_sent,verbose,fp);
+
+    alloc_singZ(err_diff,*max_seg,int,0);
+
+    /* calc an arr with the difference in the errors for each segment */
+    for (i=0;i<*num_seg;i++)
+        err_diff[i] = seg_hyp1(seg_l,i) - seg_hyp2(seg_l,i);
+
+    calc_mean_var_std_dev_Zstat(err_diff,*num_seg, 
+                                &mean,&variance,&std_dev,&median,Z_stat);
+    /* Special output routine */
+    if (verbose){  
+        fprintf(fp,"MTCH_PR_RESULTS (systems: %s %s) (# segs: %d) "
+		"(Seg per Sent: %5.3f) ",
+	       sys1->title,sys2->title,*num_seg,
+	       *seg_per_sent);
+        fprintf(fp,"(%% Error: %5.2f %5.2f) (mean: %5.3f) (std dev: %5.3f) "
+		"(Z Stat: %5.3f) (Stat Diff: %s)\n",
+	       *sys1_pct,*sys2_pct,mean,std_dev,*Z_stat,
+	       (analyze_Z_score(sys1,sys2,mean,variance,std_dev,
+				*Z_stat,SILENT) == NO_DIFF) ? "No" : "Yes"); 
+    }
+
+    free_singarr(err_diff,int);
+    free_SEG_LIST(seg_l,*max_seg);
+    *conf = 1.0 - (2.0 * (normprob((double) (fabs(*Z_stat))) - 0.50));
+
+    return(analyze_Z_score(sys1,sys2,mean,variance,std_dev,*Z_stat,SILENT));
+}
+
+/**********************************************************************/
+/*   this procedure calculates the percentages of errors for each     */
+/*   systems segments,  if verbose == TRUE then a report is printed   */
+/*   to stdout for looking at segment counts                          */
+/**********************************************************************/
+void evaluate_SEG(char *sys1_str, char *sys2_str, SEG **seg_l, int *num_seg, int sent_cnt, int min_good, double *s1_pct, double *s2_pct, double *seg_per_sent, int verbose,FILE *fp)
+{
+    int i, tot_ref=0, tot_err_h1=0, tot_err_h2=0;
+
+    /* calc the pcts */   
+    for (i=0;i<*num_seg;i++){
+        tot_ref += seg_ref(seg_l,i);
+        tot_err_h1 += seg_hyp1(seg_l,i);
+        tot_err_h2 += seg_hyp2(seg_l,i);
+    }
+    (*s1_pct) = pct(tot_err_h1,tot_ref);
+    (*s2_pct) = pct(tot_err_h2,tot_ref);
+    (*seg_per_sent) = (double)(*num_seg) / (double)sent_cnt;
+
+    /* if needed, print the systems segmentation reports */
+    if (verbose){
+      fprintf(fp,"      SEGMENTATION REPORT FOR SYSTEMS\n");
+      fprintf(fp,"              %4s and %4s\n",sys1_str, sys2_str);
+      fprintf(fp,"Minimum Number of Correct Boundary words %1d\n\n",min_good);
+      fprintf(fp,"  Number of Segments %2d,     %5.3f per sentence\n",*num_seg,
+                              (*seg_per_sent));
+      fprintf(fp,"  Number of Sentences %2d\n\n",sent_cnt);
+      fprintf(fp,"    # Ref wrds      Err %4s   Err %4s\n",sys1_str, sys2_str);
+      fprintf(fp,"---------------------------------------\n");
+      if (FALSE){
+          for (i=0;i<*num_seg;i++)
+              fprintf(fp,"       %4d           %4d        %4d\n",
+                  seg_ref(seg_l,i),seg_hyp1(seg_l,i),seg_hyp2(seg_l,i));
+          fprintf(fp,"---------------------------------------\n");
+      }
+      fprintf(fp,"Totals %4d           %4d        %4d\n\n",
+                 tot_ref,tot_err_h1,tot_err_h2);
+      fprintf(fp,"Pct err              %5.1f%%      %5.1f%%\n\n",
+                 *s1_pct,*s2_pct);
+      fprintf(fp,"\n\n\n");
+
+      /* do a McNermar-style anaylsys */ 
+      /* first calc the matrix */
+      { 
+	int **table=NULL, ans; double mcn_conf=0.0;
+	alloc_2dimZ(table,2,2,int,0);
+	for (i=0;i<*num_seg;i++){
+	  if (seg_hyp1(seg_l,i) == 0 && seg_hyp2(seg_l,i) == 0 &&
+	      seg_ref(seg_l,i) == 0){
+	    table[0][0] ++;  /* THIS WILL NEVER HAPPEN */
+	    fprintf(stderr,"Warning: MPESM test identified a correct/correct segment- %d\n",i);
+	  }
+	  else if (seg_hyp1(seg_l,i) > 0 && seg_hyp2(seg_l,i) == 0)
+	    table[1][0] ++;
+	  else if (seg_hyp1(seg_l,i) == 0 && seg_hyp2(seg_l,i) > 0)
+	    table[0][1] ++;
+	  else 
+	    table[1][1] ++;
+	}
+	ans = do_McNemar(table,sys1_str,sys2_str,verbose,fp,&mcn_conf);
+	
+	free_2dimarr(table,2,int);
+      }
+    }
+    if (0) { int i;
+      fprintf(fp,"Segment size Report\n");
+      for (i=0;i<*num_seg;i++){
+	fprintf(fp, "   Segment:  #ref=%d    sys1Err=%d   sys2Err=%d errDiff=%d\n",
+		seg_ref(seg_l,i), seg_hyp1(seg_l,i), seg_hyp2(seg_l,i),abs(seg_hyp1(seg_l,i) - seg_hyp2(seg_l,i)));
+      }
+    }
+}
+
+
+/**********************************************************************/
+/*   this function returns whether or not the Z_stat is outside the   */
+/*   range of significance, if needed, a verbose report is printed    */
+/**********************************************************************/
+int analyze_Z_score(SCORES *sys1, SCORES *sys2, double mean, double variance, double std_dev, double Z_stat, int verbose)
+{
+    int eval;
+    char pad = '\0';
+
+    if (verbose){
+      printf("\n%s\t             Analysis of matched pairs comparison\n\n",&pad);
+      printf("%s\t\t\t   Comparing %s and %s\n\n",&pad,
+	     sys1->title, sys2->title);
+      printf("%s\t       H : means of sentence segment errors are equal.\n",&pad);
+      printf("%s\t        0\n\n",&pad);
+      eval = print_Z_analysis(Z_stat);
+    }
+   else
+      eval = Z_pass(Z_stat);
+   /* if the mean is less then we want the sys1 index to be the winner */
+   if (mean < 0.0)
+       eval *= (-1);
+   return(eval);
+}
+
+/**********************************************************************/
+/*   this procedure tries to break up sentences into segments and then*/
+/*   counts and stores the segment errors and size for later          */
+/*   processing.  The algorithm is a modified state transition graph  */
+/*   that looks like this:                                            */
+/*                                                                    */
+/*      the beginning state is b:                                     */
+/*      corr means both sentences have the current word correct:      */
+/*      error means at least one sentence had an error:               */
+/*                              _                                     */
+/*                             / \ corr                               */
+/*                             v /                                    */
+/*                              b <--------------.                    */
+/*                              |                |                    */
+/*                              |  error         |                    */
+/*                              |                |                    */
+/*                              V                |                    */
+/*        ,-------------------> e<-. error       |                    */
+/*        |                     |\_/             |                    */
+/*        |                     |                |                    */
+/*        |         ,---------> |  good          |                    */
+/*        |         |           V                |                    */
+/*        |         |  if #good == min_good      |                    */
+/*   error|     corr|  then store the segment ---'                    */
+/*        |         |           |                                     */
+/*        |         |           V                                     */
+/*        |         `---------- g                                     */
+/*        |                     |                                     */
+/*        |                     |                                     */
+/*        `---------------------'                                     */
+/* Modified: Nov 10, 1994                                             */
+/*   The synchronization between reference strings is lost if altern- */
+/*   ation is use in the reference strings.  To correct the problem,  */
+/*   code was added to re-sync the reference strings if needed.       */
+/*   if the sync passes, the state is modified to dependent in the    */
+/*   words skipped to for re-synchronization.                         */
+/**********************************************************************/
+void test_comp_sents(PATH *snt1, PATH *snt2, SEG ***seg_l, int *num_seg, int *max_seg, int min_good)
+{
+    int ind1=0, ind2=0, eog1, eog2, state, num_good=0, dbg = 0;
+
+    eog1 = eog2 = 0;
+    state = 'b';
+
+    if (dbg){
+	printf("--------------------------------------------------\n");
+	printf("   number of adjacent good words %d\n\n",min_good);
+	PATH_print(snt1,stdout,100);
+	PATH_print(snt2,stdout,100);
+    }
+    while (ind1 < snt1->num){
+	if (dbg) printf("top of loop: state: %c  ind1=%d ind2=%d\n",
+			state,ind1,ind2);
+        /* if either sent has an insertion mark it and goto state 'e' */
+        while ((ind1 < snt1->num) && snt1->pset[ind1].eval == P_INS)
+            ind1++, state = 'e';
+        while ((ind2 < snt2->num) && snt2->pset[ind2].eval == P_INS)
+            ind2++, state = 'e';
+	if (dbg) printf("  rm insert: state: %c  ind1=%d ind2=%d\n",
+			state,ind1,ind2);
+	/*  Check to see if there is a synchony error, and fix it ONLY */
+	/*  if the evaluation of the words are different               */
+	if (((ind1 < snt1->num && snt1->pset[ind1].a_ptr == NULL) ^
+	     (ind2 < snt2->num && snt2->pset[ind2].a_ptr == NULL)) ||
+	    ((ind1 < snt1->num && ind2 < snt2->num) &&
+	     (strcmp(snt1->pset[ind1].a_ptr,snt2->pset[ind2].a_ptr) != 0) &&
+	     (snt1->pset[ind1].eval != snt2->pset[ind2].eval))){
+
+	    int i1=ind1, i2=ind2, synched=FALSE;
+	    if (dbg) printf("  Warning: Reference words out of alignment\n");
+	    if (dbg) printf("           Attempting to correct\n");
+	    /* the algorithm: trace the following network in a depth first  */
+	    /* manner until the first synchronization occurs:               */
+	    /*   Where A1 is the ref word s_ref_wrd(snt1,i1) . . .          */
+	    /*      A1  A2  A3  A4  ||  A1  A2  A3  A4  ||  A1  A2  A3  A4  */
+	    /*      | \             ||     /| \         ||    ____//| \     */
+            /*      |  \            ||   /  |  \        ||   /    / |  \    */
+	    /*      B1  B2  B3  B4  ||  B1  B2  B3  B4  ||  B1  B2  B3  B4  */
+	    while (i1 < snt1->num && !synched){
+		if (dbg) printf("    Starting at %d - %d\n",i1,i2);
+		while (i2 < snt2->num &&
+		       (i2 <= (ind2+1+(i1-ind1))) && !synched){
+		    if (dbg) printf("        compare %d to %d  ",i1,i2);
+		    if (strcmp(snt1->pset[i1].a_ptr,snt2->pset[i2].a_ptr) != 0)
+			synched = TRUE;
+		    if (!synched) {
+			if (dbg) printf("Not synched\n");
+			i2++;
+		    } else
+			if (dbg) printf("Synched\n");
+		}
+		if (!synched)
+		    i1++;
+	    }
+	    if (!synched) {
+		if (dbg) printf("   I FAILED to synchronize, I'm sorry\n");
+	    } else {
+		char tstate = state;
+		int ii1, ii2;
+		for (ii1 = ind1; ii1 <= i1; ii1++)
+		    if (snt1->pset[ii1].eval != P_CORR) state = 'e';
+		for (ii2 = ind2; ii2 <= i2; ii2++)
+		    if (snt2->pset[ii2].eval != P_CORR) state = 'e';
+		ind1 = i1; ind2 = i2;
+		if (dbg) {
+		    printf("   Synch WORKED, at %d - %d\n",ind1,ind2);
+		    printf("       State was '%c' now '%c'\n",tstate,state);
+		}
+	    }
+	}
+        switch (state){
+	  case 'b':
+	    /* this state means we haven't found any errors yet */
+               /* if both corr, mark the end of the good(eog) for each sent*/
+	    if (ind1 < snt1->num && snt1->pset[ind1].eval == P_CORR &&
+		ind2 < snt2->num && snt2->pset[ind2].eval == P_CORR)
+		eog1 = ind1, eog2 = ind2;
+	    else
+		state = 'e';
+	    break;
+	  case 'e':
+	    /*  if both are correct, then check to see if the num_good */
+	    /*  equals the minimum if it is mark the seg and goto 'b'  */
+	    if (ind1 < snt1->num && snt1->pset[ind1].eval == P_CORR &&
+		ind2 < snt2->num && snt2->pset[ind2].eval == P_CORR) {
+		num_good = 1;
+		if (num_good == min_good){
+		    if (dbg) printf("    segment: snt1:%d-%d  snt2:%d-%d\n",
+				    eog1-(min_good-1),ind1,eog2-(min_good-1),ind2);
+		    count_seg(snt1,snt2,eog1-(min_good-1),
+			      eog2-(min_good-1),ind1,ind2,
+			      seg_l, num_seg, max_seg, dbg);
+		    eog1 = ind1, eog2 = ind2, state = 'b';
+		}
+		else
+		    state = 'g';
+	    }
+	    break;
+	  case 'g':
+	    /* count the good words until the minimum number is found*/
+	    if (ind1 < snt1->num && snt1->pset[ind1].eval == P_CORR &&
+		ind2 < snt2->num && snt2->pset[ind2].eval == P_CORR){
+		num_good++;
+		if (num_good == min_good){
+		    if (dbg) {
+			printf("    segment: snt1:%d-%d  snt2:%d-%d\n",
+			       eog1-(min_good-1),ind1,eog2-(min_good-1),ind2);
+		    }
+		    count_seg(snt1,snt2,eog1-(min_good-1),
+			      eog2-(min_good-1),ind1,ind2,seg_l,num_seg,
+			      max_seg,dbg);
+		    eog1 = ind1, eog2 = ind2, state = 'b';
+		}
+		else
+		    state = 'g';
+	    }
+	    /* an error was found before the num_good == min_good */
+	    else
+		state = 'e';
+	    break;
+	}
+    
+	if (ind1 < snt1->num)       ind1++;
+	if (ind2 < snt2->num)	    ind2++;
+	if (dbg) printf("end of loop: state: %c  ind1=%d ind2=%d\n\n",state,ind1,ind2);
+    }
+    if (dbg) printf(    "outsid loop: state: %c  ind1=%d ind2=%d\n\n",state,ind1,ind2);
+    /*  if the state is anything but 'b' then the end of the sentence ends */
+    /*  the segment */
+    if (state != 'b' || (state == 'b' && snt2->num > ind2)){
+	if (dbg) {
+	    printf("    segment: snt1:%d-%d  snt2:%d-%d\n",
+		   eog1-(min_good-1),snt1->num-1,eog2-(min_good-1),snt2->num-1);
+	}
+        count_seg(snt1,snt2,eog1-(min_good-1), eog2-(min_good-1),snt1->num-1,
+		  snt2->num-1,seg_l,num_seg,max_seg,dbg);
+    }
+}
+
+/**********************************************************************/
+/*   this procedure only stores the errors and reference words in     */
+/*   the segment then it stores the info in a list for segments       */
+/**********************************************************************/
+void count_seg(PATH *snt1, PATH *snt2, int beg1, int beg2, int end1, int end2, SEG ***seg_l, int *num_seg, int *max_seg, int dbg)
+{
+/*     static char *ref_str=NULL, **hyp_strs;*/
+    int err1, err2, ref1, ref2;
+
+    /* do bounds checking, if indexes are < 0 */
+    if (beg1 < 0) beg1 = 0;
+    if (beg2 < 0) beg2 = 0;
+
+    if (*num_seg >= *max_seg)
+	expand_SEG_LIST(*seg_l,*num_seg,*max_seg);
+
+    seg_count(snt1,beg1,end1,&err1,&ref1);
+    seg_count(snt2,beg2,end2,&err2,&ref2);
+
+    seg_ref((*seg_l),*num_seg) = (ref1 + ref2)/2;
+    seg_hyp1((*seg_l),*num_seg) = err1;
+    seg_hyp2((*seg_l),*num_seg) = err2;
+
+    if (dbg)
+      printf("Count Segment: segment# %3d:  sys1=[beg=%d,end=%d,ref=%d,err=%d] sys2=[beg=%d,end=%d,ref=%d,err=%d]\n",
+	     *num_seg,beg1,end1,ref1,err1,beg2,end2,ref2,err2);
+
+    if (err1 == 0 && err2 == 0){
+      fprintf(scfp,"Warning: MAPSSWE segmentation produced a segment with"
+	      " errors\n         ignoring segment\n");
+    }
+
+    /* print the sentence segments to stdout */
+    if (dbg){
+	if (ref1 != ref2){
+	      printf("***** Reference Counts Not Equal\n");
+	      printf("Segment# %3d:  #ref %1d  (%d+%d)/2\n",
+		   *num_seg,(ref1+ref2)/2,ref1,ref2);
+	} else
+	    printf("Segment# %3d:  #ref %1d\n",*num_seg,ref1);
+
+	printf("  #Err.  %2d\n",err1);
+	PATH_n_print(snt1, stdout, beg1, end1+1, 100);
+	
+	printf("  #Err.  %2d\n",err2);
+	PATH_n_print(snt2, stdout, beg2, end2+1, 100);	
+	printf("\n\n");
+    }
+    if (! (err1 == 0 && err2 == 0))
+      (*num_seg)++;
+}
+
+/**********************************************************************/
+/* this routine counts the errors and ref words in a segment          */
+/**********************************************************************/
+void seg_count(PATH *snt, int from, int to, int *err, int *ref)
+{
+    int i;
+
+    *err = *ref = 0;
+    for (i=from;i<= to;i++){
+        if (snt->pset[i].eval != P_CORR)
+           (*err)++;
+        if (snt->pset[i].eval != P_INS)
+           (*ref)++;
+    }
+}
+
+/**********************************************************************/
+/*   when the average segment analysis is ran, this procedure prints  */
+/*   the report, for each number of minimum good words.  this report  */
+/*   prints the average number of segments per sentence for each      */
+/*   number of buffer words (min_good)                                */
+/**********************************************************************/
+void print_sent_seg_averaged_analysis(SCORES *scor[], int nscor, double ***seg_per_sent, char *tname)
+{
+    int sys1,sys2,num_g,num_comp;
+    double tot_sps;
+    char pad[FULL_SCREEN];
+    char *sub_title="Average Number of Segments Per Sentence Report for";
+    char *sub_title1="Changing Numbers of Buffer Words";
+
+    set_pad_cent_n(pad,strlen(sub_title), FULL_SCREEN);
+    printf("%s%s\n",pad,sub_title);
+    set_pad_cent_n(pad,strlen(sub_title1), FULL_SCREEN);
+    printf("%s%s\n",pad,sub_title1);
+    if (*tname != '\0'){
+        set_pad(pad,tname, FULL_SCREEN);
+        printf("%s%s\n",pad,tname);
+    }
+    printf("\n\n");
+
+    set_pad(pad,"--------------------------------", FULL_SCREEN);
+    printf("%s--------------------------------\n",pad);
+    printf("%s|   Num Buf   |  Ave Seg/Sent  |\n",pad);
+    printf("%s|-------------+----------------|\n",pad);
+    for (num_g=0; num_g<MAX_SEG_ANAL; num_g++){
+        num_comp = tot_sps = 0;
+        for (sys1=0;sys1<nscor;sys1++)
+            for (sys2=sys1+1;sys2<nscor;sys2++)
+                num_comp++, tot_sps+= seg_per_sent[num_g][sys1][sys2];
+        printf("%s|      %2d     |     %5.3f      |\n",
+                      pad,num_g+1,(tot_sps/num_comp));
+    }
+    printf("%s--------------------------------\n",pad);
+    form_feed(stdout);
+}
+
+/**********************************************************************/
+/*   when the long segment analysis is ran, this procedure prints     */
+/*   the  report, for each number of minimum good words.  this report */
+/*   prints the number of segments per sentence for each system       */
+/*   by system comparison                                             */
+/**********************************************************************/
+void print_sent_seg_long_analysis(SCORES *scor[], int nscor, double ***seg_per_sent, char *tname)
+{
+    int i,j,k;
+    char pad[FULL_SCREEN];
+    char *sub_title="Number of Segments Per Sentence Report for Changing";
+    char *sub_title1="Numbers of Buffer Words";
+
+    set_pad_cent_n(pad,strlen(sub_title), FULL_SCREEN);
+    printf("%s%s\n",pad,sub_title);
+    set_pad_cent_n(pad,strlen(sub_title1), FULL_SCREEN);
+    printf("%s%s\n",pad,sub_title1);
+    if (*tname != '\0'){
+        set_pad(pad,tname, FULL_SCREEN);
+        printf("%s%s\n",pad,tname);
+    }
+    printf("\n");
+
+    set_pad_cent_n(pad,(nscor+1) * 15, FULL_SCREEN);
+    printf("\n%s|-------------",pad);
+    for (k=0;k<nscor;k++)
+        printf("%13s","--------------");
+    printf("|\n");
+
+    printf("%s|             |",pad);
+    for (i=0;i<nscor;i++)
+        printf("     %4s    |",scor[i]->title);
+    printf("\n");
+
+    for (i=0;i<nscor;i++){
+        printf("%s|-------------",pad);
+        for (k=0;k<nscor;k++)
+            printf("+-------------");
+        printf("|\n");
+        printf("%s|  %4s    0  |",pad,scor[i]->title);
+        for (j=0;j<nscor;j++)
+            if (j > i)
+                printf("    %5.3f    |",seg_per_sent[0][i][j]);
+            else
+                printf("             |");
+        printf("\n");
+        if (MAX_SEG_ANAL >=2)
+            for (k=1; k<MAX_SEG_ANAL; k++){
+                printf("%s|         %2d  |",pad,k);
+                for (j=0;j<nscor;j++)
+                    if (j > i)
+                        printf("    %5.3f    |",seg_per_sent[k][i][j]);
+                    else
+                        printf("             |");
+                printf("\n");
+	    }
+    }
+    printf("%s|-------------",pad);
+    for (k=0;k<nscor;k++)
+        printf("%11s","--------------");
+    printf("|\n");
+    form_feed(stdout);
+}
+
+/**********************************************************************/
+/*   this procedure prints the Matched pairs report for a single      */
+/*   min_num_good length.                                             */
+/**********************************************************************/
+void print_compare_matrix_for_sys(SCORES *scor[], int nscor, int **winner, char *matrix_name, char *tname, char *v_desc, double **sys1_pct_arr, double **sys2_pct_arr, int **num_seg, double **Z_stat, int min_num_good, FILE *fp)
+{
+    int i,j,k,block_size,sys,max_name_len=0;
+    int hyphen_len=49, space_len=49;
+    char pad[FULL_SCREEN], hyphens[50], spaces[50], sysname_fmt[40];
+    char *min_good_title="Minimum Number of Correct Boundary words";
+
+/* init the hyphens and spaces array */
+    for (i=0; i<hyphen_len; i++){
+         hyphens[i]='-'; 
+         spaces[i]=' ';
+    }
+    hyphens[hyphen_len]='\0'; spaces[space_len]='\0';
+    /* find the largest system length */
+    for (sys=0;sys<nscor;sys++)
+      if ((i=strlen(scor[sys]->title)) > max_name_len)
+         max_name_len = i;
+    block_size = (3+max_name_len);
+    sprintf(sysname_fmt," %%%ds |",max_name_len);
+
+    set_pad(pad,matrix_name, FULL_SCREEN);
+    fprintf(fp,"\n\n\n%s%s\n",pad,matrix_name);
+    if (*v_desc != '\0'){
+        set_pad(pad,v_desc, FULL_SCREEN);
+        fprintf(fp,"%s%s\n",pad,v_desc);
+    }
+    if (*tname != '\0'){
+
+
+        set_pad(pad,tname, FULL_SCREEN);
+        fprintf(fp,"%s%s\n",pad,tname);
+    }
+    set_pad_cent_n(pad,strlen(min_good_title)+2, FULL_SCREEN);
+    fprintf(fp,"%s%s %1d\n",pad,min_good_title,min_num_good);
+    fprintf(fp,"\n");
+
+    set_pad_cent_n(pad,(nscor+1) * block_size, FULL_SCREEN);
+    fprintf(fp,"\n%s|%s",pad,hyphens+hyphen_len-MIN(hyphen_len,(block_size-1)));
+    for (k=0;k<nscor;k++)
+        fprintf(fp,"%s",hyphens+hyphen_len-MIN(hyphen_len,block_size));
+    fprintf(fp,"|\n");
+
+    fprintf(fp,"%s|%s|",pad,spaces+space_len-MIN(space_len,(block_size-1)));
+    for (i=0;i<nscor;i++)
+        fprintf(fp,sysname_fmt,scor[i]->title);
+    fprintf(fp,"\n");
+
+    for (i=0;i<nscor;i++){
+        fprintf(fp,"%s|%s",pad,hyphens+hyphen_len-MIN(hyphen_len,(block_size-1)));
+        for (k=0;k<nscor;k++)
+            fprintf(fp,"+%s",hyphens+hyphen_len-MIN(hyphen_len,(block_size-1)));
+        fprintf(fp,"|\n");
+        fprintf(fp,"%s|",pad);
+        fprintf(fp,sysname_fmt,scor[i]->title);
+        for (j=0;j<nscor;j++){
+            char *name="";
+            int t=0;
+            if (j > i){
+                if (winner[i][j] == TEST_DIFF)
+                    name=scor[j]->title;
+                else if (winner[i][j] == NO_DIFF)
+                    name="same";
+                else
+                    name=scor[i]->title;
+                t = (block_size-1-strlen(name))/2;
+            }
+            fprintf(fp,"%s%s%s|",spaces+space_len-MIN(space_len,t),name,
+                             spaces+space_len-MIN(space_len,(block_size-1-strlen(name)-t)));
+        }
+        fprintf(fp,"\n");
+#ifdef you_want_more_output
+        fprintf(fp,"%s|             |",pad);
+        for (j=0;j<nscor;j++)
+            if (j > i)
+                    fprintf(fp," %3d  %6.2f |",num_seg[i][j],Z_stat[i][j]);
+            else
+                fprintf(fp,"             |");
+        fprintf(fp,"\n");
+        fprintf(fp,"%s|             |",pad);
+        for (j=0;j<nscor;j++)
+            if (j > i)
+                fprintf(fp," %4.1f%% %4.1f%% |",sys1_pct_arr[i][j],
+                                             sys2_pct_arr[i][j]);
+            else
+                fprintf(fp,"             |");
+        fprintf(fp,"\n");
+#endif
+    }
+    fprintf(fp,"%s|%s",pad,hyphens+hyphen_len-MIN(hyphen_len,(block_size-1)));
+    for (k=0;k<nscor;k++)
+        fprintf(fp,"%s",hyphens+hyphen_len-MIN(hyphen_len,block_size));
+    fprintf(fp,"|\n");
+}
+

SCTK/src/sclite/order.c

@@ -0,0 +1,277 @@
+/**********************************************************************/
+/*                                                                    */
+/*             FILENAME:  order.c                                     */
+/*             BY:  Jonathan G. Fiscus                                */
+/*                  NATIONAL INSTITUTE OF STANDARDS AND TECHNOLOGY    */
+/*                  SPEECH RECOGNITION GROUP                          */
+/*                                                                    */
+/*           DESC:  this file contains routines to sort and rank      */
+/*                  various types of numeric arrays in INCREASING     */
+/*                  of DECREASING order                               */
+/*                                                                    */
+/**********************************************************************/
+#include "sctk.h"
+
+
+/**********************************************************************/
+/* qsort short comparison functions  */
+int qsort_short_cmp(const void *p, const void *p1);
+
+int qsort_short_deccmp(const void *p, const void *p1);
+
+static short *short_cmp_arr;
+int qsort_short_cmp(const void *p, const void *p1){
+    return(short_cmp_arr[*((int *)p)] - short_cmp_arr[*((int *)p1)]);
+}
+int qsort_short_deccmp(const void *p, const void *p1){
+    return(short_cmp_arr[*((int *)p1)] - short_cmp_arr[*((int *)p)]);
+}
+
+/**********************************************************************/
+/* qsort int comparison functions  */
+int qsort_int_cmp(const void *p, const void *p1);
+int qsort_int_deccmp(const void *p, const void *p1);
+
+static int *int_cmp_arr;
+int qsort_int_cmp(const void *p, const void *p1){
+    return(int_cmp_arr[*((int *)p)] - int_cmp_arr[*((int *)p1)]);
+}
+int qsort_int_deccmp(const void *p, const void *p1){
+    return(int_cmp_arr[*((int *)p1)] - int_cmp_arr[*((int *)p)]);
+}
+int qsort_int_compare(const void *i, const void *j) {
+    return(*((int *)i) - *((int *)j)); 
+}
+ 
+
+
+
+/**********************************************************************/
+/* qsort double comparison functions  */
+int qsort_double_cmp(const void *p, const void *p1);
+int qsort_double_deccmp(const void *p, const void *p1);
+
+static double *double_cmp_arr;
+int qsort_double_cmp(const void *p, const void *p1){
+    double f = double_cmp_arr[*((int *)p)] - double_cmp_arr[*((int *)p1)];
+    return((f < 0.0) ? -1 : ((f > 0) ? 1 : 0));
+}
+int qsort_double_deccmp(const void *p, const void *p1){
+    double f = double_cmp_arr[*((int *)p1)] - double_cmp_arr[*((int *)p)];
+    return((f < 0.0) ? -1 : ((f > 0) ? 1 : 0));
+}
+int qsort_double_compare(const void *i, const void *j) {
+    double f = *((double *)i) - *((double *)j);
+    return((f < 0.0) ? -1 : ((f > 0) ? 1 : 0));
+}
+
+/**********************************************************************/
+/*   Sort a short array                                               */
+/*        arr -> the short array                                      */
+/*        ptr_arr -> an integer array of indexes into arr.            */
+/*                   using this array to index arr, sorts the arr     */
+/**********************************************************************/
+void sort_short_arr(short int *arr, int num, int *ptr_arr, int order)
+{
+    int j;
+    /* init the indexes to 1-n */ 
+    for (j=0;j<num;j++)
+        ptr_arr[j] = j;
+
+    short_cmp_arr = arr;
+    if (order == INCREASING)
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_short_cmp);
+    else
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_short_deccmp);
+}
+
+
+/**********************************************************************/
+/*   Sort a double array                                               */
+/*        arr -> the double array                                      */
+/*        ptr_arr -> an integer array of indexes into arr.            */
+/*                   using this array to index arr, sorts the arr     */
+/**********************************************************************/
+void sort_double_arr(double *arr, int num, int *ptr_arr, int order)
+{
+    int j;
+    /* init the indexes to 1-n */ 
+    for (j=0;j<num;j++)
+        ptr_arr[j] = j;
+
+    double_cmp_arr = arr;
+    if (order == INCREASING)
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_double_cmp);
+    else
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_double_deccmp);
+}
+
+/**********************************************************************/
+/*   Sort a int array                                                 */
+/*        arr -> the int array                                        */
+/*        ptr_arr -> an integer array of indexes into arr.            */
+/*                   using this array to index arr, sorts the arr     */
+/**********************************************************************/
+void sort_int_arr(int *arr, int num, int *ptr_arr, int order)
+{
+    int j;
+    /* init the indexes to 1-n */ 
+    for (j=0;j<num;j++)
+        ptr_arr[j] = j;
+
+    int_cmp_arr = arr;
+    if (order == INCREASING)
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_int_cmp);
+    else
+	qsort((void *)ptr_arr,num,sizeof(int),qsort_int_deccmp);
+}
+
+/**********************************************************************/
+/*   Sort an array strings. to access the results, a double           */
+/*   indirection is needed.                                           */
+/*        arr -> the 2 dimarr of chars                                */
+/**********************************************************************/
+void sort_strings_using_index(char **arr, int *ind, int num, int order)
+{
+    int i, j, tmp;
+
+    for (i=0;i<num;i++)
+	ind[i] = i;
+
+    if (order == DECREASING){
+        for (j=num;j>0;j--){
+            for (i=0;i<j-1;i++)
+		if (strcmp(arr[ind[i]],arr[ind[i+1]]) < 0){
+                    tmp = ind[i];
+                    ind[i] = ind[i+1];
+                    ind[i+1] = tmp;
+	        }
+	}
+    }
+    else{
+        for (j=num;j>0;j--)
+            for (i=0;i<j-1;i++)
+                if (strcmp(arr[ind[i]],arr[ind[i+1]]) > 0){
+                    tmp = ind[i];
+                    ind[i] = ind[i+1];
+                    ind[i+1] = tmp;
+	        }
+    }
+/*    printf("sort debug\n");
+    for (i=0;i<num;i++)
+	printf("  %d arr[%d]-> %s \n",i,ind[i],arr[ind[i]]);*/
+
+}
+
+/**********************************************************************/
+/*   Sort an array strings in place                                   */
+/*        arr -> the 2 dimarr of chars                                */
+/**********************************************************************/
+void sort_strings_in_place(char **arr, int num, int order)
+{
+    int i, j;
+    char *tmp;
+
+    if (order == DECREASING){
+        for (j=num;j>0;j--){
+            for (i=0;i<j-1;i++)
+                if (strcmp(arr[i],arr[i+1]) < 0){
+                    tmp = arr[i];
+                    arr[i] = arr[i+1];
+                    arr[i+1] = tmp;
+	        }
+	}
+    }
+    else{
+        for (j=num;j>0;j--)
+            for (i=0;i<j-1;i++)
+                if (strcmp(arr[i],arr[i+1]) > 0){
+                    tmp = arr[i];
+                    arr[i] = arr[i+1];
+                    arr[i+1] = tmp;
+	        }
+    }
+/*            for (i=0;i<num;i++)
+                printf("%s ",arr[i]);
+            printf("\n");*/
+}
+
+/**********************************************************************/
+/*   Sort and rank an integer array                                   */
+/*        arr -> the integer array                                    */
+/*        ptr_arr -> an integer array of indexes into arr.            */
+/*                   using this array to index arr, sorts the arr     */
+/*        rank_arr -> an array of mean ranks of the elements in arr   */
+/**********************************************************************/
+void rank_int_arr(int *arr, int num, int *ptr_arr, double *rank_arr, int order)
+{
+    int i, j, tmp, count;
+
+
+    sort_int_arr(arr, num, ptr_arr, order);
+
+    /* mean rank the short arr                                 */
+    /*    . . . the average of the ranks for duplicate numbers */
+    for (j=0;j<num;j++){
+        /*  if the next number is = the the current number */
+        if ((j<num-1) && (arr[ptr_arr[j]] == arr[ptr_arr[j+1]])){
+            /* find the last duplicate number */
+            count = j;
+            while((count < num-1) &&
+                  (arr[ptr_arr[count]] == arr[ptr_arr[count+1]])){
+                count++;
+	    }
+            /* count the number of duplicates */
+            tmp = 0;
+            for (i=j;i<count+1;i++)
+                tmp+=i;
+            /* install the averages in all duplicates */
+            for (i=j;i<count+1;i++)
+                rank_arr[ptr_arr[i]] = (double)tmp/(double)(count+1-j) + 1.0;
+            /* go the the end of the duplicates */
+            j=i-1;
+	}
+        else{
+            /* no duplicates, use the index as the rank */
+            rank_arr[ptr_arr[j]] = (double)(j+1);
+	}
+    }
+}
+
+/**********************************************************************/
+/*   Sort and rank an double array                                     */
+/*        arr -> the double array                                      */
+/*        ptr_arr -> an integer array of indexes into arr.            */
+/*                   using this array to index arr, sorts the arr     */
+/*        rank_arr -> an array of mean ranks of the elements in arr   */
+/*                                                                    */
+/*  *** for comments, look up at sort_int_arr                         */
+/*                                                                    */
+/**********************************************************************/
+void rank_double_arr(double *arr, int num, int *ptr_arr, double *rank_arr, int order)
+{
+    int i, j, tmp, count;
+
+    sort_double_arr(arr, num, ptr_arr, order);
+
+    for (j=0;j<num;j++){
+        if ((j<num-1) && (arr[ptr_arr[j]] == arr[ptr_arr[j+1]])){
+            count = j;
+            while((count < num-1) &&
+                  (arr[ptr_arr[count]] == arr[ptr_arr[count+1]])){
+                count++;
+	    }
+            tmp = 0;
+            for (i=j;i<count+1;i++)
+                tmp+=i;
+            for (i=j;i<count+1;i++)
+                rank_arr[ptr_arr[i]] = (double)tmp/(double)(count+1-j) + 1.0;
+            j=i-1;
+	}
+        else{
+            rank_arr[ptr_arr[j]] = (double)(j+1);
+	}
+    }
+}
+
+

SCTK/src/sclite/pad.h

@@ -0,0 +1,7 @@
+void init_pad_util PROTO((int pr_width)) ;
+int pad_pr_width PROTO((void)) ;
+void set_pad PROTO((char *pad, char *str, int max)) ;
+void set_pad_n PROTO((char *pad, int n, int max)) ;
+void set_pad_cent_n PROTO((char *pad, int len, int max)) ;
+char *center PROTO((char *str, int len)) ;
+void strncpy_pad PROTO((char *to, char *from, int len, int max, char chr)) ;

SCTK/src/sclite/sclite.c

@@ -0,0 +1,764 @@
+
+#define MAIN
+#include "sctk.h"
+
+#define SCLITE_VERSION "2.10"
+
+void do_exit(char *desc, char *prog, int ret);
+void proc_args(int argc, char **argv, char *prog, char **rname, char **rfmt, char **hname, char **hfmt, int *nhyps,  enum id_types *id_type, int *case_sens, int *outputs, char **title, int *feedback, int *linewidth, int *use_diff, char **out_dir, char **out_name,int *char_align, int *pipeout, int *pipein, int *infered_wordseg, char **lexicon, int *frag_correct, int *opt_del, int *inf_flags, int *stm2ctm_reduce, int *time_align, int *conf_outputs, int *left_to_right, char **wwl_file, char **lm_file);
+int TEXT_set_lang_prof(char *lprof);
+
+#define OUT_SUM            0x0001
+#define OUT_RSUM           0x0002
+#define OUT_PRALIGN        0x0004
+#define OUT_LUR            0x0008
+#define OUT_SGML           0x0010
+#define OUT_STDOUT         0x0020
+#define OUT_SENT           0x0040
+#define OUT_SPKR           0x0080
+#define OUT_DTL            0x0100
+#define OUT_PRALIGN_FULL   0x0200
+#define OUT_WWS            0x0400
+#define OUT_NL_SGML        0x0800
+
+#define CONF_OUT_NONE      0x0000
+#define CONF_OUT_DET       0x0001
+#define CONF_OUT_BHIST     0x0002
+#define CONF_OUT_HIST      0x0004
+#define CONF_OUT_SBHIST    0x0008
+
+#define REDUCE_NOTHING        0x0000
+#define REDUCE_REF_SEGMENTS   0x0001
+#define REDUCE_HYP_WORDS      0x0002
+
+char *usage = "%s: <OPTIONS>\n" 
+"sclite Version: " SCLITE_VERSION ", SCTK Version: " TK_VERSION "\n"
+"Input Options:\n"
+"    -r reffile [ <rfmt> ]\n"
+"                Define the reference file, and it's format\n"
+"    -h hypfile [ <hfmt> <title> ]\n"
+"                Define the hypothesis file, it's format, and a 'title' used\n"
+"                for reports.  The default title is 'hypfile'.  This option\n"
+"                may be used more than once.\n"
+"    -i <ids>    Set the utterance id type.   (for transcript mode only)\n"
+"    -P          Accept the piped input from another utility.\n"
+"    -e gb|euc|utf-8 [ case-conversion-localization ]\n"
+"                Interpret characters as GB, EUC, utf-8, or the default, 8-bit ASCII.\n"
+"                Optionally, case conversion localization can be set to either 'generic',\n"
+"                'babel_turkish', or 'babel_vietnamese', 'ukranian'\n"  
+"Alignment Options:\n"
+"    -s          Do Case-sensitive alignments.\n"
+"    -d          Use GNU diff for alignments.\n"
+"    -c [ NOASCII DH ]\n"
+"                Do the alignment on characters not on words as usual by split-\n"
+"                ting words into chars. The optional argument NOASCII does not\n"
+"                split ASCII words and the optional arg. DH deletes hyphens from\n"
+"                both the ref and hyp before alingment.   Exclusive with -d.\n"
+"    -L LM       CMU-Cambridge SLM Language model file to use in alignment and scoring.\n"
+"    -S algo1 lexicon [ ASCIITOO ]\n"
+"    -S algo2 lexicon [ ASCIITOO ]\n"
+"                Instead of performing word alignments, infer the word segmenta-\n"
+"                tion using algo1 or algo2.  See sclite(1) for algorithm details.\n"
+"    -F          Score fragments as correct.  Options -F and -d are exclusive.\n"
+"    -D          Score words marked optionally deletable as correct if deleted.\n"
+"                Options -D and -d are exclusive.\n"
+"    -T          Use time information, (if available), to calculated word-to-\n"
+"                word distances based on times. Options -F and -d are exlc.\n"
+"    -w wwl      Perform Word-Weight Mediated alignments, using the WWL file 'wwl'.\n"
+"                IF wwl is 'unity' use weight 1.o for all words.\n"
+"    -m [ ref | hyp ]\n"
+"                Only used for scoring a hyp/ctm file, against a ref/stm file.\n"
+"                When the 'ref' option is used, reduce the reference segments\n"
+"                to time range of the hyp file's words.  When the 'hyp' option\n"
+"                is used, reduce the hyp words to the time range of the ref\n"
+"                segments.  The two may be used together.  The argument -m\n"
+"                by itself defaults to '-m ref'.  Exclusive with -d.\n"
+"Output Options:\n"
+"    -O output_dir\n"
+"                Writes all output files into output_dir. Defaults to the\n"
+"                hypfile's directory.\n"
+"    -f level    Defines feedback mode, default is 1\n"
+"    -l width    Defines the line width.\n"
+"    -p          Pipe the alignments to another sclite utility.  Sets -f to 0.\n"
+"Scoring Report Options:\n"
+"    -o [ sum | rsum | pralign | all | sgml | stdout | lur | snt | spk | \n"
+"         dtl | prf | wws | nl.sgml | none ]\n"
+"                Defines the output reports. Default: 'sum stdout'\n"
+"    -C [ det | bhist | sbhist | hist | none ] \n"
+"                Defines the output formats for analysis of confidence scores.\n"
+"                Default: 'none'\n"
+"    -n name     Writes all outputs using 'name' as a root filename instead of\n"
+"                'hypfile'.  For multiple hypothesis files, the root filename\n"
+"                is 'name'.'hypfile'\n";
+
+#define MAX_HYPS 40
+
+int main(int argc, char **argv){
+    char *prog = "sclite";
+    char *refname, *hypname[MAX_HYPS], *reffmt, *hypfmt[MAX_HYPS], 
+         *title[MAX_HYPS];
+    char *out_dir, *out_name;
+    char *wwl_file;
+    enum id_types id_type;
+    int errors = 0;
+    int num_hyps;
+    int linewidth;
+    int outputs;
+    int conf_outputs;
+    int feedback;
+    int use_diff;
+    SCORES *scor[MAX_HYPS];
+    int case_sense, nsc;
+    int char_align;
+    int pipeout;
+    int pipein;
+    int num_piped;
+    int nh;
+    char *hroot;
+    int hdirLen = FILENAME_MAX;
+    int outrootLen = FILENAME_MAX;
+    TEXT *hdir, *outroot;
+    int infered_wordseg;
+    char *lexicon;
+    int frag_correct;
+    int opt_del;
+    int inf_flags;
+    int stm2ctm_reduce;
+    int time_align;
+    int left_to_right;
+    WWL *wwl = (WWL *)0;
+    char *lm_file;
+
+    alloc_singZ(hdir, hdirLen, TEXT, NULL_TEXT);
+    alloc_singZ(outroot, outrootLen, TEXT, NULL_TEXT);
+
+#ifdef LM_ALIGN
+    /*    ng_t ng; */
+
+#endif
+    
+    db = db_level = 0;
+    proc_args(argc, argv, prog, &refname, &reffmt, hypname, hypfmt,
+	      &num_hyps, &id_type, &case_sense,&outputs,title,&feedback,
+	      &linewidth,&use_diff,&out_dir,&out_name,&char_align,&pipeout,
+	      &pipein, &infered_wordseg, &lexicon, &frag_correct, &opt_del,
+	      &inf_flags, &stm2ctm_reduce, &time_align, &conf_outputs,
+	      &left_to_right, &wwl_file, &lm_file);
+
+    if (feedback > 0) printf("sclite: " SCLITE_VERSION 
+			     " TK Version " TK_VERSION"\n");
+
+    if (pipein){
+	if (feedback >= 1) printf("Loading Piped input. . . ");
+	num_piped = 0;
+	if (!load_SCORES_sgml(stdin,scor,&num_piped,MAX_HYPS)){
+	    fprintf(scfp,"load_SCORES_sgml Failed\n");
+	    exit(1);
+	}
+	if (feedback >= 1) printf("%d systems loaded.\n",num_piped);	
+    } else 
+	num_piped = 0;
+
+    if (wwl_file != (char *)0){
+      if (feedback >= 1)
+	printf("Loading WWL file '%s'\n",wwl_file);
+      if (load_WWL(&wwl,(TEXT *)wwl_file) != 0){
+	fprintf(stderr,"Error: Unable to read WWL file '%s'\n",wwl_file);
+	exit(1);
+      }
+      wwl->curw = 0;
+    }
+
+    for (nsc=0; nsc<num_hyps + num_piped; nsc++){
+	if (nsc >= num_piped){
+	    nh = nsc - num_piped;
+	    if (feedback >= 1)
+		printf("Begin alignment of Ref File: '%s' and %s: '%s'\n",
+		       refname, "Hyp File", hypname[nh]);
+
+	    /* Set up the output root name */
+	    if ((hroot = strrchr(hypname[nh],'/')) != NULL){
+	      if (hroot-hypname[nh] + 1 > hdirLen){
+		expand_singarr_to_size(hdir,(hroot-hypname[nh]),hdirLen,hroot-hypname[nh]+1,TEXT);
+	      }
+	      TEXT_strBcpy((TEXT *)hdir,(TEXT *)hypname[nh],hroot-hypname[nh]);
+	      hroot++;
+	    } else {
+		hroot = hypname[nh];
+		TEXT_strcpy(hdir,(TEXT *)".");
+	    }
+	    
+	    { char *name;
+	      if ((out_dir != (char *)0) ||
+		  TEXT_strcmp(((out_dir != (char *)0) ? (TEXT *)out_dir : (TEXT *)hdir),(TEXT *)".") != 0)
+                name = rsprintf("%s/%s",((out_dir != (char *)0) ? out_dir : (char *)hdir),
+				(out_name != (char *)0) ? out_name : hroot);
+	      else
+	        name = rsprintf("%s",(out_name != (char *)0) ? out_name : hroot);
+	      if (outrootLen < TEXT_strlen((TEXT *)name)+1){
+		expand_singarr(outroot,TEXT_strlen((TEXT *)name),outrootLen,1.5,TEXT);
+	      }
+              TEXT_strcpy(outroot, (TEXT *)name);	      
+	    }
+	    if (strcmp(reffmt,"trn") == 0 && strcmp(hypfmt[nh],"trn") == 0)
+		if (!use_diff)
+		    scor[nh]=align_trans_mode_dp(refname,hypname[nh],title[nh],
+						 1,case_sense,feedback,
+						 char_align,id_type,
+						 infered_wordseg, lexicon,
+						 frag_correct, opt_del, 
+						 inf_flags, wwl, lm_file);
+		else
+		    scor[nh]=align_trans_mode_diff(refname,hypname[nh],
+						   title[nh],1,case_sense,
+						   feedback,id_type);
+	    else if (strcmp(reffmt,"stm")== 0 && strcmp(hypfmt[nh],"ctm") == 0)
+		if (!use_diff)
+		    scor[nh]=align_ctm_to_stm_dp(refname,hypname[nh],title[nh],
+						 1, case_sense,feedback,
+						 char_align,id_type,
+						 infered_wordseg, lexicon,
+						 frag_correct, opt_del,
+						 inf_flags, 
+				  BF_isSET(stm2ctm_reduce,REDUCE_REF_SEGMENTS),
+				  BF_isSET(stm2ctm_reduce,REDUCE_HYP_WORDS),
+						 left_to_right, wwl, lm_file);
+		else {
+		  if (left_to_right)
+		    scor[nh]=align_ctm_to_stm_diff(refname, hypname[nh],
+						   title[nh],1,case_sense,
+						   feedback,id_type);
+		  else {
+		    fprintf(scfp,"Error: can't perform diff alignments on "
+			    "right-to-left data\n");
+		    exit(1);
+		  }
+		}
+	    else if (strcmp(reffmt,"ctm")== 0 && strcmp(hypfmt[nh],"ctm") == 0)
+	      scor[nh]=align_ctm_to_ctm(hypname[nh], refname, title[nh],
+					feedback, frag_correct, opt_del,
+					case_sense, time_align, left_to_right, wwl,
+					lm_file);
+	    else if (strcmp(reffmt,"stm")== 0 && strcmp(hypfmt[nh],"txt") ==0){
+#if DIFF_ENABLED
+	      if (left_to_right)
+		scor[nh] =align_text_to_stm(refname, hypname[nh],title[nh],
+					    1,case_sense,feedback,id_type);
+	      else {
+		fprintf(scfp,"Error: can't perform diff alignments on "
+			"right-to-left data\n");
+		exit(1);
+	      }
+#else		
+		do_exit("Alignments via diff have been disabled",prog,1);
+#endif
+	    } else{
+		fprintf(stderr,"Error: Unable to score '%s' against '%s'\n",
+			hypfmt[nh],reffmt);
+		exit(1);
+	    }
+	    
+	    if (scor[nh] == (SCORES *)0){
+		fprintf(stderr,"%s: Alignment failed.  Exiting.\n",prog);
+		exit(1);
+	    }
+	} else { /* input came from piped input */
+	    /* Set up the output root name */
+	    if ((hroot = strrchr(scor[nsc]->title,'/')) != NULL){
+		strncpy((char *)hdir,scor[nsc]->title,hroot-scor[nsc]->title);
+		hdir[hroot-scor[nsc]->title] = '\0';
+		hroot++;
+	    } else {
+		hroot = scor[nsc]->title;
+		strcpy((char *)hdir,".");
+	    }
+	    if ((out_dir != (char *)0) ||
+		strcmp(((out_dir != (char *)0) ? out_dir : (char *)hdir),".") != 0)
+	      sprintf((char *)outroot,"%s/%s",((out_dir != (char *)0) ? out_dir : (char *)hdir),
+		      (out_name != (char *)0) ? out_name : hroot);
+	    else
+	      sprintf((char *)outroot,"%s",(out_name != (char *)0) ? out_name : hroot);
+	}	    
+	
+	if (BF_isSET(outputs,OUT_SUM))
+	  print_system_summary(scor[nsc],
+			       BF_isSET(outputs,OUT_STDOUT) ? "-": (char *)outroot,
+			       0, 0, 0, feedback);
+	if (BF_isSET(outputs,OUT_WWS)){
+	  if (scor[nsc]->weight_ali)
+	    print_system_summary(scor[nsc],
+				 BF_isSET(outputs,OUT_STDOUT) ? "-": (char *)outroot,
+				 0, 0, 1, feedback);
+	  else
+	    printf("    Skipping WWS Report, no word weights supplied.\n");
+	}
+	if (BF_isSET(outputs,OUT_RSUM))
+	    print_system_summary(scor[nsc], 
+				 BF_isSET(outputs,OUT_STDOUT) ? "-":(char *)outroot,
+				 0, 1, 0, feedback);
+	if (BF_isSET(outputs,OUT_SENT))
+	    score_dtl_sent(scor[nsc], BF_isSET(outputs,OUT_STDOUT)?"-":(char *)outroot,
+			   feedback);
+	if (BF_isSET(outputs,OUT_SPKR))
+	    score_dtl_spkr(scor[nsc], BF_isSET(outputs,OUT_STDOUT)?"-":(char *)outroot,
+			   feedback);
+	if (BF_isSET(outputs,OUT_DTL))
+	    score_dtl_overall(scor[nsc],
+			      BF_isSET(outputs,OUT_STDOUT) ? "-":(char *)outroot,
+			      feedback);
+	if (BF_isSET(outputs,OUT_LUR))
+	    print_lur(scor[nsc],
+		      BF_isSET(outputs,OUT_STDOUT) ? "-":(char *)outroot, feedback);
+	if (BF_isSET(outputs,OUT_PRALIGN)){
+	    FILE *fp = stdout;
+	    if (BF_notSET(outputs,OUT_STDOUT))
+		fp = fopen(rsprintf("%s.pra",outroot),"w");
+	    if (fp == (FILE *)0) fp = stdout;
+	    if (feedback >= 1)
+		printf("    Writing string alignments to '%s'\n",
+		       (fp == stdout) ? "stdout" :
+		       rsprintf("%s.pra",outroot));
+	    dump_SCORES_alignments(scor[nsc],fp,linewidth,0);
+	    if (fp != stdout) fclose(fp);
+	}
+	if (BF_isSET(outputs,OUT_PRALIGN_FULL)){
+	    FILE *fp = stdout;
+	    if (BF_notSET(outputs,OUT_STDOUT))
+		fp = fopen(rsprintf("%s.prf",outroot),"w");
+	    if (fp == (FILE *)0) fp = stdout;
+	    if (feedback >= 1)
+		printf("    Writing string alignments to '%s'\n",
+		       (fp == stdout) ? "stdout" :
+		       rsprintf("%s.prf",outroot));
+	    dump_SCORES_alignments(scor[nsc],fp,linewidth,1);
+	    if (fp != stdout) fclose(fp);
+	}
+	if (BF_isSET(outputs,OUT_SGML)){
+	    FILE *fp = stdout;
+	    if (BF_notSET(outputs,OUT_STDOUT))
+		fp = fopen(rsprintf("%s.sgml",outroot),"w");
+	    if (fp == (FILE *)0) fp = stdout;
+	    if (fp != stdout && feedback >= 1)
+		printf("    Writing SGML string alignments to '%s'\n",
+		       rsprintf("%s.sgml",outroot));
+	    dump_SCORES_sgml(scor[nsc],fp,(TEXT *)":",(TEXT *)",");
+	    if (fp != stdout) fclose(fp);
+	}
+	if (BF_isSET(outputs,OUT_NL_SGML)){
+	    FILE *fp = stdout;
+	    if (BF_notSET(outputs,OUT_STDOUT))
+		fp = fopen(rsprintf("%s.nl.sgml",outroot),"w");
+	    if (fp == (FILE *)0) fp = stdout;
+	    if (fp != stdout && feedback >= 1)
+		printf("    Writing Newline Separated SGML string alignments to '%s'\n",
+		       rsprintf("%s.nl.sgml",outroot));
+	    dump_SCORES_sgml(scor[nsc],fp,(TEXT *)"\n",(TEXT *)"\n");
+	    if (fp != stdout) fclose(fp);
+	}
+	if (BF_isSET(conf_outputs,CONF_OUT_DET))
+	  if (make_SCORES_DET_curve(&(scor[nsc]),1,(char *)outroot,feedback,"") != 0)
+	    errors++;
+	if (BF_isSET(conf_outputs,CONF_OUT_BHIST))
+	  if (make_binned_confidence(scor[nsc],(char *)outroot,feedback) != 0)
+	        errors++;
+	if (BF_isSET(conf_outputs,CONF_OUT_HIST))
+	  if (make_confidence_histogram(scor[nsc],(char *)outroot,feedback) != 0)
+	        errors++;
+	if (BF_isSET(conf_outputs,CONF_OUT_SBHIST))
+	  if (make_scaled_binned_confidence(scor[nsc],(char *)outroot,20,feedback) != 0)
+	        errors++;
+	
+    }
+
+    if (pipeout)
+	for (nsc=0; nsc < num_hyps + num_piped; nsc++)
+	    dump_SCORES_sgml(scor[nsc],stdout,(TEXT *)"\n",(TEXT *)"\n");
+
+    /* clean up the score structures */
+    for (nsc=0; nsc < num_hyps + num_piped; nsc++)
+        SCORES_free(scor[nsc]);
+    if (wwl != (WWL *)0) free_WWL(&wwl);
+
+    if (feedback >= 1) {
+        if (errors == 0)
+	    printf("\nSuccessful Completion\n");
+	else
+  	    printf("\nUnsuccessful Completion\n");
+    }
+    return(0);
+}
+
+void proc_args(int argc, char **argv, char *prog, char **rname, char **rfmt, char **hname, char **hfmt, int *nhyps, enum id_types *id_type, int *case_sense, int *outputs, char **title, int *feedback, int *linewidth, int *use_diff, char **out_dir, char **out_name, int *char_align, int *pipeout, int *pipein, int *infered_wordseg, char **lexicon, int *frag_correct, int *opt_del, int *inf_flags, int *stm2ctm_reduce, int *time_align, int *conf_outputs, int *left_to_right, char **wwl_file, char **lm_file){
+    int opt, fbset=0, outset=0, i, nh;
+    char *id;
+    struct stat fileinfo;
+
+    if (argc <= 1) 
+	do_exit("Arguments reguired",prog,1);
+
+    *linewidth=1000;
+    *feedback=1;
+    *nhyps = 0;
+    *rname=(char *)0;  *hname=(char *)0;
+    *rfmt=(char *)0;   *hfmt=(char *)0;
+    id=(char *)0;
+    *outputs=0;
+    *case_sense=0;
+    *title=(char *)0;
+    *out_dir = (char *)0;
+    *out_name = (char *)0;
+    *use_diff = 0;
+    *char_align = 0;
+    *pipeout = 0;
+    *pipein = 0;
+    *infered_wordseg = 0;
+    *lexicon = (char *)0;
+    *frag_correct = 0;
+    *opt_del = 0;
+    *inf_flags = 0;
+    *stm2ctm_reduce = 0;
+    *time_align = 0;
+    *conf_outputs = CONF_OUT_NONE;
+    *left_to_right = 1;
+    *wwl_file = (char *)0;
+    *lm_file = (char *)0;
+
+    for (opt = 1; opt < argc && (*(argv[opt]) == '-'); opt++){
+	/* printf("Current OP %s\n",argv[opt]); */
+	if (strcmp(argv[opt],"-r") == 0){
+	    if (argc < opt + 2) do_exit("Not enough Ref arguments",prog,1);
+	    if (*(argv[opt+1]) == '-') do_exit("Req'd Ref File name",prog,1);
+	    *rname = argv[++opt];
+	    if (argc >= opt + 2 && *(argv[opt+1]) != '-') *rfmt  = argv[++opt];
+	} else if (strcmp(argv[opt],"-h") == 0){
+	    if (argc < opt + 2) do_exit("Not enough Hyp arguments",prog,1);
+	    if (*(argv[opt+1]) == '-') do_exit("Req'd Hyp File name",prog,1);
+	    hname[*nhyps] = argv[++opt];
+	    if (argc >= opt + 2 && *(argv[opt+1]) != '-') 
+		hfmt[*nhyps]  = argv[++opt];
+	    else
+		hfmt[*nhyps] = "trn";
+	    if (argc >= opt + 2 && *(argv[opt+1]) != '-') 
+		title[*nhyps] = argv[++opt];
+	    else
+		title[*nhyps] = hname[*nhyps];
+	    (*nhyps)++;
+	} else if (strcmp(argv[opt],"-i") == 0){
+	    if (argc <= opt+1) do_exit("Not enough ID arguments",prog,1);
+	    id = argv[++opt];
+	} else if (strcmp(argv[opt],"-l") == 0){
+	    if (argc <= opt+1)
+		do_exit("Not enough Line Width arguments",prog,1);
+	    *linewidth = atoi(argv[++opt]);
+	} else if (strcmp(argv[opt],"-L") == 0){
+#ifdef WITH_SLM
+	    if (argc <= opt+1)
+		do_exit("Not enough Language Model arguments",prog,1);
+	    *lm_file = argv[++opt];
+#else
+	    do_exit("-L option used, but SLM toolkit not compiled into sctk\n",prog,1);
+#endif
+	} else if (strcmp(argv[opt],"-f") == 0){
+	    if (argc <= opt+1) do_exit("Not enough Feedback arguments",prog,1);
+	    *feedback = atoi(argv[++opt]);
+	    fbset = 1;
+	} else if (strcmp(argv[opt],"-o") == 0){
+	    opt ++;
+	    if (argc < opt + 1) do_exit("Not enough Report arguments",prog,1);
+	    while (opt < argc && *(argv[opt]) != '-'){
+		if (strcmp(argv[opt],"sum") == 0) 
+		    BF_FLIP(*outputs,OUT_SUM);
+		else if (strcmp(argv[opt],"wws") == 0) 
+		    BF_FLIP(*outputs,OUT_WWS);
+		else if (strcmp(argv[opt],"rsum") == 0) 
+		    BF_FLIP(*outputs,OUT_RSUM);
+		else if ((strcmp(argv[opt],"pralign") == 0) ||
+			 (strcmp(argv[opt],"pra") == 0)) 
+		    BF_FLIP(*outputs,OUT_PRALIGN);
+		else if (strcmp(argv[opt],"prf") == 0) 
+		    BF_FLIP(*outputs,OUT_PRALIGN_FULL);
+		else if (strcmp(argv[opt],"lur") == 0) 
+		    BF_FLIP(*outputs,OUT_LUR);
+		else if (strcmp(argv[opt],"stdout") == 0) 
+		    BF_FLIP(*outputs,OUT_STDOUT);
+		else if (strcmp(argv[opt],"sgml") == 0) 
+		    BF_FLIP(*outputs,OUT_SGML);
+		else if (strcmp(argv[opt],"nl.sgml") == 0) 
+		    BF_FLIP(*outputs,OUT_NL_SGML);
+		else if (strcmp(argv[opt],"snt") == 0) 
+		    BF_FLIP(*outputs,OUT_SENT);
+		else if (strcmp(argv[opt],"spk") == 0) 
+		    BF_FLIP(*outputs,OUT_SPKR);
+		else if (strcmp(argv[opt],"dtl") == 0) 
+		    BF_FLIP(*outputs,OUT_DTL);
+		else if (strcmp(argv[opt],"all") == 0) 
+		    BF_FLIP(*outputs,OUT_PRALIGN + OUT_SUM + OUT_RSUM);
+		else if (strcmp(argv[opt],"none") == 0) 
+		    *outputs = 0;
+		else
+		    fprintf(stderr,"Unknown report '%s'\n",argv[opt]);
+		opt++;
+	    }
+	    /* backup if we've gone to far */
+	    if (opt < argc && *(argv[opt]) == '-') opt--;
+	    outset=1;
+	} else if (strcmp(argv[opt],"-C") == 0){
+	    opt ++;
+	    if (argc < opt + 1)
+	      do_exit("Not enough Confidence Report arguments",prog,1);
+	    while (opt < argc && *(argv[opt]) != '-'){
+		if (strcmp(argv[opt],"det") == 0) 
+		    BF_FLIP(*conf_outputs,CONF_OUT_DET);
+		else if (strcmp(argv[opt],"bhist") == 0) 
+		    BF_FLIP(*conf_outputs,CONF_OUT_BHIST);
+		else if (strcmp(argv[opt],"sbhist") == 0) 
+		    BF_FLIP(*conf_outputs,CONF_OUT_SBHIST);
+		else if (strcmp(argv[opt],"hist") == 0) 
+		    BF_FLIP(*conf_outputs,CONF_OUT_HIST);
+		else if (strcmp(argv[opt],"none") == 0) 
+		    *conf_outputs = CONF_OUT_NONE;
+		else
+		    fprintf(stderr,"Unknown confidence report '%s'\n",
+			    argv[opt]);
+		opt++;
+	    }
+	    /* backup if we've gone to far */
+	    if (opt < argc && *(argv[opt]) == '-') opt--;
+	    outset=1;
+	} else if (strcmp(argv[opt],"-O") == 0){
+	    if (argc <= opt+1)do_exit("Output directory not specified",prog,1);
+	    *out_dir = argv[++opt];
+	} else if (strcmp(argv[opt],"-n") == 0){
+	    if (argc <= opt+1)do_exit("Output name not specified",prog,1);
+	    *out_name = argv[++opt];
+	} else if (strcmp(argv[opt],"-d") == 0){
+#if DIFF_ENABLED
+	    *use_diff = 1;
+#else
+	    do_exit("Alignments via diff have been disabled",prog,1);
+#endif
+	} else if (strcmp(argv[opt],"-s") == 0){
+	    *case_sense = 1;
+	} else if (strcmp(argv[opt],"-m") == 0){
+	    *stm2ctm_reduce = REDUCE_NOTHING;
+	    while (opt+1 < argc && *(argv[opt+1]) != '-') {
+		opt++;
+		if (strcmp(argv[opt],"ref") == 0)
+		    BF_SET(*stm2ctm_reduce,REDUCE_REF_SEGMENTS);
+		else if (strcmp(argv[opt],"hyp") == 0)
+		    BF_SET(*stm2ctm_reduce,REDUCE_HYP_WORDS);
+		else
+		    do_exit(rsprintf("Unrecognized -m option '%s'",
+				     argv[opt]),prog,1);
+	    }
+	    if (*stm2ctm_reduce == REDUCE_NOTHING)
+		*stm2ctm_reduce = REDUCE_REF_SEGMENTS;
+	} else if (strcmp(argv[opt],"-F") == 0){
+	    *frag_correct = 1;
+	} else if (strcmp(argv[opt],"-D") == 0){
+	    *opt_del = 1;
+	} else if (strcmp(argv[opt],"-c") == 0){
+            *char_align = CALI_ON;
+	    while (opt+1 < argc && *(argv[opt+1]) != '-') {
+		opt++;
+		if (strcmp(argv[opt],"NOASCII") == 0)
+		    BF_SET(*char_align,CALI_NOASCII);
+		else if (strcmp(argv[opt],"DH") == 0)
+		    BF_SET(*char_align,CALI_DELHYPHEN);
+		else
+		    do_exit(rsprintf("Unrecognized character alignment "
+				     "option '%s'",argv[opt]),prog,1);
+	    }
+	} else if (strcmp(argv[opt],"-e") == 0){
+	    if (opt+1 >= argc || (opt+1 < argc && *(argv[opt+1]) == '-'))
+		do_exit("Argument required for character encoding\n",prog,1);
+	    opt++;
+	    if (!TEXT_set_encoding(argv[opt]))
+		do_exit(rsprintf("Unrecognized character encoding "
+				 "option '%s'",argv[opt]),prog,1);
+            // Parse the optional localization
+	    if (opt+1 < argc && *(argv[opt+1]) != '-'){
+	        if (!TEXT_set_lang_prof(argv[opt+1]))
+		    do_exit(rsprintf("Optional case conversion localization failed /%s/\n",
+		            argv[opt+1]),prog,1);
+  	        opt++;			 
+  	     }
+	} else if (strcmp(argv[opt],"-p") == 0){
+	    *pipeout = 1;
+	    outset = 1;
+	} else if (strcmp(argv[opt],"-P") == 0){
+	    *pipein = 1;
+	} else if (strcmp(argv[opt],"-T") == 0){
+	    *time_align = 1;
+	} else if (strcmp(argv[opt],"-S") == 0){
+	    opt ++;
+	    if (argc < opt + 2) do_exit("Not enough Inferred Segmentation "
+					"arguments",prog,1);
+	    if (opt < argc && *(argv[opt]) != '-')
+		if (strcmp(argv[opt],"algo1") == 0) 
+		    *infered_wordseg ^= INF_SEG_ALGO1;
+	        else if (strcmp(argv[opt],"algo2") == 0) 
+		    *infered_wordseg ^= INF_SEG_ALGO2;
+		else 
+		    do_exit(rsprintf("Unrecognize inferred segmentation "
+				     "algorithm '%s'",argv[opt]),prog,1);
+	    else
+		do_exit("No Inferred Segmentation algorithm",prog,1);
+	    opt++;
+	    if (opt < argc && *(argv[opt]) != '-'){
+		if (stat(argv[opt],&fileinfo) != 0) 
+		    do_exit(rsprintf("Inferred Segmentation lexicon '%s'"
+				     " does not exist",argv[opt]),prog,1);
+		*lexicon = argv[opt];
+	    } else
+		do_exit("No Inferred Segmentation lexicon",prog,1);
+	    if (opt+1 < argc && *(argv[opt+1]) != '-'){
+		opt++;
+		if (strcmp(argv[opt],"ASCIITOO") == 0)
+		    BF_SET(*inf_flags,INF_ASCII_TOO);
+		else
+		    do_exit(rsprintf("Unrecognized Inferred Segmentation"
+				     " flag '%s'",argv[opt]),prog,1);
+	    }
+	} else if (strcmp(argv[opt],"-w") == 0){
+	    if (argc <= opt+1)do_exit("Word Weight list file not specified",prog,1);
+	    *wwl_file = argv[++opt];
+	} else {
+            (void) fprintf(stderr,usage,prog);
+            printf("Illegal argument: %s\n",argv[opt]);
+            exit(1);
+        }
+    }
+
+    if (*pipeout){
+	*feedback = 0;
+	if (BF_isSET(*outputs,OUT_STDOUT))
+	    do_exit("Error: output can not be both to stdout and a pipe",
+		    prog,1);
+    }
+
+    /* set the default outputs */
+    if (*outputs == 0 && !outset)
+	BF_SET(*outputs,OUT_SUM + OUT_STDOUT);
+
+    /* reset feedback to 0 if STDOUT is used */
+    if ((BF_isSET(*outputs,OUT_STDOUT) && outset) &&
+	(*feedback > 0 && !fbset)){
+	/* fprintf(stderr,"Feedback level changed to 0\n");*/
+	*feedback = 0;
+    }
+
+    /* Check the input arguments */
+    if (! *pipein && ! (*rname != (char *)0 && *nhyps > 0))
+	do_exit("Input not specified, use transcription input or "
+		"piped input",prog,1);
+
+    if (*rname == (char *)0 && *nhyps > 0)
+	do_exit("Hypothesis input(s) specified, but no reference file",
+		prog,1);
+    if (*rname != (char *)0 && *nhyps == 0)
+	do_exit("Reference input specified, but no Hypothesis input(s).",
+		prog,1);
+
+    if (*rname != (char *)0 && *nhyps > 0){
+	if (stat(*rname,&fileinfo) != 0) 
+	    do_exit(rsprintf("Reference file '%s' does not exist",*rname),
+		    prog,1);
+
+	for (nh=0; nh < *nhyps; nh++)
+	    if (stat(hname[nh],&fileinfo) != 0) 
+		do_exit(rsprintf("Hypothesis file '%s' does not exist",
+				 hname[nh]),prog,1);
+
+	/* check the ref and hyp format arguments */
+	if (*rfmt == (char *)0) *rfmt = "trn";
+	if (! (strcmp(*rfmt,"trn") == 0 || strcmp(*rfmt,"ctm") == 0 || 
+	       strcmp(*rfmt,"tmk") == 0 || strcmp(*rfmt,"stm") == 0))
+	    do_exit(rsprintf("Reference file format '%s' not acceptable",*rfmt),
+		    prog,1);
+	
+	if (*hfmt == (char *)0) *hfmt = "trn";
+	for (i=0; i<*nhyps; i++)
+	    if (! (strcmp(hfmt[i],"trn") == 0 || strcmp(hfmt[i],"ctm") == 0 || 
+		   strcmp(hfmt[i],"tmk") == 0  || strcmp(hfmt[i],"txt") == 0)) 
+		do_exit(rsprintf("Hypothesis file format '%s' not acceptable",
+				 hfmt[i]),prog,1);
+	
+	/* check the id arguments */
+	if (id == (char *)0){
+	    if (strcmp(*rfmt,"trn") == 0)
+		do_exit("Required utterance id (option -i) for"
+			" transcript mode", prog,1);
+	    for (i=0; i<*nhyps; i++){
+		if (strcmp(hfmt[i],"trn") == 0)
+		    do_exit("Required utterance id (option -i) for"
+			    " transcript mode",prog,1);
+	    }
+	}
+
+	/* only do time alignment on ctm to ctm */
+	if (*time_align){
+	    if (strcmp(*rfmt,"ctm") != 0)
+		do_exit("Time-mediated alignments require the reference"
+			" file to be in CTM format",prog,1);
+	    for (i=0; i<*nhyps; i++)
+		if (strcmp(hfmt[i],"ctm") != 0)
+		    do_exit("Time-mediated alignments require the hypothesis"
+			    " file to be in CTM format",prog,1);
+	}
+    }
+    
+    if (id != (char *)0){
+	if (strcmp(id,"swb") == 0)         *id_type=SWB;
+	else if (strcmp(id,"rm") == 0)     *id_type=RM;
+	else if (strcmp(id,"wsj") == 0)    *id_type=WSJ;
+	else if (strcmp(id,"spu_id") == 0) *id_type=SPUID;
+        else if (strcmp(id,"sp") == 0)     *id_type=SP;
+	else
+	    do_exit(rsprintf("Utterance id type '%s' not acceptable",id),
+		    prog,1);
+    }
+
+    if (*stm2ctm_reduce){
+	if (strcmp(*rfmt,"stm") != 0)
+	    do_exit("-m flag requires stm format reference file",prog,1);
+	for (i=0; i<*nhyps; i++)
+	    if (strcmp(hfmt[i],"ctm") != 0)
+		do_exit("-m flag requires ctm format hypothesis file",prog,1);
+	if (*use_diff)
+	    do_exit("-m flag can not be used with -d flag",prog,1);
+    }
+
+    if (*char_align && (*use_diff ||
+		       (strcmp(*rfmt,"stm") == 0 && strcmp(*hfmt,"txt")==0)))
+	do_exit("Unable to do character alignments using diff\n",prog,1);
+    
+    if (*wwl_file != (char *)0 && *lm_file != (char *)0)
+      do_exit("Flags '-L' and '-w' are mutually exclusive\n",prog,1);
+
+    if (*wwl_file != (char *)0){
+      if (*time_align) do_exit("Flags '-T' and '-w' are mutually exclusive\n",prog,1);
+      if (*char_align) do_exit("Flags '-c' and '-w' are mutually exclusive\n",prog,1);
+      if (*infered_wordseg) do_exit("Flags '-S' and '-w' are mutually exclusive\n",prog,1);
+      if (*use_diff) do_exit("Flags '-d' and '-w' are mutually exclusive\n",prog,1);
+    }
+    if (*lm_file != (char *)0){
+      if (*time_align) do_exit("Flags '-T' and '-L' are mutually exclusive\n",prog,1);
+      if (*char_align) do_exit("Flags '-c' and '-L' are mutually exclusive\n",prog,1);
+      if (*infered_wordseg) do_exit("Flags '-S' and '-L' are mutually exclusive\n",prog,1);
+      if (*use_diff) do_exit("Flags '-d' and '-L' are mutually exclusive\n",prog,1);
+    }
+
+    if (BF_isSET(*outputs,OUT_WWS))
+      if ((*wwl_file == (char *)0) && (*lm_file == (char *)0))
+	do_exit("-o wws option specified without a word weight file (via -w) or "
+		"LM file (via -L) specified\n",
+		prog,1)	;
+}
+
+void do_exit(char *desc, char *prog, int ret){
+    fprintf(stderr,usage,prog);
+    fprintf(stderr,"\n%s: Error, %s\n\n",prog,desc);
+    exit(ret);
+}
+

SCTK/src/sclite/signtest.c

@@ -0,0 +1,304 @@
+/**********************************************************************/
+/*                                                                    */
+/*           FILE: signtest.c                                         */
+/*           WRITTEN BY: Jonathan G. Fiscus                           */
+/*           DATE: April 14 1989                                      */
+/*                  NATIONAL INSTITUTE OF STANDARDS AND TECHNOLOGY    */
+/*                  SPEECH RECOGNITION GROUP                          */
+/*                                                                    */
+/*           USAGE: This uses the rank structure to perform           */
+/*                  the non-parametric Sign Test and generates        */
+/*                  a report.                                         */
+/*                                                                    */
+/*           SOURCE:Statistics: Basic Techniques for Solving          */
+/*                  Applied Problems, by Stephen A. Book, 1977        */
+/*                                                                    */
+/*           See Also: The documentation file "signtest.doc"          */
+/*                                                                    */
+/**********************************************************************/
+
+#include "sctk.h"
+
+/****************************************************************/
+/*   main procedure to perform the sign test on the RANK        */
+/*   structure.                                                 */
+/****************************************************************/
+void perform_signtest(RANK *rank, int verbose, int report, char *formula_str, char formula_id, int ***out_winner, char *outroot, int feedback, double ***out_confidence)
+{
+    int comp1, comp2, **winner;
+    double **confidence;
+    FILE *fp = stdout;
+    
+    if (report || verbose){
+	char *f = rsprintf("%s.sign",outroot);
+	if ((fp=(strcmp(outroot,"-") == 0) ? stdout : fopen(f,"w")) ==
+	    (FILE *)0){
+	    fprintf(stderr,"Warning: Open of %s for write failed.  "
+		           "Using stdout instead.\n",f);
+	    fp = stdout;
+	} else
+	    if (feedback >= 1) printf("        Output written to '%s'\n",f);
+    }
+
+    alloc_2dimZ(winner,rank->n_trt,rank->n_trt,int,NO_DIFF);
+    alloc_2dimZ(confidence,rank->n_trt,rank->n_trt,double,0.0);
+    *out_confidence = confidence;
+    *out_winner = winner;
+
+    for (comp1=0; comp1 < rank->n_trt  -1; comp1++)
+        for (comp2=comp1+1; comp2< rank->n_trt ; comp2++){
+            winner[comp1][comp2] = 
+               compute_signtest_for_treatment(rank,comp1,comp2,"Spkr",
+					      formula_str,verbose,
+					      formula_id=='E',fp,
+					      &(confidence[comp1][comp2]));
+        }
+
+    if (report) print_trt_comp_matrix_for_RANK_one_winner(winner,rank,
+                 "Comparison Matrix for the Sign Test",formula_str,
+                 "Speaker",fp);
+    if (fp != stdout) fclose(fp);
+}
+
+/****************************************************************/
+/*   Given the two indexes of treatments to compare (in the     */
+/*   RANK Struct) compute the Rank Sum statistics               */
+/*   vars:                                                      */
+/*       zero_is_best : This identifies the "ideal" value for   */
+/*                      the value computed in the rank struct   */
+/*                      percentages.                            */
+/****************************************************************/
+int compute_signtest_for_treatment(RANK *rank, int treat1, int treat2, char *block_id, char *formula_str, int verbose, int zero_is_best, FILE *fp, double *confidence)
+{
+    int sum_plus=0, sum_minus=0, sum_equal=0, i;
+    int block, max_len_block=0, max_len_treat=6;
+    int tptr[2]; /* a sorting pointer array for the treatment numbers */
+    char *pct_format, thresh_str[140];
+    TEXT *title_line = (TEXT *)0;
+    char *diff_line;
+    int paper_width = 79, rep_width, diff_col_len;
+    double equal_thresh = 0.005;
+    double sum_trt1=0.0, sum_trt2=0.0;
+
+    /* compute the maximum block title length */
+    max_len_block = strlen(block_id);
+    for (block=0; block <rank->n_blk ; block++)
+         if (strlen(rank->blk_name [ block ] ) > max_len_block)
+             max_len_block = strlen(rank->blk_name [ block ] );
+
+    if (max_len_treat < strlen(rank->trt_name [ treat1 ] ))
+        max_len_treat = strlen(rank->trt_name [ treat1 ] );
+
+    if (max_len_treat < strlen(rank->trt_name [ treat2 ] ))
+        max_len_treat = strlen(rank->trt_name [ treat2 ] );
+
+    /* set the treatment sorting array */
+    for (block=0; block <rank->n_blk ; block++){
+        sum_trt1 += rank->pcts [ block ][ treat1 ] ;
+        sum_trt2 += rank->pcts [ block ][ treat2 ] ;
+    }
+    if (sum_trt1 > sum_trt2)
+        tptr[0] = treat1, tptr[1] = treat2;
+    else
+        tptr[0] = treat2, tptr[1] = treat1;
+
+    /* set up format strings and titles */
+    sprintf(thresh_str,"(Threshold for equal percentages +- %5.3f)",
+                       equal_thresh);
+    alloc_singarr(pct_format, max_len_treat + 2,char); 
+    alloc_singarr(diff_line, max_len_treat*2 + 7,char); 
+    pct_format[0] = '\0';
+    strcat(pct_format,center("",(max_len_treat-6)/2));
+    strcat(pct_format,"%6.2f");
+    strcat(pct_format,center("",max_len_treat - ((max_len_treat-6)/2 + 6) ));
+    sprintf(diff_line,"[%s - %s]",rank->trt_name[tptr[0]],
+	  rank->trt_name[tptr[1]]);
+    diff_col_len = 15;
+    if (strlen(diff_line) > diff_col_len) diff_col_len=strlen(diff_line);
+
+    rep_width = max_len_block + max_len_treat * 2 + 4 * 3 + diff_col_len;
+
+    /*  Print a detailed table showing sign differences */
+    if (verbose) {
+        sum_trt1 = sum_trt2 = 0.0;
+        fprintf(fp,"%s\n",center("Sign Test Calculations Table Comparing",
+				 paper_width));
+        title_line = TEXT_strdup((TEXT *)rsprintf("%s %s Percentages for Systems %s and %s",
+		block_id, formula_str,rank->trt_name[tptr[0]],
+		rank->trt_name [ tptr[1] ] ));
+        fprintf(fp,"%s\n",center((char *)title_line,paper_width));
+        fprintf(fp,"%s\n\n",center(thresh_str,paper_width));
+
+        fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+        fprintf(fp,"%s",center("",max_len_block));
+        fprintf(fp,"    "); fprintf(fp,"%s",center("",max_len_treat));
+        fprintf(fp,"    "); fprintf(fp,"%s",center("",max_len_treat));
+        fprintf(fp,"    ");
+        fprintf(fp,"%s\n",center("Difference Sign",diff_col_len));
+
+        fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+        fprintf(fp,"%s",center(block_id,max_len_block));
+        fprintf(fp,"    ");
+        fprintf(fp,"%s",center(rank->trt_name [ tptr[0] ] ,max_len_treat));
+        fprintf(fp,"    ");
+        fprintf(fp,"%s",center(rank->trt_name [ tptr[1] ] ,max_len_treat));
+        fprintf(fp,"    ");
+        fprintf(fp,"%s\n",center(diff_line,diff_col_len));
+        fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+        for (i=0; i<rep_width; i++)
+            fprintf(fp,"-");
+        fprintf(fp,"\n");
+    }
+    for (block=0; block <rank->n_blk ; block++){
+         if (verbose) {
+             fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+             fprintf(fp,"%s",center(rank->blk_name [ block ] ,max_len_block));
+             fprintf(fp,"    ");
+             fprintf(fp,pct_format,rank->pcts [ block ][ tptr[0] ] );
+             fprintf(fp,"    ");
+             fprintf(fp,pct_format,rank->pcts [ block ][ tptr[1] ] );
+             fprintf(fp,"    ");
+         }
+         if (fabs(rank->pcts [ block ][ tptr[0] ] - rank->pcts [ block ][ tptr[1] ] ) <=
+                  equal_thresh){
+             if (verbose) fprintf(fp,"%s\n",center("0",diff_col_len));
+             sum_equal++;
+         }
+         else if (rank->pcts [ block ][ tptr[0] ]  < rank->pcts [ block ][ tptr[1] ] ){
+             if (verbose) fprintf(fp,"%s\n",center("-",diff_col_len));
+             sum_minus++;
+         }
+         else {
+             if (verbose) fprintf(fp,"%s\n",center("+",diff_col_len));
+             sum_plus++;
+         }
+         sum_trt1 += rank->pcts [ block ][ tptr[0] ] ; 
+         sum_trt2 += rank->pcts [ block ][ tptr[1] ] ;
+    }
+    if (verbose){
+        fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+        for (i=0; i<rep_width; i++)
+            fprintf(fp,"-");
+        fprintf(fp,"\n");
+        /* an Average line */
+        fprintf(fp,"%s",center("",(paper_width - rep_width)/2));
+        fprintf(fp,"%s",center("Avg.",max_len_block));
+        fprintf(fp,"    ");
+        fprintf(fp,pct_format,sum_trt1 /rank->n_blk );
+        fprintf(fp,"    ");
+        fprintf(fp,pct_format,sum_trt2 /rank->n_blk );
+        fprintf(fp,"\n\n");
+
+        sprintf((char *)title_line,"No. Speakers with Positive %s Differences = N(+) = %2d",formula_str,sum_plus);
+        fprintf(fp,"%s\n",center((char *)title_line,paper_width));
+        sprintf((char *)title_line,"No. Speakers with Negative %s Differences = N(-) = %2d",formula_str,sum_minus);
+        fprintf(fp,"%s\n",center((char *)title_line,paper_width));
+        sprintf((char *)title_line,"No. Speakers with No %s Differences = N(0) = %2d",formula_str,sum_equal);
+        fprintf(fp,"%s\n",center((char *)title_line,paper_width));
+        fprintf(fp,"\n\n");
+    }
+
+    free_singarr(pct_format,char);
+    free_singarr(diff_line,char);
+    if (title_line != (TEXT *)0)
+        free_singarr(title_line, TEXT);
+    /* Analyze the Results */
+    { int result;
+      result = sign_test_analysis(sum_plus,sum_minus,sum_equal,"+","-",0,
+				  0.05,verbose, rank->trt_name [ tptr[0] ] ,
+				  rank->trt_name [ tptr[1] ] ,
+				  tptr,zero_is_best,fp,confidence);
+      /* if the result is significant, system which is better depends on if */
+      /* the treatments have been swapped, a negative result means tprt[0] is */
+      /* better, positive one tprt[1] is better */
+      return(result * ((tptr[0] == treat1) ? 1 : (-1)));
+    }
+}
+
+/****************************************************************/
+/*   Given the vital numbers for computing a rank sum test,     */
+/*   Compute it, and if requested, print a verbose analysis     */
+/****************************************************************/
+int sign_test_analysis(int num_a, int num_b, int num_z, char *str_a, char *str_b, int str_z, double alpha, int verbose, char *treat1_str, char *treat2_str, int *tptr, int zero_is_best, FILE *fp, double *confidence)
+{
+    double test_stat, p=0.5;
+    int i;
+
+    num_b += (num_z / 2) + (num_z % 2);
+    num_a += (num_z / 2);
+    num_z = 0;
+
+    /* multiplication by 2 means it's a two-tailed test */
+    if (num_a != num_b) 
+        test_stat = 2.0 * compute_acc_binomial(MIN(num_a,num_b),num_a+num_b,p);
+    else
+        test_stat = 1.0;
+    *confidence = test_stat;
+
+    if (verbose){
+        fprintf(fp,"The NULL Hypothesis:\n\n");
+        fprintf(fp,"     The number of speakers for which the differences is positive\n");
+        fprintf(fp,"     equals the number of speakers for which the differences is\n");
+        fprintf(fp,"     negative.\n");
+        fprintf(fp,"			 P(N(+)) = P(N(-)) = 0.50\n\n");
+
+        fprintf(fp,"Alternate Hypothesis:\n\n");
+        fprintf(fp,"     The number of speakers for which the differences is positive \n");
+        fprintf(fp,"     is NOT equal to the number of speakers for which the difference\n");
+        fprintf(fp,"     is negative.\n\n");
+        fprintf(fp,"Decision Analysis:\n\n");
+        fprintf(fp,"     Assumptions:\n");
+        fprintf(fp,"        A1: The distibrution of positive and negative differences\n");
+        fprintf(fp,"            follows the binomial distribution for N fair coin tosses.\n");
+        fprintf(fp,"\n");
+        fprintf(fp,"        A2: In order to resolve the complication caused by cases where the\n");
+        fprintf(fp,"            difference in Word Accuracy is zero, half of the cases will\n");
+        fprintf(fp,"            be assigned to N(+) and half to N(-).  In the event of an\n");
+        fprintf(fp,"            odd number of zero differences, N(-) will get one extra, this\n");
+        fprintf(fp,"            reduces the probability of there being a difference between\n");
+        fprintf(fp,"            the two systems.\n\n");
+        fprintf(fp,"     Rejection criterion:\n");
+        fprintf(fp,"        Reject the null hypothesis at the 95%% confidence level based\n");
+        fprintf(fp,"        on the following critical values table.  N is the number of\n");
+        fprintf(fp,"        speakers being compared and N(-) is the number of negative\n");
+        fprintf(fp,"        differences.\n\n");
+        /* print a table of critical values */
+        fprintf(fp,"          MIN(N(-),N(+))      P(MIN(N(-),N(+)) | N=%2d)\n",num_a+num_b);
+
+        fprintf(fp,"          --------------      ------------------------\n");
+        for (i=0; i <= (num_a+num_b)/2 && (i - 3) <= num_b; i++){
+	    double val = 2.0*compute_acc_binomial(i,num_a+num_b,p),
+	           valp1 = 2.0*compute_acc_binomial(i+1,num_a+num_b,p);
+	    if (val >= 0.0005)
+	      fprintf(fp,"          %3d                 %5.3f", i,val);
+	    else 
+	      fprintf(fp,"          %3d                <0.001  ", i);
+                   
+            if ((val < alpha) && (valp1 > alpha))
+                fprintf(fp,"  <---  Null Hypothesis rejected at or below this point\n");
+            else
+                fprintf(fp,"\n");
+        }
+        fprintf(fp,"\n");
+        fprintf(fp,"     Decision:\n");
+
+        fprintf(fp,"        There were N(-)=%d negative differences , the probability of\n",num_b);
+        fprintf(fp,"        it occurring is %5.3f, therefore the null hypothesis ",test_stat);
+        if (test_stat < alpha){            
+            fprintf(fp,"is REJECTED\n");
+            fprintf(fp,"        in favor of the Alternate Hypothesis.  Further, %s is the\n",
+                   (zero_is_best) ? treat2_str : treat1_str);
+            fprintf(fp,"        better System.\n");
+        } else{
+            fprintf(fp,"is ACCEPTED\n");
+            fprintf(fp,"        There is no statistical difference between %s and %s\n",treat1_str,treat2_str);
+        }
+        form_feed(fp);
+    }
+    if (test_stat < alpha){
+        if (0) fprintf(fp,"Returning Result %d\n",TEST_DIFF * ((zero_is_best) ? 1 : -1));
+        return(TEST_DIFF * ((zero_is_best) ? 1 : -1));
+      }
+    return(NO_DIFF);
+}
+

SCTK/src/sclite/slm_intf.c

@@ -0,0 +1,195 @@
+#include "sctk.h"
+
+
+/* 
+   File: slm_intf.c
+   Date: March 20, 1998
+   
+
+   This file contains interface functions to lookup probs from a language
+   model read in by the SLM toolkit libraries.
+   
+   The only function that is always defined is lookup_lm_word_weight.  If the
+   SLM is not included, then executing this function is an error. 
+   
+   */
+
+
+void lookup_lm_word_weight(ARC *arc, void *ptr){
+
+#ifdef WITH_SLM
+  WORD *tw = (WORD *)(arc->data);
+  static init = 1;
+  static ng_t ng;
+  static fb_info *fb_list;
+  WORD *w = (WORD *)arc->data, *w_m1, *w_m2;
+  double sum_weight, lu_prob, def_w;
+  int nprob;
+  ARC_LIST_ATOM *oarc_a, *oarc_b;
+  int dbg = 0;
+
+  if (init){
+    initialize_lm(&ng,&fb_list, (char *)ptr, dbg);    
+    init = 0;
+  }    
+
+#define W_func(_d) (log(1.0 / (_d)) * 1.4426950408889634074)
+
+  /* Set the default weight for this word */
+  def_w = (w->opt_del) ? 0.0 : 20.0;
+
+  nprob = 0, sum_weight = 0;
+
+  if (dbg) printf("w->value= %s\n",w->value);
+  /* search for the previous word */
+  if (! arc->from_node->start_state && ng.n > 1 && ! w->opt_del){
+    for (oarc_a = arc->from_node->in_arcs; oarc_a != NULL; oarc_a=oarc_a->next){
+      w_m1 = (WORD*)(oarc_a->arc->data);
+      
+      if (! oarc_a->arc->from_node->start_state && ng.n > 2){
+	for (oarc_b = oarc_a->arc->from_node->in_arcs; oarc_b != NULL; oarc_b=oarc_b->next){
+	  w_m2 = (WORD*)(oarc_b->arc->data);
+	  if (dbg) printf("      w-1= %s w-2= %s\n",w_m1->value,w_m2->value);
+	  sum_weight += 
+	    (tg_lookup(w, w_m1, w_m2, &ng, fb_list, &lu_prob, dbg)) ? W_func(lu_prob) : def_w;
+	  nprob ++;
+	}
+      } else {
+	w_m2 = (WORD *)0;
+	if (dbg) printf("      w-1= %s w-2= No-context\n",w_m1->value);
+	sum_weight += 
+	  (tg_lookup(w, w_m1, w_m2, &ng, fb_list, &lu_prob, dbg)) ? W_func(lu_prob) : def_w;
+	nprob ++;
+      }
+    }
+  } else {
+    w_m1 = w_m2 = (WORD *)0;
+    if (dbg) printf("      w-1= No-context w-2= No-context\n");
+    if (! w->opt_del)
+      sum_weight += 
+	(tg_lookup(w, w_m1, w_m2, &ng, fb_list, &lu_prob, dbg)) ? W_func(lu_prob) : def_w;
+    nprob ++;
+  }
+  if (dbg) printf("     AvgWeight=   \t%g\n",sum_weight/nprob);
+  tw->weight = sum_weight / nprob;
+#else
+  fprintf(scfp,"Error: a call to lookup_lm_word_weight() was made even though\n"
+	  "       the SLM toolkit has not been included in the compilation of SCTK\n");
+  exit(1);
+#endif
+}
+
+#ifdef WITH_SLM
+
+/*****************************************************************/
+/* given upto three WORDS, lookup the N-gram in a language model */
+
+int tg_lookup(WORD *w, WORD *w_m1, WORD *w_m2, ng_t *ng, fb_info *fb_list, double *prob, int dbg){
+    int32 h_lu, m1_lu, m2_lu;
+    id__t h_id, m_id[3];
+    int bo_case = 0, acl = 0;
+    int actual_context;
+
+    if (sih_lookup(ng->vocab_ht,
+		   (char *)((w->opt_del)?w->intern_value:w->value),&h_lu) == 0){
+      if (dbg) printf("   Head Word Not in Vocab\n");
+      *prob = 0.0;
+      return(0);
+    }
+    h_id = (id__t)h_lu;
+
+    if (w_m1 != (WORD *)0){
+      sih_lookup(ng->vocab_ht,
+		 (char *)((w_m1->opt_del)?w_m1->intern_value:w_m1->value),&m1_lu);
+      actual_context = 1;
+      m_id[0] = (id__t)m1_lu;
+
+      if (w_m2 != (WORD *)0){
+	sih_lookup(ng->vocab_ht,
+		   (char *)((w_m2->opt_del)?w_m2->intern_value:w_m2->value),&m2_lu);
+	actual_context = 2;
+	m_id[0] = (id__t)m2_lu;
+	m_id[1] = (id__t)m1_lu;
+      }
+    } else {
+      actual_context = 0;
+    }
+
+    *prob = calc_prob_of(h_id,m_id,actual_context,ng,(arpa_lm_t *)0,
+			fb_list,&bo_case,&acl,0);
+
+    if (dbg){
+      printf("           Texts: P(%s | ",(char *)w->value);
+      if (ng->n > 2) printf("%s ",(w_m2 == (WORD *)0) ? "UNK" : (char *)w_m2->value);
+      if (ng->n > 1) printf("%s",(w_m1 == (WORD *)0) ? "UNK" : (char *)w_m1->value);
+      printf(") = %g  bo_case ",*prob);
+      decode_bo_case(bo_case,acl,stdout);
+    }
+    return(1);
+}
+
+/*****************************************************************/
+/* Read in a lanagauge model, an set up the structures.          */
+
+void initialize_lm(ng_t *ng, fb_info **fb_list, char *lm_file, int dbg){
+  
+  if (lm_file != (char *)0){
+    if (dbg)   printf("Loading LM file '%s'\n",lm_file);
+    load_lm(ng,lm_file);       /* if it failes, it dies */
+    if (dbg) {
+      printf("LM read in -\n");
+      printf(" version: %d\n",ng->version);
+      printf(" #-gram order: %d\n",ng->n);
+      printf(" vocabulary size: %d\n",ng->vocab_size);
+    }
+    *fb_list = gen_fb_list(ng->vocab_ht,
+			  ng->vocab_size,
+			  ng->vocab,
+			  ng->context_cue,
+			  0, 0, 0, 0,
+			  "");
+    
+    /* This code works as a test, except the printout for bo_case is wrong for  
+      the trigram.  it output 3x2x1 instead of 3x2-1 */
+    
+    if (dbg){ 
+      int32 lu1, lu2, lu3; id__t h_id, m_id[4]; double prob;
+      int bo_case = 0, acl = 0;
+      
+      if (sih_lookup(ng->vocab_ht,"the",&lu1) == 0)
+	printf("   Head Word Not in Vocab\n");
+      m_id[0] = lu1;
+      h_id = lu1;
+      prob = calc_prob_of(h_id,m_id,0,ng,(arpa_lm_t *)0,*fb_list,&bo_case,&acl,0);
+      printf("     P(%s| ) = %g   bo_case ",ng->vocab[lu1],prob);
+      decode_bo_case(bo_case,acl,stdout);
+
+      if (ng->n > 1) {
+	if (sih_lookup(ng->vocab_ht,"dow",&lu2) == 0) 
+	  printf("   Head Word Not in Vocab\n");
+	m_id[0] = lu1;
+	h_id = lu2;
+	prob = calc_prob_of(h_id,m_id,1,ng,(arpa_lm_t *)0,*fb_list,&bo_case,&acl,0);
+	printf("     P(%s | the) = %g   bo_case ",ng->vocab[lu2],prob);
+	decode_bo_case(bo_case,acl,stdout); 
+	
+	if (ng->n > 2) {
+	  if (sih_lookup(ng->vocab_ht,"jones",&lu3) == 0)
+	    printf("   Head Word Not in Vocab\n");
+	  m_id[0] = lu1;
+	  m_id[1] = lu2;
+	  h_id = lu3;
+	  prob = calc_prob_of(h_id,m_id,2,ng,(arpa_lm_t *)0,*fb_list,&bo_case,&acl,0);
+	  printf("     P(jones | the dow) = %g   bo_case ",prob);
+	  decode_bo_case(bo_case,acl,stdout);
+	}
+      }
+    }
+    
+    
+  } else {
+    fprintf(stderr,"Error: no lm_file name passed to alignment\n");
+  }
+}
+
+#endif

SCTK/src/sclite/stm.h

@@ -0,0 +1,31 @@
+
+typedef struct stm_seg_struct{
+    TEXT *file;
+    TEXT *chan;
+    TEXT *spkr;
+    double t1;
+    double t2;
+    int  flag1;
+    TEXT *labels;
+    TEXT *text;
+} STM_SEG;
+
+typedef struct stm_struct{
+    int max;
+    int s;
+    int num;
+    STM_SEG *seg;
+} STM;
+
+STM *alloc_STM(int n);
+void expand_STM(STM *stm);
+void free_STM(STM *stm);
+void fill_STM(FILE *fp, STM *stm, char *fname, boolean *end_of_file, int case_sense, int *perr);
+void locate_STM_boundary(STM *stm, int start, int by_file, int by_chan, int *end);
+void dump_STM_words(STM *stm,int s, int e, char *file);
+void dump_STM(STM *stm, int s, int e);
+void read_stm_line(TEXT **buf, int *len, FILE *fp);
+void parse_stm_line(STM_SEG *seg, TEXT **buf_ptr, int *buf_len, int case_sense, int dbg);
+void free_STM_SEG(STM_SEG *seg);
+void convert_stm_to_word_list(char *file, char *words, int case_sense, int *num_ref);
+void fill_STM_structure(STM *stm, FILE *fp_stm, char *stm_file, int *stm_file_end, int case_sense);