Upload 416 files

.gitattributes

@@ -270,3 +270,9 @@ tools/mgiza/mgizapp/inst/hmmnorm filter=lfs diff=lfs merge=lfs -text
 tools/mgiza/mgizapp/inst/lib/libmgiza.a filter=lfs diff=lfs merge=lfs -text
 tools/mgiza/mgizapp/inst/mgiza filter=lfs diff=lfs merge=lfs -text
 tools/mgiza/mgizapp/lib/libmgiza.a filter=lfs diff=lfs merge=lfs -text
+tools/giza-pp/GIZA++-v2/GIZA++ filter=lfs diff=lfs merge=lfs -text
+tools/giza/d4norm filter=lfs diff=lfs merge=lfs -text
+tools/giza/GIZA++ filter=lfs diff=lfs merge=lfs -text
+tools/giza/GIZA++-v2/GIZA++ filter=lfs diff=lfs merge=lfs -text
+tools/giza/hmmnorm filter=lfs diff=lfs merge=lfs -text
+tools/giza/mgiza filter=lfs diff=lfs merge=lfs -text

tools/giza-pp/GIZA++-v2/ATables.cpp

@@ -0,0 +1,119 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "ATables.h" 
+#include "Globals.h"
+#include "myassert.h"
+#include "Parameter.h"
+
+GLOBAL_PARAMETER(bool,CompactADTable,"compactadtable","1: only 3-dimensional alignment table for IBM-2 and IBM-3",PARLEV_MODELS,1);
+GLOBAL_PARAMETER(float,amodel_smooth_factor,"model23SmoothFactor","smoothing parameter for IBM-2/3 (interpolation with constant)",PARLEV_SMOOTH,0.0);
+
+template <class VALTYPE>
+void amodel<VALTYPE>::printTable(const char *filename) const
+  // print amodel to file with the name filename (it'll be created or overwritten 
+  // format : for a table : 
+  //              aj j l m  val 
+  // where aj is source word pos, j target word pos, l source sentence length, 
+  //       m is target sentence length.
+  //          
+{
+  //return;
+  if (is_distortion)
+    cout << "Dumping distortion table (d) to file:" << filename <<'\n';
+  else 
+    cout << "Dumping alignment table (a) to file:" << filename <<'\n';
+
+  ofstream of(filename);
+  double ssum=0.0;
+  for(WordIndex l=0; l < MaxSentLength; l++)
+    for(WordIndex m=0;m<MaxSentLength;m++)
+      {
+	if( CompactADTable && l!=m )
+	  continue;
+	unsigned int L=((CompactADTable&&is_distortion)?MaxSentLength:(l+1))-1;
+	unsigned int M=((CompactADTable&&!is_distortion)?MaxSentLength:(m+1))-1;
+	if( is_distortion==0 )
+	  for(WordIndex j=1;j<=M; j++)
+	    {
+	      double sum=0.0;
+	      for(WordIndex i=0;i<=L; i++)
+		{
+		  VALTYPE x=getValue(i, j, L, M); 
+		  if( x>PROB_SMOOTH )
+		    {
+		      of << i << ' ' << j << ' ' << L << ' ' << M << ' ' << x << '\n';
+		      sum+=x;
+		    }
+		}
+	      ssum+=sum;
+	    }
+	else
+	  for(WordIndex i=0;i<=L;i++)
+	    {
+	      double sum=0.0;
+	      for(WordIndex j=1;j<=M;j++)
+		
+		{
+		  VALTYPE x=getValue(j, i, L, M);
+		  if( x>PROB_SMOOTH )
+		    {
+		      of << j << ' ' << i << ' ' << L << ' ' << M << ' ' << x << '\n';
+		      sum+=x;
+		    }
+		}
+	      ssum+=sum;
+	    }
+      }
+}
+
+extern short NoEmptyWord;
+
+template <class VALTYPE>
+void amodel<VALTYPE>::readTable(const char *filename)
+{
+  /* This function reads the a table from a file.
+     Each line is of the format:  aj j l m  val
+     where aj is the source word position, j the target word position, 
+     l the source sentence length, and m the target sentence length
+     
+     This function also works for a d table, where the positions
+     of aj and i are swapped.  Both the a and d tables are 4 dimensional
+     hashes; this function will simply read in the four values and keep
+     them in that order when hashing the fifth value.
+     NAS, 7/11/99
+  */
+  ifstream inf(filename);
+  cout << "Reading a/d table from " << filename << "\n";
+  if(!inf){
+    cerr << "\nERROR: Cannot open " << filename<<"\n";
+    return;
+  }
+  WordIndex w, x, l, m;
+  VALTYPE prob;
+  while(inf >> w >> x >> l >> m >> prob )
+    // the NULL word is added to the length
+    // of the sentence in the tables, but discount it when you write the tables. 
+    setValue(w, x, l, m, prob);
+}
+
+template class amodel<COUNT> ; 
+//template class amodel<PROB> ; 

tools/giza-pp/GIZA++-v2/ATables.h

@@ -0,0 +1,172 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* --------------------------------------------------------------------------*
+ *                                                                           *
+ * Module :ATables                                                           *
+ *                                                                           *
+ * Prototypes File: ATables.h                                                *
+ *                                                                           *
+ * Objective: Defines clases and methods for handling I/O for distortion  &  *
+ *            alignment tables.                                              *
+ *****************************************************************************/
+
+#ifndef _atables_h
+#define _atables_h 1
+
+#include "defs.h"
+#include <cassert>
+#include <iostream>
+#include <algorithm>
+#include <functional>
+#include <map>
+#include <set>
+#include "Vector.h"
+#include <utility>
+#include <fstream>
+#include "Array4.h"
+#include "myassert.h"
+#include "Globals.h"
+
+extern bool CompactADTable;
+extern float amodel_smooth_factor;
+extern short NoEmptyWord;
+
+/* ------------------- Class Defintions of amodel ---------------------------*/
+/* Class Name: amodel:
+   Objective: This defines the underlying data structure for distortiont prob.
+   and count tables. They are defined as a hash table. Each entry in the hash
+   table is the probability (d(j/l,m,i), where j is word target position, i is
+   source word position connected to it, m is target sentence length, and l is
+   source sentence length) or count collected for it. The  probability and the
+   count are represented as log integer probability as 
+   defined by the class LogProb .  
+
+  This class is used to represents a Tables (probabiliity) and d (distortion)
+  tables and also their corresponding count tables .
+  
+  *--------------------------------------------------------------------------*/
+
+inline int Mabs(int a)
+{
+  if(a<0)
+    return -a;
+  else
+    return a;
+}
+
+template <class VALTYPE>
+class amodel
+{
+ public:
+  Array4<VALTYPE> a;
+  bool is_distortion ; 
+  WordIndex MaxSentLength;
+  bool ignoreL, ignoreM;
+  VALTYPE get(WordIndex aj, WordIndex j, WordIndex l, WordIndex m)const
+    {
+      massert( (!is_distortion) || aj<=m );massert( (!is_distortion) || j<=l );massert( (!is_distortion) || aj!=0 );
+      massert( is_distortion    || aj<=l );massert( is_distortion    || j<=m );massert( (is_distortion) || j!=0 );
+      massert( l<MaxSentLength );massert( m<MaxSentLength );
+      return a.get(aj, j, (CompactADTable&&is_distortion)?MaxSentLength:(l+1),(CompactADTable&&!is_distortion)?MaxSentLength:(m+1));
+    }
+  static float smooth_factor;
+  amodel(bool flag)
+    : a(MAX_SENTENCE_LENGTH+1,0.0), is_distortion(flag), MaxSentLength(MAX_SENTENCE_LENGTH)
+    {}; 
+  VALTYPE&getRef(WordIndex aj, WordIndex j, WordIndex l, WordIndex m)
+    {
+      massert( (!is_distortion) || aj<=m );massert( (!is_distortion) || j<=l );
+      massert( is_distortion    || aj<=l );massert( is_distortion    || j<=m );massert( (is_distortion) || j!=0 );
+      massert( l<MaxSentLength );massert( m<MaxSentLength );
+      return a(aj, j, (CompactADTable&&is_distortion)?MaxSentLength:(l+1),(CompactADTable&&!is_distortion)?MaxSentLength:(m+1));
+    }
+  void setValue(WordIndex aj, WordIndex j, WordIndex l, WordIndex m, VALTYPE val)
+    {
+      getRef(aj, j, l, m)=val;
+    }
+  VALTYPE getValue(WordIndex aj, WordIndex j, WordIndex l, WordIndex m) const
+    {
+      if( is_distortion==0 )
+	return max(double(PROB_SMOOTH),amodel_smooth_factor/(l+1)+(1.0-amodel_smooth_factor)*get(aj, j, l, m));
+      else
+	return max(double(PROB_SMOOTH),amodel_smooth_factor/m+(1.0-amodel_smooth_factor)*get(aj, j, l, m));	
+    }
+  void printTable(const char* filename)const ; 
+  template<class COUNT>
+  void normalize(amodel<COUNT>& aTable)const
+    {
+      WordIndex i, j, l, m ;
+      COUNT total;
+      int nParam=0;
+      for(l=0;l<MaxSentLength;l++)
+	for(m=0;m<MaxSentLength;m++)
+	  {
+	    if( CompactADTable && l!=m )
+	      continue;
+	    unsigned int L=((CompactADTable&&is_distortion)?MaxSentLength:(l+1))-1;
+	    unsigned int M=((CompactADTable&&!is_distortion)?MaxSentLength:(m+1))-1;
+	    if( is_distortion==0 )
+	      for(j=1;j<=M; j++)
+		{
+		  total=0.0;
+		  for(i=0;i<=L;i++)
+		    {
+		      total+=get(i, j, L, M);
+		    }
+		  if( total )
+		    for(i=0;i<=L;i++)
+		      {
+			nParam++;
+			aTable.getRef(i, j, L, M)=get(i, j, L, M)/total;
+			massert(aTable.getRef(i,j,L,M)<=1.0);
+			if( NoEmptyWord&&i==0 )
+			  aTable.getRef(i,j,L,M)=0;
+		      }
+		}
+	    else
+	      for(i=0;i<=L;i++)
+		{
+		  total=0.0;
+		  for(j=1;j<=M;j++)
+		    total+=get(j, i, L, M);
+		  if( total )
+		    for(j=1;j<=M;j++)
+		      {
+			aTable.getRef(j, i, L, M)=amodel_smooth_factor/M+(1.0-amodel_smooth_factor)*get(j, i, L, M)/total;
+			nParam++;
+			massert(aTable.getRef(j,i,L,M)<=1.0);
+			if( NoEmptyWord&&i==0 )
+			  aTable.getRef(j,i,L,M)=0;
+		      }
+		}
+	  }
+      cout << "A/D table contains " << nParam << " parameters.\n";
+    }
+
+  void readTable(const char *filename); 
+  void clear()
+    {a.clear();}
+};
+
+/* ------------------- End of amodel Class Definitions ----------------------*/
+
+#endif

tools/giza-pp/GIZA++-v2/AlignTables.cpp

@@ -0,0 +1,44 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "AlignTables.h"
+
+bool alignmodel::insert(Vector<WordIndex>& aj, LogProb val)
+{
+  hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment >::iterator i;
+  i = a.find(aj);
+  if(i != a.end() || val <= 0)
+    return false ;
+  a.insert(pair<const Vector<WordIndex>, LogProb>(aj, val));
+  return true ;
+}
+ 
+
+LogProb alignmodel::getValue(Vector<WordIndex>& align) const
+{
+  const LogProb zero = 0.0 ;
+  hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment >::const_iterator i;
+  i = a.find(align);
+  if(i == a.end())
+    return zero;
+  else
+    return (*i).second;
+}

tools/giza-pp/GIZA++-v2/AlignTables.h

@@ -0,0 +1,118 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _aligntables_h
+#define _aligntables_h 1
+
+#include "defs.h"
+ 
+
+#include <cassert>
+ 
+#include <iostream>
+#include <algorithm>
+#include <functional>
+#include <map>
+#include <set>
+//#include <vector>
+#include "Vector.h"
+#include <utility>
+#include <math.h>
+#include <fstream>
+#include "transpair_model1.h"
+
+
+/* ----------------- Class Defintions for hashmyalignment --------------------
+   Objective: This class is used to define a hash mapping function to map
+   an alignment (defined as a vector of integers) into a hash key
+ ----------------------------------------------------------------------------*/
+
+class hashmyalignment : public unary_function< Vector<WordIndex>, size_t >
+{
+public:
+  size_t operator() (const Vector<WordIndex>& key) const
+    // to define the mapping function. it takes an alignment (a vector of
+    // integers) and it returns an integer value (hash key). 
+    {
+      WordIndex j ;
+      size_t s  ;
+      size_t key_sum = 0 ;
+      //      logmsg << "For alignment:" ;
+      for (j = 1 ; j < key.size() ; j++){
+	//	logmsg << " " << key[j] ;
+	key_sum += (size_t) (int) pow(double(key[j]), double((j % 6)+1));
+      }
+      //      logmsg << " , Key value was : " <<  key_sum;
+      s = key_sum % 1000000 ;      
+      //      logmsg << " h(k) = " << s << endl ;
+      return(s);
+    }
+};
+
+class equal_to_myalignment{
+  // returns true if two alignments are the same (two vectors have same enties)
+public:
+  bool operator()(const Vector<WordIndex> t1, 
+		  const Vector<WordIndex> t2) const
+    {WordIndex j ;
+    if (t1.size() != t2.size())
+      return(false);
+    for (j = 1 ; j < t1.size() ; j++)
+      if (t1[j] != t2[j])
+	return(false);
+    return(true);
+    }
+    
+};
+
+/* ---------------- End of Class Defnition for hashmyalignment --------------*/
+
+
+/* ------------------ Class Defintions for alignmodel -----------------------
+ Class Name: alignmodel
+ Objective: Alignments neighborhhoods (collection of alignments) are stored in
+ a hash table (for easy lookup). Each alignment vector is mapped into a hash 
+ key using the operator defined above.
+ *--------------------------------------------------------------------------*/
+
+class alignmodel{
+private:
+  hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment > a;
+private:
+  //  void erase(Vector<WordIndex>&);
+public:
+
+  // methods;
+
+  inline hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment >::iterator begin(void){return a.begin();} // begining of hash
+  inline hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment >::iterator end(void){return a.end();} // end of hash
+  inline const hash_map<Vector<WordIndex>, LogProb, hashmyalignment, equal_to_myalignment >& getHash() const {return a;}; // reference to hash table
+  bool insert(Vector<WordIndex>&, LogProb val=0.0); // add a alignmnet
+ //  void setValue(Vector<WordIndex>&, LogProb val); // not needed
+  LogProb getValue(Vector<WordIndex>&)const; // retrieve prob. of alignment
+  inline void clear(void){ a.clear();}; // clear hash table 
+  //  void printTable(const char* filename);
+  //inline void resize(WordIndex n) {a.resize(n);}; // resize table
+
+};
+
+/* -------------- End of alignmode Class Definitions ------------------------*/
+#endif

tools/giza-pp/GIZA++-v2/Array.h

@@ -0,0 +1,5 @@
+#ifndef GIZA_ARRAY_H_DEFINED
+#define GIZA_ARRAY_H_DEFINED
+#include "Vector.h"
+#define Array Vector
+#endif

tools/giza-pp/GIZA++-v2/Array2.h

@@ -0,0 +1,107 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/*--
+Array2: Implementation of a twodimensional checked array allowing for
+a specified underlieing one-dimensional data-structure.
+
+Franz Josef Och (30/07/99)
+--*/
+#ifndef CLASS_Array2_DEFINED
+#define CLASS_Array2_DEFINED
+
+#include "mystl.h"
+#include <string>
+#include <vector>
+
+template<class T, class Y=vector<T> > class Array2
+{
+private:
+  Y p;
+  //  short h1, h2;
+  unsigned int h1, h2;
+public:
+  Array2(unsigned int _h1, unsigned int _h2) 
+    : p(_h1*_h2), h1(_h1), h2(_h2) {}
+  Array2(unsigned int _h1, unsigned int _h2, const T&_init) 
+    : p(_h1*_h2, _init), h1(_h1), h2(_h2) {}
+  Array2() 
+    : h1(0), h2(0) {}
+  inline T &operator()(unsigned int i, unsigned int j)
+    { assert(i<h1);assert(j<h2);return p[i*h2+j]; }
+  inline const T&operator()(unsigned int i, unsigned int j) const
+    { assert(i<h1);assert(j<h2);return p[i*h2+j]; }
+  inline T get(unsigned int i, unsigned int j)
+    { assert(i<h1);assert(j<h2);return p[i*h2+j]; }
+  inline void set(unsigned int i, unsigned int j, T x)
+    { assert(i<h1);assert(j<h2);p[i*h2+j]=x; }
+  inline const T get(unsigned int i, unsigned int j) const
+    { assert(i<h1);assert(j<h2);return p[i*h2+j]; }
+  inline unsigned int getLen1() const
+    { return h1; }
+  inline unsigned int getLen2() const
+    { return h2; }
+
+inline T*begin(){ 
+  if( h1==0||h2==0)return 0;
+  return &(p[0]); 
+}
+inline T*end(){ 
+  if( h1==0||h2==0)return 0;
+  return &(p[0])+p.size(); 
+}
+
+  inline const T*begin()const{ return p.begin(); }
+  inline const T*end()const{return p.end();}
+
+  friend ostream&operator<<(ostream&out, const Array2<T, Y>&ar)
+    {
+      for(unsigned int i=0;i<ar.getLen1();i++)
+	{
+	  //out << i << ": ";
+	  for(unsigned int j=0;j<ar.getLen2();j++)
+	    out << ar(i, j) << ' ';
+	  out << '\n';
+	}
+      return out << endl;
+    }
+  inline void resize(unsigned int a,unsigned int b)
+    {
+      if( !(a==h1&&b==h2))
+	{
+	  h1=a;
+	  h2=b;
+	  p.resize(h1*h2);
+	}
+    }
+  inline void resize(unsigned int a,unsigned int b,const T&t)
+    {
+      if( !(a==h1&&b==h2))
+	{
+	  h1=a;
+	  h2=b;
+	  p.resize(h1*h2);
+	  fill(p.begin(),p.end(),t);
+	}
+    }
+};
+
+#endif

tools/giza-pp/GIZA++-v2/Array4.h

@@ -0,0 +1,78 @@
+/*
+
+Copyright (C) 2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef AlignmentArray4_h_DEFINED
+#define AlignmentArray4_h_DEFINED
+
+#include "Array2.h"
+template<class T> class Array4
+{
+ private:
+  Array2< Array2<T>* > A;
+  int M;
+  T init;
+ public:
+  Array4(int m,const T&_init)
+    : A(m,m,0),M(m),init(_init) {}
+  ~Array4()
+    {
+      for(int l=0;l<M;++l)
+	for(int m=0;m<M;++m)
+	  delete A(l,m);
+    }
+  const T&operator()(int i, int j, int l, int m)const
+    {
+      if( A(l,m)==0 )
+	return init;
+      else
+	return (*A(l,m))(i,j);
+    }
+  const T&get(int i, int j, int l, int m)const
+    {
+      if( A(l,m)==0 )
+	return init;
+      else
+	return (*A(l,m))(i,j);
+    }
+  T&operator()(int i, int j, int l, int m)
+    {
+      if( A(l,m)==0 )
+	{
+	  A(l,m)=new Array2<T>(max(l+1,m+1),max(l+1,m+1),init);
+	}
+      return (*A(l,m))(i,j);
+    }
+  void clear()
+    {
+      for(int l=0;l<M;++l)
+	for(int m=0;m<M;++m)
+	  if( A(l,m) )
+	    {
+	      Array2<T>&a=*A(l,m);
+	      for(int i=0;i<=l;++i)
+		for(int j=0;j<=m;++j)
+		  a(i,j)=0.0;
+	    }
+    }
+};
+
+#endif

tools/giza-pp/GIZA++-v2/D4Tables.h

@@ -0,0 +1,460 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _d4tables_h_define
+#define _d4tables_h_define
+#include <math.h>
+#include "WordClasses.h"
+#include "Globals.h"
+#include "myassert.h"
+
+extern float d4modelsmooth_factor;
+
+class m4_key
+{
+ public:
+  int deps;
+  int l;
+  int m;
+  int F;
+  int E;
+  int prevj;
+  int vacancies1,vacancies2;
+  m4_key(int _deps,int _l,int _m,int _F,int _E,int _prevj,int _v1,int _v2)
+    : deps(_deps),l(_l),m(_m),F(_F),E(_E),prevj(_prevj),vacancies1(_v1),vacancies2(_v2) {}
+  friend ostream&print1(ostream&out,const m4_key&x,const WordClasses&wce,const WordClasses&wcf)
+    {
+      if(x.deps&DEP_MODEL_l)out << "l: " << x.l<<' ';
+      if(x.deps&DEP_MODEL_m)out << "m: " << x.m<<' ';
+      if(x.deps&DEP_MODEL_F)out << "F: " << wcf.classString(x.F)<< ' ';
+      if(x.deps&DEP_MODEL_E)out << "E: " << wce.classString(x.E)<< ' ';
+      //      if(x.deps&DEP_MODEL_pj)out << "j-1: " << x.prevj<<' ';
+      if(x.vacancies1!=-1)out << "v1: " << x.vacancies1 << ' ';
+      if(x.vacancies2!=-1)out << "v2: " << x.vacancies2 << ' ';
+      return out << '\n';
+    }
+  friend ostream&print1_m5(ostream&out,const m4_key&x,const WordClasses&wce,const WordClasses&wcf)
+    {
+      out << ((x.deps&DEP_MODEL_E)?wce.classString(x.E):string("0"))<< ' ';
+      out << ((x.deps&DEP_MODEL_F)?wcf.classString(x.F):string("0"))<< ' ';
+      out << x.vacancies1 << ' ';
+      out << x.vacancies2 << ' ';
+      return out;
+    }
+  friend ostream&printb1(ostream&out,const m4_key&x,const WordClasses&wce,const WordClasses&wcf)
+    {
+      if(x.deps&DEP_MODELb_l)out << "l: " << x.l<<' ';
+      if(x.deps&DEP_MODELb_m)out << "m: " << x.m<<' ';
+      if(x.deps&DEP_MODELb_F)out << "F: " << wcf.classString(x.F) << ' ';
+      if(x.deps&DEP_MODELb_E)out << "E: " << wce.classString(x.E) << ' ';
+      if(x.vacancies1!=-1)out << "v1: " << x.vacancies1 << ' ';
+      if(x.vacancies2!=-1)out << "v2: " << x.vacancies2 << ' ';
+      return out << '\n';
+    }
+  friend ostream&printb1_m5(ostream&out,const m4_key&x,const WordClasses&wcf)
+    {
+      out << "-1 " << ((x.deps&DEP_MODEL_F)?wcf.classString(x.F):string("0"))<< ' ';
+      out << x.vacancies1 << ' ';
+      out << x.vacancies2 << ' ';
+      return out;
+    }
+};
+
+class compare1
+{
+ private:
+  int deps;
+ public:
+  compare1(int _deps) : deps(_deps) {}
+  bool operator()(const m4_key&a,const m4_key&b)const
+    {
+      if(deps&DEP_MODEL_l){if( a.l<b.l )return 1;if( b.l<a.l )return 0;}
+      if(deps&DEP_MODEL_m){if( a.m<b.m )return 1;if( b.m<a.m )return 0;}
+      if(deps&DEP_MODEL_F){if( a.F<b.F )return 1;if( b.F<a.F )return 0;}
+      if(deps&DEP_MODEL_E){if( a.E<b.E )return 1;if( b.E<a.E )return 0;}
+      //if(deps&DEP_MODEL_pj){if( a.prevj<b.prevj )return 1;if( b.prevj<a.prevj )return 0;}
+      if(a.vacancies1<b.vacancies1)return 1;if(b.vacancies1<a.vacancies1)return 0;
+      if(a.vacancies2<b.vacancies2)return 1;if(b.vacancies2<a.vacancies2)return 0;
+      return 0;
+    }
+};
+
+class compareb1
+{
+ private:
+  int deps;
+ public:
+  compareb1(int _deps) : deps(_deps) {}
+  bool operator()(const m4_key&a,const m4_key&b)const
+    {
+      if(deps&DEP_MODELb_l){if( a.l<b.l )return 1;if( b.l<a.l )return 0;}
+      if(deps&DEP_MODELb_m){if( a.m<b.m )return 1;if( b.m<a.m )return 0;}
+      if(deps&DEP_MODELb_F){if( a.F<b.F )return 1;if( b.F<a.F )return 0;}
+      if(deps&DEP_MODELb_E){if( a.E<b.E )return 1;if( b.E<a.E )return 0;}
+      //if(deps&DEP_MODELb_pj){if( a.prevJ<b.prevJ )return 1;if( b.prevJ<a.prevJ )return 0;}
+      if(a.vacancies1<b.vacancies1)return 1;if(b.vacancies1<a.vacancies1)return 0;
+      if(a.vacancies2<b.vacancies2)return 1;if(b.vacancies2<a.vacancies2)return 0;
+      return 0;
+    }
+};
+
+inline void tokenize(const string&in,Vector<string>&out)
+{
+  string s;
+  istringstream l(in);
+  while(l>>s)
+    out.push_back(s);
+}
+
+class d4model
+{
+ public:
+  typedef Vector<pair<COUNT,PROB> > Vpff;
+  map<m4_key,Vpff,compare1 > D1;
+  map<m4_key,Vpff,compareb1> Db1;
+  PositionIndex msl;
+  WordClasses ewordclasses;
+  WordClasses fwordclasses;
+  template<class MAPPER>
+  void makeWordClasses(const MAPPER&m1,const MAPPER&m2,string efile,string ffile)
+    {
+      ifstream estrm(efile.c_str()),fstrm(ffile.c_str());
+      if( !estrm )
+	{
+	  cerr << "ERROR: can not read " << efile << endl;
+	}
+      else
+	ewordclasses.read(estrm,m1);
+      if( !fstrm )
+	cerr << "ERROR: can not read " << ffile << endl;
+      else
+	fwordclasses.read(fstrm,m2);
+    }
+  d4model(PositionIndex _msl)
+    : D1(compare1(M4_Dependencies)),Db1(compareb1(M4_Dependencies)),msl(_msl)
+    {}
+  COUNT&getCountRef_first(WordIndex j,WordIndex j_cp,int E,int F,int l,int m)
+    {
+      assert(j>=1);
+      m4_key key(M4_Dependencies,l,m,F,E,j_cp,-1,-1);
+      map<m4_key,Vpff,compare1 >::iterator p=D1.find(key);
+      if(p==D1.end())p=D1.insert(make_pair(key,Vpff(msl*2+1,pair<COUNT,PROB>(0.0,0.0)))).first;
+      assert(p!=D1.end());
+      return (p->second)[j-j_cp+msl].first;
+    }
+  COUNT&getCountRef_bigger(WordIndex j,WordIndex j_prev,int E,int F,int l,int m)
+    {
+      assert(j>=1);
+      assert(j_prev>=1);
+      m4_key key(M4_Dependencies,l,m,F,E,j_prev,-1,-1);
+      map<m4_key,Vpff,compareb1 >::iterator p=Db1.find(key);
+      if(p==Db1.end())p=Db1.insert(make_pair(key,Vpff(msl*2+1,pair<COUNT,PROB>(0.0,0.0)))).first;
+      assert(p!=Db1.end());
+      return (p->second)[j-j_prev+msl].first;
+    }
+  map<m4_key,Vpff,compare1 >::const_iterator getProb_first_iterator(int E,int F,int l,int m)const
+    {return D1.find(m4_key(M4_Dependencies,l,m,F,E,0,-1,-1));}
+  PROB getProb_first_withiterator(WordIndex j,WordIndex j_cp,int m,const map<m4_key,Vpff,compare1 >::const_iterator& p)const
+    {
+      assert(j>=1);//assert(j_cp>=0);
+      assert(j<=msl);assert(j_cp<=msl);
+      if(p==D1.end())
+	{
+	return PROB_SMOOTH;
+	}
+      else
+	{
+	  massert((p->second)[j-j_cp+msl].second<=1.0);
+	  return max(PROB_SMOOTH,d4modelsmooth_factor/(2*m-1)+(1-d4modelsmooth_factor)*(p->second)[j-j_cp+msl].second);
+	}
+    }
+  PROB getProb_first(WordIndex j,WordIndex j_cp,int E,int F,int l,int m)const
+    {
+      assert(j>=1);//assert(j_cp>=0);
+      assert(j<=msl);assert(j_cp<=msl);
+      m4_key key(M4_Dependencies,l,m,F,E,j_cp,-1,-1);
+      map<m4_key,Vpff,compare1 >::const_iterator p=D1.find(key);
+      if(p==D1.end())
+	{
+	return PROB_SMOOTH;
+	}
+      else
+	{
+	  massert((p->second)[j-j_cp+msl].second<=1.0);
+	  return max(PROB_SMOOTH,d4modelsmooth_factor/(2*m-1)+(1-d4modelsmooth_factor)*(p->second)[j-j_cp+msl].second);
+	}
+    }
+  map<m4_key,Vpff,compareb1 >::const_iterator getProb_bigger_iterator(int E,int F,int l,int m)const
+    {
+      return Db1.find(m4_key(M4_Dependencies,l,m,F,E,0,-1,-1));
+    }
+  PROB getProb_bigger_withiterator(WordIndex j,WordIndex j_prev,int m,const map<m4_key,Vpff,compareb1 >::const_iterator&p)const
+    {
+      massert(j>=1);massert(j_prev>=1);
+      massert(j>j_prev);
+      massert(j<=msl);massert(j_prev<=msl);
+      if(p==Db1.end())
+	{
+	  return PROB_SMOOTH;
+	}
+      else
+	{
+	  massert((p->second)[j-j_prev+msl].second<=1.0 );
+	  return max(PROB_SMOOTH,d4modelsmooth_factor/(m-1)+(1-d4modelsmooth_factor)*(p->second)[j-j_prev+msl].second);
+	}
+    }
+    
+  PROB getProb_bigger(WordIndex j,WordIndex j_prev,int E,int F,int l,int m)const
+    {
+      massert(j>=1);massert(j_prev>=1);
+      massert(j>j_prev);
+      massert(j<=msl);massert(j_prev<=msl);
+      m4_key key(M4_Dependencies,l,m,F,E,j_prev,-1,-1);
+      map<m4_key,Vpff,compareb1 >::const_iterator p=Db1.find(key);
+      if(p==Db1.end())
+	{
+	  return PROB_SMOOTH;
+	}
+      else
+	{
+	  massert((p->second)[j-j_prev+msl].second<=1.0 );
+	  return max(PROB_SMOOTH,d4modelsmooth_factor/(m-1)+(1-d4modelsmooth_factor)*(p->second)[j-j_prev+msl].second);
+	}
+    }
+  void normalizeTable()
+    {
+      int nParams=0;
+      for(map<m4_key,Vpff,compare1 >::iterator i=D1.begin();i!=D1.end();++i)
+	{
+	  Vpff&d1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    sum+=d1[i].first;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    {
+	      d1[i].second=sum?(d1[i].first/sum):(1.0/d1.size());
+	      nParams++;
+	    }
+	}
+      for(map<m4_key,Vpff,compareb1 >::iterator i=Db1.begin();i!=Db1.end();++i)
+	{
+	  Vpff&db1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    sum+=db1[i].first;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    {
+	      db1[i].second=sum?(db1[i].first/sum):(1.0/db1.size());
+	      nParams++;
+	    }
+	}
+      cout << "D4 table contains " << nParams << " parameters.\n";      
+    }
+   void clear()
+    {
+      for(map<m4_key,Vpff,compare1 >::iterator i=D1.begin();i!=D1.end();++i)
+	{
+	  Vpff&d1=i->second;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    d1[i].first=0.0;
+	}
+      for(map<m4_key,Vpff,compareb1 >::iterator i=Db1.begin();i!=Db1.end();++i)
+	{
+	  Vpff&db1=i->second;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    db1[i].first=0.0;
+	}
+    }
+ 
+  void printProbTable(const char*fname1,const char*fname2)
+    {
+      ofstream out(fname1);
+      double ssum=0.0;
+      out << "# Translation tables for Model 4 .\n";
+      out << "# Table for head of cept.\n";
+      for(map<m4_key,Vpff,compare1 >::const_iterator i=D1.begin();i!=D1.end();++i)
+	{
+	  const Vpff&d1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex ii=0;ii<d1.size();ii++)sum+=d1[ii].first;
+	  if ( sum )
+	    {
+	      print1(out,i->first,ewordclasses,fwordclasses);
+	      out << "SUM: " << sum << ' '<< '\n';
+	      for(unsigned ii=0;ii<d1.size();ii++)
+		if( d1[ii].first )
+		  out << (int)(ii)-(int)(msl) << ' ' << d1[ii].first << '\n';
+	      out << endl;
+	    }
+	  ssum+=sum;
+	}
+      out << "# Table for non-head of cept.\n";
+      for(map<m4_key,Vpff,compareb1 >::const_iterator i=Db1.begin();i!=Db1.end();++i)
+	{
+	  const Vpff&db1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex ii=0;ii<db1.size();++ii)sum+=db1[ii].first;
+	  if( sum )
+	    {
+	      printb1(out,i->first,ewordclasses,fwordclasses);
+	      out << "SUM: " << sum << ' '<<'\n';
+	      for(unsigned ii=0;ii<db1.size();ii++)
+		if( db1[ii].first )
+		  {
+		    out << (int)(ii)-(int)(msl) << ' ' << db1[ii].first << '\n';
+		  }
+	      out << endl;
+	    }
+	  ssum+=sum;
+	}
+      out << endl << "FULL-SUM: " << ssum << endl;
+      if( M4_Dependencies==76 )
+	{
+	  ofstream out2(fname2);
+	  for(map<m4_key,Vpff,compare1 >::const_iterator i=D1.begin();i!=D1.end();++i)
+	    {
+	      const Vpff&d1=i->second;
+	      for(unsigned ii=0;ii<d1.size();ii++)
+		if( d1[ii].first )
+		  out2 << ewordclasses.classString(i->first.E) << ' ' << fwordclasses.classString(i->first.F) << ' ' << (int)(ii)-(int)(msl) << ' ' << d1[ii].second << '\n';
+	    }
+	  for(map<m4_key,Vpff,compareb1 >::const_iterator i=Db1.begin();i!=Db1.end();++i)
+	    {
+	      const Vpff&db1=i->second;
+	      for(unsigned ii=0;ii<db1.size();ii++)
+		if( db1[ii].first )
+		  out2 << -1 << ' ' << fwordclasses.classString(i->first.F) << ' ' << (int)(ii)-(int)(msl) << ' ' << db1[ii].second << '\n';
+	    }
+	}
+    }
+  bool readProbTable(const char *fname)
+    {
+      cerr << "Reading D4Tables from " << fname << endl;
+      ifstream file(fname);
+      string line;
+      do
+	{
+	  getline(file,line);
+	} while(line.length()&&line[0]=='#');
+      
+      do
+	{
+	  while(line.length()==0)
+	    getline(file,line);
+	  if( line[0]=='#')
+	    break;
+	  Vector<string> linestr;
+	  tokenize(line,linestr);
+	  m4_key k(M4_Dependencies,0,0,0,0,0,-1,-1);
+	  for(unsigned int i=0;i<linestr.size();i+=2)
+	    {
+	      if( linestr[i]=="l:" ){k.l=atoi(linestr[i+1].c_str());iassert(M4_Dependencies&DEP_MODEL_l);}
+	      if( linestr[i]=="m:" ){k.m=atoi(linestr[i+1].c_str());iassert(M4_Dependencies&DEP_MODEL_m);}
+	      if( linestr[i]=="F:" ){k.F=fwordclasses(linestr[i+1]);iassert(M4_Dependencies&DEP_MODEL_F);}
+	      if( linestr[i]=="E:" ){k.E=ewordclasses(linestr[i+1]);iassert(M4_Dependencies&DEP_MODEL_E);}
+	      //if( linestr[i]=="j-1:" ){k.prevj=atoi(linestr[i+1].c_str());iassert(M4_Dependencies&DEP_MODEL_pj);}
+	    }
+	  string str;
+	  double sum;
+	  file >> str >> sum;
+	  iassert(str=="SUM:");
+	  if( str!="SUM:")
+	    cerr << "ERROR: string is " << str << " and not sum " << endl;
+	  
+	  do
+	    {
+	      int value;
+	      double count;
+	      getline(file,line);
+	      istringstream twonumbers(line);
+	      if(twonumbers >> value >> count)
+		{
+		  if( D1.count(k)==0 )
+		    D1.insert(make_pair(k,Vpff(msl*2+1,pair<COUNT,PROB>(0.0,0.0))));
+		  D1[k][value+msl]=make_pair(count,count/sum);
+		}
+	    }while(line.length());
+	}while(file);
+      do
+	{
+	  getline(file,line);
+	} while(line.length()&&line[0]=='#');
+      do
+	{
+	  while(line.length()==0)
+	    getline(file,line);
+	  if( line[0]=='#')
+	    break;
+	  Vector<string> linestr;
+	  tokenize(line,linestr);
+	  m4_key k(M4_Dependencies,0,0,0,0,0,-1,-1);
+	  bool sumRead=0;
+	  for(unsigned int i=0;i<linestr.size();i+=2)
+	    {
+	      if( linestr[i]=="l:" ){k.l=atoi(linestr[i+1].c_str());iassert(M4_Dependencies&DEP_MODELb_l);}
+	      else if( linestr[i]=="m:" ){k.m=atoi(linestr[i+1].c_str());iassert(M4_Dependencies&DEP_MODELb_m);}
+	      else if( linestr[i]=="F:" ){k.F=fwordclasses(linestr[i+1]);iassert(M4_Dependencies&DEP_MODELb_F);}
+	      else if( linestr[i]=="E:" ){k.E=ewordclasses(linestr[i+1]);iassert(M4_Dependencies&DEP_MODELb_E);}
+	      else if( linestr[i]=="SUM:" )
+		{
+		  cerr << "Warning: obviously no dependency.\n";
+		  sumRead=1;
+		}
+	      else if( linestr[i]=="FULL-SUM:" )
+		{
+		  break;
+		}
+	      else
+		{
+		  cerr << "ERROR: error in reading d4 tables: " << linestr[i] << ' ' << linestr[i+1] << endl;
+		}
+	    }
+	  string str;
+	  double sum;
+	  if( sumRead==0 )
+	    file >> str >> sum;
+	  else
+	    {
+	      str=linestr[0];
+	      sum=atof(linestr[1].c_str());
+	    }
+	  if( str!="SUM:" )
+	    cerr << "ERROR: should read SUM but read " << str << endl;
+	  do
+	    {
+	      int value;
+	      double count;
+	      getline(file,line);
+	      istringstream twonumbers(line);
+	      if(twonumbers >> value >> count)
+		{
+		  if( Db1.count(k)==0 )
+		    Db1.insert(make_pair(k,Vpff(msl*2+1,pair<COUNT,PROB>(0.0,0.0))));
+		  Db1[k][value+msl]=make_pair(count,count/sum);
+		}
+	    }while(file&&line.length());
+	}while(file);
+      return 1;
+    }
+};
+
+#endif

tools/giza-pp/GIZA++-v2/D5Tables.h

@@ -0,0 +1,235 @@
+/*
+
+Copyright (C) 2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _d5tables_h_define
+#define _d5tables_h_define
+#include <math.h>
+#include "D4Tables.h"
+
+extern float d5modelsmooth_countoffset;
+extern float d5modelsmooth_factor;
+
+#define UNSEENPROB (1.0/vacancies_total)
+
+class d5model
+{
+ private:
+  typedef Vector < pair < COUNT,PROB > >Vpff;
+  map< m4_key,Vpff,compare1 > D1;
+  map< m4_key,Vpff,compareb1 > Db1;
+ public:
+  d4model&d4m;
+  WordClasses ewordclasses,fwordclasses;
+  template<class MAPPER>
+  void makeWordClasses(const MAPPER&m1,const MAPPER&m2,string efile,string ffile)
+    {
+      ifstream estrm(efile.c_str()),fstrm(ffile.c_str());
+      if( !estrm )
+	cerr << "ERROR: can not read classes from " << efile << endl;
+      else
+	ewordclasses.read(estrm,m1);
+      if( !fstrm )
+	cerr << "ERROR: can not read classes from " << ffile << endl;
+      else
+	fwordclasses.read(fstrm,m2);
+    }
+  d5model (d4model&_d4m)
+    :D1 (compare1(M5_Dependencies)), Db1 (compareb1(M5_Dependencies)),d4m(_d4m)
+    {}
+  COUNT &getCountRef_first (PositionIndex vacancies_j,
+			    PositionIndex vacancies_jp, int F,
+			    PositionIndex l, PositionIndex m,
+			    PositionIndex vacancies_total)
+  {
+    massert(vacancies_j>0);
+    massert(vacancies_total>0);
+    //massert(vacancies_jp<=vacancies_total);
+    massert(vacancies_j <=vacancies_total);
+    massert(vacancies_total<=m);
+    m4_key key(M5_Dependencies,l,m,F,0,0,vacancies_jp,vacancies_total);
+    map<m4_key,Vpff,compare1 >::iterator p=D1.find(key);
+    if(p==D1.end())
+      p=D1.insert(make_pair(key,Vpff(vacancies_total+1,make_pair(0,UNSEENPROB)))).first; // !!! constrain length
+    massert(p!=D1.end());
+    return (p->second)[vacancies_j].first;
+  }
+  COUNT &getCountRef_bigger (PositionIndex vacancies_j,
+			     PositionIndex vacancies_jp, int F,
+			     PositionIndex l, PositionIndex m,
+			     PositionIndex vacancies_total)
+  {
+    massert(vacancies_j>0);
+    massert(vacancies_total>0);
+    massert (vacancies_jp <= vacancies_j);
+    massert (vacancies_j-vacancies_jp <= vacancies_total);
+    m4_key key(M5_Dependencies,l,m,F,0,0,-1,vacancies_total);
+    map<m4_key,Vpff,compareb1 >::iterator p=Db1.find(key);
+    if(p==Db1.end())
+      p=Db1.insert(make_pair(key,Vpff(vacancies_total+1,make_pair(0,UNSEENPROB)))).first; // !!! constrain length
+    massert(p!=Db1.end());
+    return (p->second)[vacancies_j - vacancies_jp].first;
+  }
+  PROB getProb_first (PositionIndex vacancies_j, PositionIndex vacancies_jp,
+		       int F, PositionIndex l, PositionIndex m,
+		       PositionIndex vacancies_total) const
+  {
+    massert(vacancies_j>0);
+    massert(vacancies_total>0);
+    //massert(vacancies_jp<=vacancies_total);
+    massert(vacancies_j <=vacancies_total);
+    massert(vacancies_total<=m);
+    m4_key key(M5_Dependencies,l,m,F,0,0,vacancies_jp,vacancies_total);
+    map<m4_key,Vpff,compare1 >::const_iterator p=D1.find(key);
+    if( p==D1.end() )
+      return UNSEENPROB;
+    else
+      return max(PROB_SMOOTH,d5modelsmooth_factor/(vacancies_total)+(1-d5modelsmooth_factor)*(p->second)[vacancies_j].second);
+  }
+  PROB getProb_bigger (PositionIndex vacancies_j, PositionIndex vacancies_jp,
+			int F, PositionIndex l, PositionIndex m,
+			PositionIndex vacancies_total) const
+  {
+    massert(vacancies_j>0);
+    massert(vacancies_total>0);
+    massert (vacancies_jp <= vacancies_j);
+    massert (vacancies_j-vacancies_jp <= vacancies_total);
+    m4_key key(M5_Dependencies,l,m,F,0,0,-1,vacancies_total);
+    map<m4_key,Vpff,compareb1 >::const_iterator p=Db1.find(key);
+    if(p==Db1.end())
+      return UNSEENPROB;
+    else
+      return max(PROB_SMOOTH,d5modelsmooth_factor/(vacancies_total)+(1-d5modelsmooth_factor)*(p->second)[vacancies_j - vacancies_jp].second);
+  }
+  void normalizeTable ()
+    {
+      int nParams=0;
+      for(map<m4_key,Vpff,compare1 >::iterator i=D1.begin();i!=D1.end();++i)
+	{
+	  Vpff&d1=i->second;
+	  COUNT sum=0.0;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    sum+=d1[i].first+d5modelsmooth_countoffset;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    {
+	      d1[i].second=sum?((d1[i].first+d5modelsmooth_countoffset)/sum):(1.0/d1.size());
+	      nParams++;
+	    }
+	}
+      for(map<m4_key,Vpff,compareb1 >::iterator i=Db1.begin();i!=Db1.end();++i)
+	{
+	  Vpff&db1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    sum+=db1[i].first+d5modelsmooth_countoffset;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    {
+	      db1[i].second=sum?((db1[i].first+d5modelsmooth_countoffset)/sum):(1.0/db1.size());
+	      nParams++;
+	    }
+	}
+      cout << "D5 table contains " << nParams << " parameters.\n";
+    }
+  
+  friend ostream&operator<<(ostream&out,d5model&d5m)
+    {
+      out << "# Translation tables for Model 5 .\n";
+      out << "# Table for head of cept.\n";
+      for(map<m4_key,Vpff,compare1 >::const_iterator i=d5m.D1.begin();i!=d5m.D1.end();++i)
+	{
+	  const Vpff&d1=i->second;
+	  COUNT sum=0.0;
+	  for(PositionIndex ii=0;ii<d1.size();ii++)sum+=d1[ii].first;
+	  if ( sum )
+	    {
+	      for(unsigned ii=0;ii<d1.size();ii++)
+		{
+		  print1_m5(out,i->first,d5m.ewordclasses,d5m.fwordclasses);
+		  out << (int)(ii) << ' ' << d1[ii].second  << ' ' << d1[ii].first << '\n';
+		}
+	      out << endl;
+	    }
+	}
+      out << "# Table for non-head of cept.\n";
+      for(map<m4_key,Vpff,compareb1 >::const_iterator i=d5m.Db1.begin();i!=d5m.Db1.end();++i)
+	{
+	  const Vpff&db1=i->second;
+	  double sum=0.0;
+	  for(PositionIndex ii=0;ii<db1.size();++ii)sum+=db1[ii].first;
+	  if( sum )
+	    {
+	      for(unsigned ii=0;ii<db1.size();ii++)
+		{
+		  printb1_m5(out,i->first,d5m.fwordclasses);
+		  out << (int)(ii) << ' ' << db1[ii].second << ' ' << db1[ii].first << '\n';
+		}
+	      out << endl;
+	    }
+	}
+      return out;
+    }
+  void readProbTable(const char*x)
+    {
+      ifstream f(x);
+      string l;
+      while(getline(f,l))
+	{
+	  if(l.length()&&l[0]=='#')
+	    continue;
+	  istringstream is(l.c_str());
+	  string E,F;
+	  int v1,v2,ii;
+	  double prob,count;
+	  if(is>>E>>F>>v1>>v2>>ii>>prob>>count)
+	    {
+	      //cerr << "Read: " << E << " " << F << " " << v1 << " " << v2 << " " << prob<< endl;
+	      if( count>0 )
+		if( E=="-1")
+		  getCountRef_bigger(ii,0,fwordclasses(F),1000,1000,v2)+=count;
+		else
+		  getCountRef_first(ii,v1,fwordclasses(F),1000,1000,v2)+=count;
+	    }
+	}
+      normalizeTable();
+      ofstream of("M5FILE");
+      of << (*this);
+    }
+  void clear()
+    {
+      for(map<m4_key,Vpff,compare1 >::iterator i=D1.begin();i!=D1.end();++i)
+	{
+	  Vpff&d1=i->second;
+	  for(PositionIndex i=0;i<d1.size();i++)
+	    d1[i].first=0.0;
+	}
+      for(map<m4_key,Vpff,compareb1 >::iterator i=Db1.begin();i!=Db1.end();++i)
+	{
+	  Vpff&db1=i->second;
+	  for(PositionIndex i=0;i<db1.size();i++)
+	    db1[i].first=0.0;
+	}
+    }
+};
+
+#endif
+
+
+

tools/giza-pp/GIZA++-v2/Dictionary.cpp

@@ -0,0 +1,94 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* Noah A. Smith
+   Dictionary object for dictionary filter in Model 1 training
+
+   Dictionary file must be in order (sorted) by Foreign vocab id, but English
+   vocab ids may be in any order.
+
+   9 August 1999
+*/
+
+#include "Dictionary.h"
+
+#include <cstring>
+
+Dictionary::Dictionary(const char *filename){
+  if(!strcmp(filename, "")){
+    dead = true;
+    return;
+  }
+  dead = false;
+  cout << "Reading dictionary from:  " << filename << '\n';
+  ifstream dFile(filename);
+  if(!dFile){
+    cerr << "ERROR: Can't open dictionary: " << filename << '\n';
+    exit(1);
+  }
+
+  currindexmin = 0;
+  currindexmax = 0;
+  currval = 0;
+  int p, q;
+  while((dFile >> p >> q)){
+    pairs[0].push_back(p);
+    pairs[1].push_back(q);
+  }
+  cout << "Dictionary read; " << pairs[0].size() << " pairs loaded." << '\n';
+  dFile.close();
+}
+
+
+bool Dictionary::indict(int p, int q){
+  if(dead) return false;
+  if(p == 0 && q == 0) return false;
+  if(currval == p){
+    for(int i = currindexmin; i <= currindexmax; i++)
+      if(pairs[1][i] == q) return true;
+    return false;
+  }
+  else{
+    int begin = 0, end = pairs[0].size() - 1, middle = 0;
+    unsigned int t;
+    bool ret = false;
+    while(begin <= end){
+      middle = begin + ((end - begin) >> 1);
+      if(p < pairs[0][middle]) end = middle - 1;
+      else if(p > pairs[0][middle]) begin = middle + 1;
+      else{
+	break;
+      }
+    }
+    t = middle;
+    while(pairs[0][t] == p )
+      if(pairs[1][t--] == q) ret = true;
+    currindexmin = t + 1;
+    t = middle + 1;
+    while(pairs[0][t] == p && t < pairs[0].size())
+      if(pairs[1][t++] == q) ret = true;
+    currindexmax = t - 1;
+    currval = p;
+    return ret;
+  }
+}
+  
+

tools/giza-pp/GIZA++-v2/Dictionary.h

@@ -0,0 +1,48 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* Noah A. Smith
+   Dictionary object for dictionary filter in Model 1 training
+
+   9 August 1999
+*/
+
+#include <iostream>
+#include <fstream>
+
+#include "Vector.h"
+
+#ifndef DICTIONARY_H
+#define DICTIONARY_H
+
+class Dictionary{
+ private:
+  Vector<int> pairs[2];
+  int currval;
+  int currindexmin;
+  int currindexmax;
+  bool dead;
+ public:
+  Dictionary(const char *);
+  bool indict(int, int);
+};
+
+#endif

tools/giza-pp/GIZA++-v2/FlexArray.h

@@ -0,0 +1,58 @@
+/*
+
+Copyright (C) 1988,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef CLASS_FlexArray_defined
+#define CLASS_FlexArray_defined
+#include "Array.h"
+#include <iostream>
+#include <fstream>
+template<class T>
+class FlexArray
+{
+private:
+  Array<T> p;
+  int start,End;
+public:
+  FlexArray(int _start=0,int _end=-1)
+    : p(_end-_start+1),start(_start),End(_end) {}
+  FlexArray(int _start,int _end,const T&init)
+    : p(_end-_start+1,init),start(_start),End(_end) {}
+  T&operator[](int i)
+    {return p[i-start];}
+  const T&operator[](int i)const
+    {return p[i-start];}
+  int low()const{return start;}
+  int high()const{return End;}
+  T*begin(){return conv<double>(p.begin());}
+  T*end(){return conv<double>(p.end());}
+};
+
+template<class T>
+inline ostream&operator<<(ostream&out,const FlexArray<T>&x)
+{
+  for(int i=x.low();i<=x.high();++i)
+    out << i << ':' << x[i] << ';' << ' ';
+  return out;
+}
+
+
+#endif

tools/giza-pp/GIZA++-v2/ForwardBackward.cpp

@@ -0,0 +1,242 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef NO_TRAINING
+#include "ForwardBackward.h"
+#include "Globals.h"
+#include "myassert.h"
+#include "HMMTables.h"
+#include "mymath.h"
+
+
+double ForwardBackwardTraining(const HMMNetwork&net,Array<double>&g,Array<Array2<double> >&E){
+  const int I=net.size1(),J=net.size2(),N=I*J;
+  Array<double> alpha(N,0),beta(N,0),sum(J);
+  for(int i=0;i<I;i++)
+    beta[N-I+i]=net.getBetainit(i);
+  double * cur_beta=conv<double>(beta.begin())+N-I-1;
+  for(int j=J-2;j>=0;--j)
+    for(int ti=I-1;ti>=0;--ti,--cur_beta) {
+      const double *next_beta=conv<double>(beta.begin())+(j+1)*I;
+      const double *alprob=&net.outProb(j,ti,0),*next_node=&net.nodeProb(0,j+1);
+      for(int ni=0;ni<I;++ni,(next_node+=J)){
+	massert(cur_beta<next_beta&& &net.outProb(j,ti,ni)==alprob);
+	massert(next_node == &net.nodeProb(ni,j+1));
+	/*	if( VERB&&(*next_beta)*(*alprob)*(*next_node) )
+	  cout << "B= " << (int)(cur_beta-beta.begin()) << " += " << (*next_beta) << "(" 
+	  << next_beta-beta.begin() << ") alprob:" << (*alprob) << "  lexprob:" << (*next_node) << endl;*/
+	(*cur_beta)+=(*next_beta++)*(*alprob++)*(*next_node);
+      }
+    }
+  for(int i=0;i<I;i++)  
+    alpha[i]=net.getAlphainit(i)*net.nodeProb(i,0);
+  double* cur_alpha=conv<double>(alpha.begin())+I;
+  cur_beta=conv<double>(beta.begin())+I;
+  for(int j=1;j<J;j++){
+    Array2<double>&e=E[ (E.size()==1)?0:(j-1) ];
+    if( (E.size()!=1) || j==1 )
+      {
+	e.resize(I,I);
+	fill(e.begin(),e.end(),0.0);
+      }
+    
+    for(int ti=0;ti<I;++ti,++cur_alpha,++cur_beta) {
+      const double * prev_alpha=conv<double>(alpha.begin())+I*(j-1);
+      double *cur_e= &e(ti,0);
+      double this_node=net.nodeProb(ti,j);
+      const double* alprob= &net.outProb(j-1,0,ti);
+      for(int pi=0;pi<I;++pi,++prev_alpha,(alprob+=I)){
+	massert(prev_alpha<cur_alpha&& &net.outProb(j-1,pi,ti)==alprob);
+	massert(&e(ti,pi)==cur_e);
+	const double alpha_increment= *prev_alpha*(*alprob)*this_node;
+	(*cur_alpha)+=alpha_increment;
+	(*cur_e++)+=alpha_increment*(*cur_beta);
+      }
+    }
+  }
+  g.resize(N);
+  transform(alpha.begin(),alpha.end(),beta.begin(),g.begin(),multiplies<double>());
+  double bsum=0,esum=0,esum2;
+  for(int i=0;i<I;i++)
+    bsum+=beta[i]*net.nodeProb(i,0)*net.getAlphainit(i);
+  for(unsigned int j=0;j<(unsigned int)E.size();j++)
+    {
+      Array2<double>&e=E[j];
+      const double *epe=e.end();
+      for(const double*ep=e.begin();ep!=epe;++ep)
+	esum+=*ep;
+    }
+  if( J>1 ) 
+    esum2=esum/(J-1);
+  else
+    esum2=0.0;
+  if(!(esum2==0.0||mfabs(esum2-bsum)/bsum<1e-3*I))
+    cout << "ERROR2: " << esum2 <<" " <<bsum << " " << esum << net << endl;
+  double * sumptr=conv<double>(sum.begin());
+  double* ge=conv<double>(g.end());
+  for(double* gp=conv<double>(g.begin());gp!=ge;gp+=I)
+    {
+      *sumptr++=normalize_if_possible(gp,gp+I);
+      if(bsum && !(mfabs((*(sumptr-1)-bsum)/bsum)<1e-3*I))
+	cout << "ERROR: " << *(sumptr-1) << " " << bsum << " " << mfabs((*(sumptr-1)-bsum)/bsum) << ' ' << I << ' ' << J << endl;
+    }
+  for(unsigned int j=0;j<(unsigned int)E.size();j++)
+    {
+      Array2<double>&e=E[j];
+      double* epe=e.end();
+      if( esum )
+	for(double*ep=e.begin();ep!=epe;++ep)
+	  *ep/=esum;
+      else
+	for(double*ep=e.begin();ep!=epe;++ep)
+	  *ep/=1.0/(max(I*I,I*I*(J-1)));
+    }
+  if( sum.size() )
+    return sum[0];
+  else
+    return 1.0;
+}
+void HMMViterbi(const HMMNetwork&net,Array<int>&vit) {
+  const int I=net.size1(),J=net.size2();
+  vit.resize(J);
+  Array<double>g;
+  Array<Array2<double> >e(1);
+  ForwardBackwardTraining(net,g,e);
+  for(int j=0;j<J;j++) {
+    double * begin=conv<double>(g.begin())+I*j;
+    vit[j]=max_element(begin,begin+I)-begin;
+  }
+}
+void HMMViterbi(const HMMNetwork&net,Array<double>&g,Array<int>&vit) {
+  const int I=net.size1(),J=net.size2();
+  vit.resize(J);
+  for(int j=0;j<J;j++) {
+    double* begin=conv<double>(g.begin())+I*j;
+    vit[j]=max_element(begin,begin+I)-begin;
+  }
+}
+
+double HMMRealViterbi(const HMMNetwork&net,Array<int>&vitar,int pegi,int pegj,bool verbose){
+  const int I=net.size1(),J=net.size2(),N=I*J;
+  Array<double> alpha(N,-1);
+  Array<double*> bp(N,(double*)0);  
+  vitar.resize(J);
+  if( J==0 )
+    return 1.0;
+  for(int i=0;i<I;i++)
+    {
+      alpha[i]=net.getAlphainit(i)*net.nodeProb(i,0);
+      if( i>I/2 ) 
+	alpha[i]=0; // only first empty word can be chosen
+      bp[i]=0;
+    }
+  double *cur_alpha=conv<double>(alpha.begin())+I;
+  double **cur_bp=conv<double*>(bp.begin())+I;
+  for(int j=1;j<J;j++)
+    {
+      if( pegj+1==j)
+	for(int ti=0;ti<I;ti++)
+	  if( (pegi!=-1&&ti!=pegi)||(pegi==-1&&ti<I/2) )
+	    (cur_alpha-I)[ti]=0.0;
+      for(int ti=0;ti<I;++ti,++cur_alpha,++cur_bp) {
+	double* prev_alpha=conv<double>(alpha.begin())+I*(j-1);
+	double this_node=net.nodeProb(ti,j);
+	const double *alprob= &net.outProb(j-1,0,ti);
+	for(int pi=0;pi<I;++pi,++prev_alpha,(alprob+=I)){
+	  massert(prev_alpha<cur_alpha&& &net.outProb(j-1,pi,ti)==alprob);
+	  const double alpha_increment= *prev_alpha*(*alprob)*this_node;
+	  if( alpha_increment> *cur_alpha )
+	    {
+	      (*cur_alpha)=alpha_increment;
+	      (*cur_bp)=prev_alpha;
+	    }
+	}
+      }
+    }
+  for(int i=0;i<I;i++)
+    alpha[N-I+i]*=net.getBetainit(i);
+  if( pegj==J-1)
+    for(int ti=0;ti<I;ti++)
+      if( (pegi!=-1&&ti!=pegi)||(pegi==-1&&ti<I/2) )
+	(alpha)[N-I+ti]=0.0;
+
+  int j=J-1;
+  cur_alpha=conv<double>(alpha.begin())+j*I;
+  vitar[J-1]=max_element(cur_alpha,cur_alpha+I)-cur_alpha;
+  double ret= *max_element(cur_alpha,cur_alpha+I);
+  while(bp[vitar[j]+j*I])
+    {
+      cur_alpha-=I;
+      vitar[j-1]=bp[vitar[j]+j*I]-cur_alpha;
+      massert(vitar[j-1]<I&&vitar[j-1]>=0);
+      j--;
+    }
+  massert(j==0);
+  if( verbose )
+    {
+      cout << "VERB:PEG: " << pegi << ' ' << pegj << endl;
+      for(int j=0;j<J;j++)
+	cout << "NP " << net.nodeProb(vitar[j],j) << ' ' << "AP " << ((j==0)?net.getAlphainit(vitar[j]):net.outProb(j-1,vitar[j-1],vitar[j])) << " j:" << j << " i:" << vitar[j] << ";  ";
+      cout << endl;
+    }
+  return ret;
+}
+
+double MaximumTraining(const HMMNetwork&net,Array<double>&g,Array<Array2<double> >&E){
+  Array<int> vitar;
+  double ret=HMMRealViterbi(net,vitar);
+  const int I=net.size1(),J=net.size2();
+  if( E.size()==1 )
+    {
+      Array2<double>&e=E[0];
+      e.resize(I,I);
+      g.resize(I*J);
+      fill(g.begin(),g.end(),0.0);
+      fill(e.begin(),e.end(),0.0);
+      for(int i=0;i<J;++i)
+	{
+	  g[i*I+vitar[i]]=1.0;
+	  if( i>0 )
+	    e(vitar[i],vitar[i-1])++;
+	}
+    }
+  else
+    {
+      g.resize(I*J);
+      fill(g.begin(),g.end(),0.0);
+      for(int i=0;i<J;++i)
+	{
+	  g[i*I+vitar[i]]=1.0;
+	  if( i>0 )
+	    {
+	      Array2<double>&e=E[i-1];
+	      e.resize(I,I);
+	      fill(e.begin(),e.end(),0.0);
+	      e(vitar[i],vitar[i-1])++;
+	    }
+	}
+    }    
+  return ret;
+}
+
+#endif
+

tools/giza-pp/GIZA++-v2/ForwardBackward.h

@@ -0,0 +1,62 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef NO_EM_MARKOF_ZEUGS_DEFINED
+#define NO_EM_MARKOF_ZEUGS_DEFINED
+#ifndef NO_TRAINING
+#include "myassert.h"
+#include "Array.h"
+#include "Array2.h"
+
+class HMMNetwork
+{
+ public:
+  int as,bs;
+  Array2<double> n;
+  Array<Array2<double> > e;
+  Array<double> alphainit;
+  Array<double> betainit;
+  int ab;
+  double finalMultiply;
+  HMMNetwork(int I,int J)
+    : as(I),bs(J),n(as,bs),/*e(as,as,0.0),*/e(0),alphainit(as,1.0/as),betainit(as,1.0),ab(as*bs),finalMultiply(1.0)
+    {}
+  double getAlphainit(int i)const{return alphainit[i];}
+  double getBetainit(int i)const{return betainit[i];}
+  inline int size1()const{return as;}
+  inline int size2()const{return bs;}
+  inline const double&nodeProb(int i,int j)const
+    {return n(i,j);}
+  inline const double&outProb(int j,int i1,int i2)const
+    {/*massert(e[min(int(e.size())-1,j)](i1,i2) );*/ return e[min(int(e.size())-1,j)](i1,i2);}
+  friend ostream&operator<<(ostream&out,const HMMNetwork&x)
+    {
+      return out <<"N: \n"<< x.n << endl << "E: \n" << x.e << "A:\n" << x.alphainit << "B:\n" << x.betainit << endl;
+    }
+};
+double ForwardBackwardTraining(const HMMNetwork&mc,Array<double>&gamma,Array<Array2<double> >&epsilon);
+void HMMViterbi(const HMMNetwork&mc,Array<int>&vit);
+double HMMRealViterbi(const HMMNetwork&net,Array<int>&vit,int pegi=-1,int pegj=-1,bool verbose=0);
+double MaximumTraining(const HMMNetwork&net,Array<double>&g,Array<Array2<double> >&e);
+void HMMViterbi(const HMMNetwork&net,Array<double>&g,Array<int>&vit);
+#endif
+#endif

tools/giza-pp/GIZA++-v2/GIZA++

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cd1757193a60c612d5eae91cd457399e43dfa45a036fa20ec2b11cfda5915f7
+size 1139144

tools/giza-pp/GIZA++-v2/GNU.GPL

@@ -0,0 +1,282 @@
+
+
+Preamble
+
+The licenses for most software are designed to take away your freedom
+to share and change it. By contrast, the GNU General Public License is
+intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the
+rights. These restrictions translate to certain responsibilities for
+you if you distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on,
+we want its recipients to know that what they have is not the
+original, so that any problems introduced by others will not reflect
+on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at
+all.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License applies to any program or other work which contains a
+notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the Program
+(independent of having been made by running the Program). Whether that
+is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the Program's source
+code as you receive it, in any medium, provided that you conspicuously
+and appropriately publish on each copy an appropriate copyright notice
+and disclaimer of warranty; keep intact all the notices that refer to
+this License and to the absence of any warranty; and give any other
+recipients of the Program a copy of this License along with the
+Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a
+fee.
+
+2. You may modify your copy or copies of the Program or any portion of
+it, thus forming a work based on the Program, and copy and distribute
+such modifications or work under the terms of Section 1 above,
+provided that you also meet all of these conditions:
+
+     a) You must cause the modified files to carry prominent notices
+     stating that you changed the files and the date of any change.
+
+     b) You must cause any work that you distribute or publish, that
+     in whole or in part contains or is derived from the Program or
+     any part thereof, to be licensed as a whole at no charge to all
+     third parties under the terms of this License.
+
+     c) If the modified program normally reads commands interactively
+     when run, you must cause it, when started running for such
+     interactive use in the most ordinary way, to print or display an
+     announcement including an appropriate copyright notice and a
+     notice that there is no warranty (or else, saying that you
+     provide a warranty) and that users may redistribute the program
+     under these conditions, and telling the user how to view a copy
+     of this License. (Exception: if the Program itself is interactive
+     but does not normally print such an announcement, your work based
+     on the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+     a) Accompany it with the complete corresponding machine-readable
+     source code, which must be distributed under the terms of
+     Sections 1 and 2 above on a medium customarily used for software
+     interchange; or,
+
+     b) Accompany it with a written offer, valid for at least three
+     years, to give any third party, for a charge no more than your
+     cost of physically performing source distribution, a complete
+     machine-readable copy of the corresponding source code, to be
+     distributed under the terms of Sections 1 and 2 above on a medium
+     customarily used for software interchange; or,
+
+     c) Accompany it with the information you received as to the offer
+     to distribute corresponding source code. (This alternative is
+     allowed only for noncommercial distribution and only if you
+     received the program in object code or executable form with such
+     an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt otherwise
+to copy, modify, sublicense or distribute the Program is void, and
+will automatically terminate your rights under this License. However,
+parties who have received copies, or rights, from you under this
+License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted
+herein. You are not responsible for enforcing compliance by third
+parties to this License.
+
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+9. The Free Software Foundation may publish revised and/or new
+versions of the General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation. If the Program does not specify a
+version number of this License, you may choose any version ever
+published by the Free Software Foundation.
+
+10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the
+author to ask for permission. For software which is copyrighted by the
+Free Software Foundation, write to the Free Software Foundation; we
+sometimes make exceptions for this. Our decision will be guided by the
+two goals of preserving the free status of all derivatives of our free
+software and of promoting the sharing and reuse of software generally.
+
+NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE
+LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS
+AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF
+ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+END OF TERMS AND CONDITIONS

tools/giza-pp/GIZA++-v2/Globals.h

@@ -0,0 +1,73 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef Globals_asdf_defined
+#define Globals_asdf_defined
+#include <string>
+#include <fstream>
+#include <map>
+#include "defs.h"
+#include "Vector.h"
+
+extern float PROB_SMOOTH;
+extern bool Verbose, Log, Peg, Transfer, Transfer2to3, useDict ;
+extern string Prefix, LogFilename, OPath, 
+  SourceVocabFilename, TargetVocabFilename, CorpusFilename, TestCorpusFilename, 
+  t_Filename, a_Filename, p0_Filename, d_Filename, n_Filename, dictionary_Filename;
+extern ofstream logmsg ;
+extern double M5P0,P0 ;
+extern bool NODUMPS, FEWDUMPS  ;
+extern string Usage ;
+extern unsigned int MAX_SENTENCE_LENGTH ;
+extern int PegUntil;
+
+extern short DeficientDistortionForEmptyWord;
+
+extern int M4_Dependencies;
+extern int M5_Dependencies;
+
+extern short OutputInAachenFormat;
+
+#define DEP_MODEL_l 1
+#define DEP_MODEL_m 2
+#define DEP_MODEL_F 4
+#define DEP_MODEL_E 8
+
+#define DEP_MODELb_l 16
+#define DEP_MODELb_m 32
+#define DEP_MODELb_F 64
+#define DEP_MODELb_E 128
+
+#define DEP_SUM 256
+
+class vcbList;
+
+extern vcbList *globeTrainVcbList, *globfTrainVcbList;
+
+extern short PredictionInAlignments;
+extern short SmoothHMM;
+#define VERB Verbose
+
+double ErrorsInAlignment(const map< pair<int,int>,char >&reference,const Vector<WordIndex>&test,int l,int&missing,int&toomuch,int&eventsMissing,int&eventsToomuch,int);
+extern Vector<map< pair<int,int>,char > > ReferenceAlignment;
+void printGIZAPars(ostream&out);
+
+#endif

tools/giza-pp/GIZA++-v2/HMMTables.cpp

@@ -0,0 +1,177 @@
+/*
+
+Copyright (C) 1998,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "HMMTables.h"
+#include <fstream>
+#include "Globals.h"
+#include "Parameter.h"
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+void HMMTables<CLS,MAPPERCLASSTOSTRING>::writeJumps(ostream&out) const
+{
+  double ssum=0.0;
+  for(typename map<AlDeps<CLS>,FlexArray<double> >::const_iterator i=alProb.begin();i!=alProb.end();++i)
+    {
+      double sum=0.0;
+      out << "\n\nDistribution for: ";
+      printAlDeps(out,i->first,*mapper1,*mapper2);
+      out << ' ';
+      for(int a=i->second.low();a<=i->second.high();++a)
+	if( i->second[a] )
+	  {
+	    out << a << ':' << i->second[a] << ';' << ' ';
+	    sum+=i->second[a];
+	  }
+      out << '\n' << '\n';
+      out << "SUM: " << sum << '\n';
+      ssum+=sum;
+    }
+  out << "FULL-SUM: " << ssum << '\n';
+}
+template<class CLS,class MAPPERCLASSTOSTRING>
+void HMMTables<CLS,MAPPERCLASSTOSTRING>::readJumps(istream&)
+{
+}
+template<class CLS,class MAPPERCLASSTOSTRING>
+double HMMTables<CLS,MAPPERCLASSTOSTRING>::getAlProb(int istrich,int k,int sentLength,int J,CLS w1,CLS w2,int j,int iter) const
+{
+  massert(k<sentLength&&k>=0);
+  massert(istrich<sentLength&&istrich>=-1);
+  int pos=istrich-k;
+  switch(PredictionInAlignments)
+    {
+    case 0: pos=istrich-k; break;
+    case 1: pos=k; break;
+    case 2: 
+      pos=(k*J-j*sentLength);
+      if( pos>0 ) pos+=J/2; else pos-=J/2;
+      pos/=J;
+      break;
+    default:abort();
+    }
+  typename map<AlDeps<CLS>,FlexArray<double> >::const_iterator p=alProb.find(AlDeps<CLS>(sentLength,istrich,j,w1,w2));
+  if( p!=alProb.end() )
+    {
+      return (p->second)[pos];
+    }
+  else
+    {
+      if( iter>0&&iter<5000 )
+      	cout << "WARNING: Not found: " << ' ' << J << ' ' << sentLength << '\n';;
+     return 1.0/(2*sentLength-1); 
+    }
+}
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+void HMMTables<CLS,MAPPERCLASSTOSTRING>::addAlCount(int istrich,int k,int sentLength,int J,CLS w1,CLS w2,int j,double value,double valuePredicted)
+{
+  int pos=istrich-k;
+  switch(PredictionInAlignments)
+    {
+    case 0: pos=istrich-k; break;
+    case 1: pos=k; break;
+    case 2:
+      pos=(k*J-j*sentLength);
+      if( pos>0 ) pos+=J/2; else pos-=J/2;
+      pos/=J;
+      break;
+    default:abort();
+    }
+  AlDeps<CLS> deps(AlDeps<CLS>(sentLength,istrich,j,w1,w2));
+
+  {
+    typename map<AlDeps<CLS>,FlexArray<double> >::iterator p=alProb.find(deps);
+    if( p==alProb.end() ) 
+      {
+	if( (CompareAlDeps&1)==0 )
+	  p=alProb.insert(make_pair(deps,FlexArray<double> (-MAX_SENTENCE_LENGTH,MAX_SENTENCE_LENGTH,0.0))).first;
+	else
+	  p=alProb.insert(make_pair(deps,FlexArray<double> (-sentLength,sentLength,0.0))).first;
+      }
+    p->second[pos]+=value;
+  }
+
+  if( valuePredicted )
+    {
+      typename map<AlDeps<CLS>,FlexArray<double> >::iterator p=alProbPredicted.find(deps);
+      if( p==alProbPredicted.end() ) 
+	{
+	  if( (CompareAlDeps&1)==0 )
+	    p=alProbPredicted.insert(make_pair(deps,FlexArray<double> (-MAX_SENTENCE_LENGTH,MAX_SENTENCE_LENGTH,0.0))).first;
+	  else
+	    p=alProbPredicted.insert(make_pair(deps,FlexArray<double> (-sentLength,sentLength,0.0))).first;
+	}
+      p->second[pos]+=valuePredicted;
+    }
+}
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+Array<double>&HMMTables<CLS,MAPPERCLASSTOSTRING>::doGetAlphaInit(int I)
+{
+  if( !init_alpha.count(I) )
+    init_alpha[I]=Array<double>(I,0);
+  return init_alpha[I];
+}
+template<class CLS,class MAPPERCLASSTOSTRING>
+Array<double>&HMMTables<CLS,MAPPERCLASSTOSTRING>::doGetBetaInit(int I)
+{
+  if( !init_beta.count(I) )
+    init_beta[I]=Array<double>(I,0);
+  return init_beta[I];
+}
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+bool HMMTables<CLS,MAPPERCLASSTOSTRING>::getAlphaInit(int I,Array<double>&x)const
+{
+  hash_map<int,Array<double> >::const_iterator i=init_alpha.find(I);
+  if( i==init_alpha.end() )
+    return 0;
+  else
+    {
+      x=i->second;
+      for(unsigned int j=x.size()/2+1;j<x.size();++j) // only first empty word can be chosen
+	x[j]=0;
+      return 1;
+    }
+}
+template<class CLS,class MAPPERCLASSTOSTRING>
+bool HMMTables<CLS,MAPPERCLASSTOSTRING>::getBetaInit(int I,Array<double>&x)const
+{
+  hash_map<int,Array<double> >::const_iterator i=init_beta.find(I);
+  if( i==init_beta.end() )
+    return 0;
+  else
+    {
+      x=i->second;
+      return 1;
+    }
+}
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+HMMTables<CLS,MAPPERCLASSTOSTRING>::  HMMTables(double _probForEmpty,const MAPPERCLASSTOSTRING&m1,const MAPPERCLASSTOSTRING&m2): 
+  probabilityForEmpty(mfabs(_probForEmpty)),
+  updateProbabilityForEmpty(_probForEmpty<0.0),
+  mapper1(&m1),
+  mapper2(&m2)
+{}
+template<class CLS,class MAPPERCLASSTOSTRING>
+HMMTables<CLS,MAPPERCLASSTOSTRING>::~HMMTables() {}

tools/giza-pp/GIZA++-v2/HMMTables.h

@@ -0,0 +1,172 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef HMM_TABLES_H_ASDF_DEFINED
+#define HMM_TABLES_H_ASDF_DEFINED
+#include "FlexArray.h"
+
+#include "Array.h"
+#include <map>
+#include "mymath.h"
+
+template<class T>
+T normalize_if_possible(T*a,T*b)
+{
+  T sum=0;
+  for(T*i=a;i!=b;++i)
+    sum+=*i;
+  if( sum )
+    for(T*i=a;i!=b;++i)
+      *i/=sum;
+  else
+    fill(a,b,1.0/(b-a));
+  return sum;
+}
+
+extern short CompareAlDeps;
+template<class CLS>
+class AlDeps
+{
+ public:
+  int englishSentenceLength;
+  CLS classPrevious;
+  int previous;
+  int j;
+  CLS Cj;
+  AlDeps(int l,int p=0,int _j=0,CLS s1=0,CLS _Cj=0)
+    : englishSentenceLength(l),classPrevious(s1),previous(p),j(_j),Cj(_Cj)
+    {}
+  friend bool operator<(const AlDeps&x,const AlDeps&y)
+  {
+    if( (CompareAlDeps&1) && x.englishSentenceLength<y.englishSentenceLength ) return 1;
+    if( (CompareAlDeps&1) && y.englishSentenceLength<x.englishSentenceLength ) return 0;
+    if( (CompareAlDeps&2) && x.classPrevious<y.classPrevious ) return 1;
+    if( (CompareAlDeps&2) && y.classPrevious<x.classPrevious ) return 0;
+    if( (CompareAlDeps&4) && x.previous<y.previous ) return 1;
+    if( (CompareAlDeps&4) && y.previous<x.previous ) return 0;
+    if( (CompareAlDeps&8) && x.j<y.j ) return 1;
+    if( (CompareAlDeps&8) && y.j<x.j ) return 0;
+    if( (CompareAlDeps&16) && x.Cj<y.Cj ) return 1;
+    if( (CompareAlDeps&16) && y.Cj<x.Cj ) return 0;
+    return 0;
+  }
+  friend bool operator==(const AlDeps&x,const AlDeps&y)
+  { return !( x<y || y<x );  }
+};
+
+template<class CLS>
+class Hash_AlDeps
+{
+ public:
+  unsigned 
+    int 
+    operator()
+    (const AlDeps<CLS>&x)
+    const
+    {
+      unsigned int hash=0;
+      if( (CompareAlDeps&1) ) { hash=hash+x.englishSentenceLength;hash*=31;}
+      if( (CompareAlDeps&2) ) { hash=hash+x.classPrevious;hash*=31;}
+      if( (CompareAlDeps&4) ) { hash=hash+x.previous;hash*=31;}
+      if( (CompareAlDeps&8) ) { hash=hash+x.j;hash*=31;}
+      if( (CompareAlDeps&16) ) { hash=hash+x.Cj;hash*=31;}
+      return hash;
+      
+    }
+};
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+class HMMTables
+{
+ protected:
+  double probabilityForEmpty;
+  bool updateProbabilityForEmpty;
+  hash_map<int,Array<double> > init_alpha;
+  hash_map<int,Array<double> > init_beta;
+  map<AlDeps<CLS>,FlexArray<double> > alProb;
+  map<AlDeps<CLS>,FlexArray<double> > alProbPredicted;
+  int globalCounter;
+  double divSum;
+  double p0_count,np0_count;
+  const MAPPERCLASSTOSTRING*mapper1;
+  const MAPPERCLASSTOSTRING*mapper2;
+ public:
+  const HMMTables<CLS,MAPPERCLASSTOSTRING>*getThis()const {return this;}
+  HMMTables(double _probForEmpty,const MAPPERCLASSTOSTRING&m1,const MAPPERCLASSTOSTRING&m2);
+  virtual ~HMMTables();
+  virtual double getAlProb(int i,int k,int sentLength,int J,CLS w1,CLS w2,int j,int iter=0) const;
+  virtual void writeJumps(ostream&) const;
+  void addAlCount(int i,int k,int sentLength,int J,CLS w1,CLS w2,int j,double value,double valuePredicted);
+  virtual void readJumps(istream&);
+  virtual bool getAlphaInit(int I,Array<double>&x)const;
+  virtual bool getBetaInit(int I,Array<double>&x)const;
+  Array<double>&doGetAlphaInit(int I);
+  Array<double>&doGetBetaInit(int I);
+  virtual double getProbabilityForEmpty()const
+    {return probabilityForEmpty;}  
+  void performGISIteration(const HMMTables<CLS,MAPPERCLASSTOSTRING>*old)
+    {
+      cout << "OLDSIZE: " << (old?(old->alProb.size()):0) << " NEWSIZE:"<< alProb.size()<< endl;
+      for(typename map<AlDeps<CLS>,FlexArray<double> >::iterator i=alProb.begin();i!=alProb.end();++i)
+	{
+	  if( alProbPredicted.count(i->first))
+	    {
+	      normalize_if_possible(i->second.begin(),i->second.end());
+	      normalize_if_possible(alProbPredicted[i->first].begin(),alProbPredicted[i->first].end());
+	      for(int j=i->second.low();j<=i->second.high();++j)
+		{
+		  if( i->second[j] )
+		    if(alProbPredicted[i->first][j]>0.0 )
+		      {
+			double op=1.0;
+			if( old && old->alProb.count(i->first) )
+			  op=(old->alProb.find(i->first)->second)[j];
+			//cerr << "GIS: " << j << ' ' << " OLD:"
+			//     << op << "*true:" 
+			//     << i->second[j] << "/pred:" << alProbPredicted[i->first][j] << " -> ";
+			i->second[j]= op*(i->second[j]/alProbPredicted[i->first][j]);
+			//cerr << i->second[j] << endl;
+		      }
+		    else
+		      {
+			cerr << "ERROR2 in performGISiteration: " << i->second[j] << endl;
+		      }
+		}
+	    }
+	  else
+	    cerr << "ERROR in performGISIteration: " << alProbPredicted.count(i->first) << endl;
+	}
+    }
+};
+
+template<class CLS,class MAPPERCLASSTOSTRING>
+inline void printAlDeps(ostream&out,const AlDeps<CLS>&x,const MAPPERCLASSTOSTRING&mapper1,const MAPPERCLASSTOSTRING&mapper2)
+{
+  if( (CompareAlDeps&1) ) out << "sentenceLength: " << x.englishSentenceLength<< ' ';
+  if( (CompareAlDeps&2) ) out << "previousClass: " << mapper1.classString(x.classPrevious) << ' ';
+  if( (CompareAlDeps&4) ) out << "previousPosition: " << x.previous << ' ';
+  if( (CompareAlDeps&8) ) out << "FrenchPosition: " << x.j << ' ';
+  if( (CompareAlDeps&16) ) out << "FrenchClass: " << mapper2.classString(x.Cj) << ' ';
+  //out << '\n';
+}
+
+#endif

tools/giza-pp/GIZA++-v2/LICENSE

@@ -0,0 +1,282 @@
+
+
+Preamble
+
+The licenses for most software are designed to take away your freedom
+to share and change it. By contrast, the GNU General Public License is
+intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the
+rights. These restrictions translate to certain responsibilities for
+you if you distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on,
+we want its recipients to know that what they have is not the
+original, so that any problems introduced by others will not reflect
+on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at
+all.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License applies to any program or other work which contains a
+notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the Program
+(independent of having been made by running the Program). Whether that
+is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the Program's source
+code as you receive it, in any medium, provided that you conspicuously
+and appropriately publish on each copy an appropriate copyright notice
+and disclaimer of warranty; keep intact all the notices that refer to
+this License and to the absence of any warranty; and give any other
+recipients of the Program a copy of this License along with the
+Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a
+fee.
+
+2. You may modify your copy or copies of the Program or any portion of
+it, thus forming a work based on the Program, and copy and distribute
+such modifications or work under the terms of Section 1 above,
+provided that you also meet all of these conditions:
+
+     a) You must cause the modified files to carry prominent notices
+     stating that you changed the files and the date of any change.
+
+     b) You must cause any work that you distribute or publish, that
+     in whole or in part contains or is derived from the Program or
+     any part thereof, to be licensed as a whole at no charge to all
+     third parties under the terms of this License.
+
+     c) If the modified program normally reads commands interactively
+     when run, you must cause it, when started running for such
+     interactive use in the most ordinary way, to print or display an
+     announcement including an appropriate copyright notice and a
+     notice that there is no warranty (or else, saying that you
+     provide a warranty) and that users may redistribute the program
+     under these conditions, and telling the user how to view a copy
+     of this License. (Exception: if the Program itself is interactive
+     but does not normally print such an announcement, your work based
+     on the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+     a) Accompany it with the complete corresponding machine-readable
+     source code, which must be distributed under the terms of
+     Sections 1 and 2 above on a medium customarily used for software
+     interchange; or,
+
+     b) Accompany it with a written offer, valid for at least three
+     years, to give any third party, for a charge no more than your
+     cost of physically performing source distribution, a complete
+     machine-readable copy of the corresponding source code, to be
+     distributed under the terms of Sections 1 and 2 above on a medium
+     customarily used for software interchange; or,
+
+     c) Accompany it with the information you received as to the offer
+     to distribute corresponding source code. (This alternative is
+     allowed only for noncommercial distribution and only if you
+     received the program in object code or executable form with such
+     an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt otherwise
+to copy, modify, sublicense or distribute the Program is void, and
+will automatically terminate your rights under this License. However,
+parties who have received copies, or rights, from you under this
+License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted
+herein. You are not responsible for enforcing compliance by third
+parties to this License.
+
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+9. The Free Software Foundation may publish revised and/or new
+versions of the General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation. If the Program does not specify a
+version number of this License, you may choose any version ever
+published by the Free Software Foundation.
+
+10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the
+author to ask for permission. For software which is copyrighted by the
+Free Software Foundation, write to the Free Software Foundation; we
+sometimes make exceptions for this. Our decision will be guided by the
+two goals of preserving the free status of all derivatives of our free
+software and of promoting the sharing and reuse of software generally.
+
+NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE
+LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS
+AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF
+ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+END OF TERMS AND CONDITIONS

tools/giza-pp/GIZA++-v2/Makefile

@@ -0,0 +1,140 @@
+.SUFFIXES: .out .o .c .e .r .f .y .l .s .p .cpp .alpha2o .pentiumo .sgio .alphao
+
+INSTALLDIR ?= /usr/local/bin/
+
+#CXX = g++
+
+CFLAGS = $(CFLAGS_GLOBAL) -Wall -Wno-parentheses
+#CFLAGS_OPT = $(CFLAGS) -O3 -DNDEBUG -DWORDINDEX_WITH_4_BYTE -O3 -DNDEBUG -DWORDINDEX_WITH_4_BYTE -ffast-math
+CFLAGS_OPT = $(CFLAGS) -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE -DWORDINDEX_WITH_4_BYTE
+CFLAGS_PRF = $(CFLAGS) -O2 -pg -DNDEBUG -DWORDINDEX_WITH_4_BYTE
+CFLAGS_DBG = $(CFLAGS) -g -DDEBUG -DWORDINDEX_WITH_4_BYTE
+CFLAGS_NRM = $(CFLAGS) -DWORDINDEX_WITH_4_BYTE
+CFLAGS_VDBG = $(CFLAGS) -g -DDEBUG -DWORDINDEX_WITH_4_BYTE -DVDEBUG
+SRC = *.cpp
+TYPE = 
+LDFLAGS =
+
+include Makefile.src
+
+OBJ_DIR_PRF = profile/
+OBJ_DIR_OPT = optimized/
+OBJ_DIR_DBG = debug/
+OBJ_DIR_VDBG = vdebug/
+OBJ_DIR_NRM = norm/
+OBJ_OPT2 = ${SRC2:%.cpp=$(OBJ_DIR_OPT)%.o}
+OBJ_OPT = ${SRC:%.cpp=$(OBJ_DIR_OPT)%.o}
+OBJ_DBG = ${SRC:%.cpp=$(OBJ_DIR_DBG)%.o}
+OBJ_VDBG = ${SRC:%.cpp=$(OBJ_DIR_VDBG)%.o}
+OBJ_NRM = ${SRC:%.cpp=$(OBJ_DIR_NRM)%.o}
+OBJ_PRF = ${SRC:%.cpp=$(OBJ_DIR_PRF)%.o}
+OBJ_DIR = 
+DATE = `date +%d-%m-%Y`
+
+opt: GIZA++ snt2plain.out plain2snt.out snt2cooc.out
+
+GIZA++: $(OBJ_DIR_OPT) $(OBJ_OPT) 
+	$(CXX) $(CFLAGS_OPT) $(OBJ_OPT) $(LDFLAGS) -o GIZA++
+
+prf: GIZA++.prf
+
+GIZA++.prf: $(OBJ_DIR_PRF) $(OBJ_PRF) 
+	$(CXX) $(CFLAGS_PRF) $(OBJ_PRF) -o GIZA++.prf $(LDFLAGS)
+
+dbg: GIZA++.dbg
+
+GIZA++.dbg: $(OBJ_DIR_DBG) $(OBJ_DBG) 
+	$(CXX) $(CFLAGS_DBG) $(OBJ_DBG) -o GIZA++.dbg $(LDFLAGS)
+
+vdbg: GIZA++.vdbg
+
+GIZA++.vdbg: $(OBJ_DIR_VDBG) $(OBJ_VDBG) 
+	$(CXX) $(CFLAGS_VDBG) $(OBJ_VDBG) -o GIZA++.vdbg $(LDFLAGS)
+
+nrm: GIZA++.nrm 
+
+GIZA++.nrm: $(OBJ_DIR_NRM) $(OBJ_NRM) 
+	$(CXX) $(CFLAGS_NRM) $(OBJ_NRM) -o GIZA++.nrm $(LDFLAGS)
+
+all: dbg opt nrm prf 
+
+$(OBJ_DIR_PRF): $(OBJ_DIR)
+	-mkdir $(OBJ_DIR_PRF)
+
+$(OBJ_DIR_OPT): $(OBJ_DIR)
+	-mkdir $(OBJ_DIR_OPT)
+
+$(OBJ_DIR_DBG): $(OBJ_DIR)
+	-mkdir $(OBJ_DIR_DBG)
+
+$(OBJ_DIR_VDBG): $(OBJ_DIR)
+	-mkdir $(OBJ_DIR_VDBG)
+
+$(OBJ_DIR_NRM): $(OBJ_DIR)
+	-mkdir $(OBJ_DIR_NRM)
+
+$(OBJ_DIR):
+	-mkdir $(OBJ_DIR)
+
+$(OBJ_DIR_DBG)%.o: %.cpp
+	$(CXX)  $(CFLAGS_DBG)  -c $< -o $@
+
+$(OBJ_DIR_VDBG)%.o: %.cpp
+	$(CXX)  $(CFLAGS_VDBG)  -c $< -o $@
+
+$(OBJ_DIR_NRM)%.o: %.cpp
+	$(CXX)  $(CFLAGS_NRM)  -c $< -o $@
+
+$(OBJ_DIR_PRF)%.o: %.cpp
+	$(CXX)  $(CFLAGS_PRF) -c $< -o $@
+
+$(OBJ_DIR_OPT)%.o: %.cpp
+	$(CXX)  $(CFLAGS_OPT)  -c $< -o $@
+
+iinstall: opt prf dbg
+	-mkdir $(INSTALLDIR)/$(ARCH)
+	-cp GIZA++ $(INSTALLDIR)/GIZA++
+	-cp GIZA++.prf $(INSTALLDIR)/GIZA++.prf
+	-cp GIZA++.dbg $(INSTALLDIR)/GIZA++.dbg
+
+install: opt 
+	-mkdir $(INSTALLDIR)
+	-cp GIZA++ $(INSTALLDIR)/GIZA++
+
+clean:
+	-rm -f $(OBJ_DIR_NRM)/*.o $(OBJ_DIR_DBG)/*.o $(OBJ_DIR_VDBG)/*.o $(OBJ_DIR_PRF)/*.o $(OBJ_DIR_OPT)/*.o
+	-rm -rf $(OBJ_DIR_NRM) $(OBJ_DIR_DBG) $(OBJ_DIR_VDBG) $(OBJ_DIR_PRF) $(OBJ_DIR_OPT)
+	-rm -f snt2plain.out plain2snt.out snt2cooc.out GIZA++
+
+
+backup: clean
+	tar cf - . | gzip -9 > ../GIZA++src.tar.gz
+
+depend: depend_CLEAN dependencies
+
+depend_CLEAN:
+	rm dependencies
+
+dependencies:
+	@(echo "#Automatically generated dependecy list" >>  dependencies ;\
+	$(CXX) -MM *.cpp $(CFLAGS_OPT) | perl -e 'while(<>){s?^([^\:]+\.o:)?$(OBJ_DIR_OPT)\1?g;print;}'>>  dependencies)
+	@(echo "#Automatically generated dependecy list" >>  dependencies ;\
+	$(CXX) -MM *.cpp $(CFLAGS_DBG) | perl -e 'while(<>){s?^([^\:]+\.o:)?$(OBJ_DIR_DBG)\1?g;print;}'>>  dependencies)
+	@(echo "#Automatically generated dependecy list" >>  dependencies ;\
+	$(CXX) -MM *.cpp $(CFLAGS_VDBG) | perl -e 'while(<>){s?^([^\:]+\.o:)?$(OBJ_DIR_VDBG)\1?g;print;}'>>  dependencies)
+	@(echo "#Automatically generated dependecy list" >>  dependencies ;\
+	$(CXX) -MM *.cpp $(CFLAGS_NRM) | perl -e 'while(<>){s?^([^\:]+\.o:)?$(OBJ_DIR_NRM)\1?g;print;}'>>  dependencies)
+	@(echo "#Automatically generated dependecy list" >>  dependencies ;\
+	$(CXX) -MM *.cpp $(CFLAGS_PRF) | perl -e 'while(<>){s?^([^\:]+\.o:)?$(OBJ_DIR_PRF)\1?g;print;}'>>  dependencies)
+
+-include dependencies
+
+snt2plain.out: snt2plain.cpp
+	$(CXX) $(LDFLAGS) -O3 -W -Wall snt2plain.cpp -o snt2plain.out
+
+plain2snt.out: plain2snt.cpp
+	$(CXX) $(LDFLAGS) -O3 -W -Wall plain2snt.cpp -o plain2snt.out
+
+snt2cooc.out: snt2cooc.cpp
+	$(CXX) $(LDFLAGS) -O3 -g -W -Wall snt2cooc.cpp -o snt2cooc.out
+

tools/giza-pp/GIZA++-v2/Makefile.src

@@ -0,0 +1,2 @@
+SRC = Parameter.cpp myassert.cpp Perplexity.cpp model1.cpp model2.cpp model3.cpp getSentence.cpp TTables.cpp ATables.cpp AlignTables.cpp main.cpp NTables.cpp model2to3.cpp collCounts.cpp alignment.cpp vocab.cpp MoveSwapMatrix.cpp transpair_model3.cpp transpair_model5.cpp transpair_model4.cpp utility.cpp parse.cpp reports.cpp model3_viterbi.cpp model3_viterbi_with_tricks.cpp Dictionary.cpp model345-peg.cpp hmm.cpp HMMTables.cpp ForwardBackward.cpp
+

tools/giza-pp/GIZA++-v2/MoveSwapMatrix.cpp

@@ -0,0 +1,231 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "MoveSwapMatrix.h"
+
+template<class TRANSPAIR>
+MoveSwapMatrix<TRANSPAIR>::MoveSwapMatrix(const TRANSPAIR&_ef, const alignment&_a)
+  : alignment(_a), ef(_ef), l(ef.get_l()), m(ef.get_m()), _cmove(l+1, m+1), _cswap(m+1, m+1), 
+  delmove(l+1, m+1,0),delswap(m+1, m+1,0),changed(l+2, 0), changedCounter(1), 
+  modelnr(_ef.modelnr()),lazyEvaluation(0),centerDeleted(0)
+{
+  double thisValue=ef.scoreOfAlignmentForChange((*this));
+  if( lazyEvaluation==0)
+    for(WordIndex j=1;j<=m;j++)updateJ(j, 0,thisValue);
+}
+
+template<class TRANSPAIR>
+void MoveSwapMatrix<TRANSPAIR>::updateJ(WordIndex j, bool useChanged,double thisValue)
+{
+  massert( lazyEvaluation==0 );
+  for(WordIndex i=0;i<=l;i++)
+    if( (useChanged==0||changed[i]!=changedCounter) )
+      if( get_al(j)!=i ) 
+	_cmove(i, j)=ef.scoreOfMove((*this), i, j,thisValue);
+      else
+	_cmove(i, j)=1.0;
+  for(WordIndex j2=j+1;j2<=m;j2++)
+    if( get_al(j)!=get_al(j2) )
+      _cswap(j, j2)=ef.scoreOfSwap((*this), j, j2,thisValue);
+    else
+      _cswap(j, j2)=1.0;
+  for(WordIndex j2=1;j2<j;j2++)
+    if( get_al(j)!=get_al(j2) )
+      _cswap(j2, j)=ef.scoreOfSwap((*this), j2, j,thisValue);
+    else
+      _cswap(j2, j)=1.0;
+}
+template<class TRANSPAIR>
+void MoveSwapMatrix<TRANSPAIR>::updateI(WordIndex i,double thisValue)
+{
+  massert( lazyEvaluation==0);
+  for(WordIndex j=1;j<=m;j++)
+    if( get_al(j)!=i )
+      _cmove(i, j)=ef.scoreOfMove((*this), i, j,thisValue);
+    else
+      _cmove(i, j)=1.0;
+}
+
+template<class TRANSPAIR>
+void MoveSwapMatrix<TRANSPAIR>::printWrongs()const{
+  for(WordIndex i=0;i<=l;i++)
+    {
+      for(WordIndex j=1;j<=m;j++)
+	if( get_al(j)==i)
+	  cout << "A";
+	else
+	  {
+	    LogProb real=_cmove(i, j), wanted=ef.scoreOfMove((*this), i, j);
+	    if( fabs(1.0-real/wanted)>1e-3 )
+	      cout << 'b';
+	    else if(fabs(1.0-real/wanted)>1e-10 )
+	      cout << 'e';
+	    else if(real!=wanted)
+	      cout << 'E';
+	    else
+	      cout << ' ';
+	  }
+      cout << endl;
+    }
+  cout << endl;
+  for(WordIndex j=1;j<=m;j++)
+    {
+      for(WordIndex j1=1;j1<=m;j1++)
+	if( j1>j )
+	  {
+	    if( get_al(j)==get_al(j1) )
+	      cout << 'A';
+	    else 
+	      cout << (_cswap(j, j1)==ef.scoreOfSwap((*this), j, j1));
+	  }
+	else
+	  cout << ' ';
+      cout << endl;  
+    }
+  massert(0);
+}
+template<class TRANSPAIR>
+bool MoveSwapMatrix<TRANSPAIR>::isRight()const{
+  if( lazyEvaluation ) 
+    return 1;
+  for(WordIndex i=0;i<=l;i++)
+    for(WordIndex j=1;j<=m;j++)
+      if( get_al(j)!=i && (!(doubleEqual(_cmove(i, j), ef.scoreOfMove((*this), i, j)))) )
+	{
+	  cerr << "DIFF: " << i << " " << j << " " << _cmove(i, j) << " " << ef.scoreOfMove((*this), i, j) << endl;
+	  return 0;
+	}
+  for(WordIndex j=1;j<=m;j++)
+    for(WordIndex j1=1;j1<=m;j1++)
+      if( j1>j&&get_al(j)!=get_al(j1)&&(!doubleEqual(_cswap(j, j1), ef.scoreOfSwap((*this), j, j1))) )
+	{
+	  cerr << "DIFFERENT: " << j << " " << j1 << " " << _cswap(j, j1) << " " << ef.scoreOfSwap((*this), j, j1) << endl;
+	  return 0;
+	}
+  return 1;
+}
+
+template<class TRANSPAIR>
+void MoveSwapMatrix<TRANSPAIR>::doMove(WordIndex _i, WordIndex _j)
+{
+  WordIndex old_i=get_al(_j);
+  if( lazyEvaluation )
+    set(_j,_i);
+  else
+    {
+      if ( modelnr==5||modelnr==6 )
+	{
+	  set(_j, _i);
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  for(WordIndex j=1;j<=m;j++)updateJ(j, 0,thisValue);
+	}
+      else if ( modelnr==4 )
+	{
+	  changedCounter++;
+	  for(unsigned int k=prev_cept(old_i);k<=next_cept(old_i);++k)changed[k]=changedCounter;
+	  for(unsigned int k=prev_cept(_i);k<=next_cept(_i);++k)changed[k]=changedCounter;
+	  set(_j, _i);
+	  for(unsigned int k=prev_cept(old_i);k<=next_cept(old_i);++k)changed[k]=changedCounter;
+	  for(unsigned int k=prev_cept(_i);k<=next_cept(_i);++k)changed[k]=changedCounter;
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  for(unsigned int i=0;i<=l;i++)
+	    if(changed[i]==changedCounter)
+	      updateI(i,thisValue);
+	  for(unsigned int j=1;j<=m;j++)
+	    if( changed[get_al(j)]==changedCounter )
+	      updateJ(j, 1,thisValue);
+	}
+      else
+	{
+	  assert(modelnr==3);
+	  set(_j, _i);
+	  changedCounter++;
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  updateI(old_i,thisValue);
+	  changed[old_i]=changedCounter;
+	  updateI(_i,thisValue);
+	  changed[_i]=changedCounter;
+	  for(WordIndex j=1;j<=m;j++)
+	    if( get_al(j)==_i || get_al(j)==old_i )
+	      updateJ(j, 1,thisValue);
+	}
+    }
+}
+template<class TRANSPAIR>
+void MoveSwapMatrix<TRANSPAIR>::doSwap(WordIndex _j1, WordIndex _j2)
+{
+  assert( cswap(_j1, _j2)>1 );
+  WordIndex i1=get_al(_j1), i2=get_al(_j2);
+  if( lazyEvaluation==1 )
+    {
+      set(_j1, i2);
+      set(_j2, i1);
+    }
+  else
+    {
+      if ( modelnr==5||modelnr==6 )
+	{
+	  set(_j1, i2);
+	  set(_j2, i1);
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  for(WordIndex j=1;j<=m;j++)updateJ(j, 0,thisValue);
+	}
+      else if( modelnr==4 )
+	{
+	  changedCounter++;
+	  for(unsigned int k=prev_cept(i1);k<=next_cept(i1);++k)changed[k]=changedCounter;
+	  for(unsigned int k=prev_cept(i2);k<=next_cept(i2);++k)changed[k]=changedCounter;
+	  set(_j1, i2);
+	  set(_j2, i1);
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  for(unsigned int i=0;i<=l;i++)
+	    if(changed[i]==changedCounter)
+	      updateI(i,thisValue);
+	  for(unsigned int j=1;j<=m;j++)
+	    if( changed[get_al(j)]==changedCounter )
+	      updateJ(j, 1,thisValue);
+	}
+      else
+	{
+	  assert(modelnr==3);
+	  set(_j1, i2);
+	  set(_j2, i1);
+	  changedCounter++;
+	  double thisValue=ef.scoreOfAlignmentForChange((*this));
+	  updateI(i1,thisValue);
+	  changed[i1]=changedCounter;
+	  updateI(i2,thisValue);
+	  changed[i2]=changedCounter;
+	  updateJ(_j1, 1,thisValue);
+	  updateJ(_j2, 1,thisValue);
+	}
+    }
+}
+
+#include "transpair_model3.h"
+#include "transpair_model4.h"
+#include "transpair_model5.h"
+#include "transpair_modelhmm.h"
+template class MoveSwapMatrix<transpair_model3>;
+template class MoveSwapMatrix<transpair_model4>;
+template class MoveSwapMatrix<transpair_model5>;
+template class MoveSwapMatrix<transpair_modelhmm>;

tools/giza-pp/GIZA++-v2/MoveSwapMatrix.h

@@ -0,0 +1,116 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/*--
+MoveSwapMatrix: Efficient representation for moving and swapping
+around in IBM3 training.
+Franz Josef Och (30/07/99)
+--*/
+#ifndef moveswap2_costs_h_defined
+#define moveswap2_costs_h_defined
+#include "alignment.h"
+#include "transpair_model3.h"
+#include "myassert.h"
+
+extern short DoViterbiTraining;
+
+template<class TRANSPAIR>
+class MoveSwapMatrix : public alignment
+{
+ private:
+  const TRANSPAIR&ef;
+  const WordIndex l, m;
+  Array2<LogProb, Vector<LogProb> > _cmove, _cswap;
+  Array2<char,Vector<char> > delmove,delswap;
+  Vector<int> changed;
+  int changedCounter;
+  const int modelnr;
+  bool lazyEvaluation;
+  bool centerDeleted;
+ public:
+  bool check()const
+    {
+	  return 1;
+    }
+  const TRANSPAIR&get_ef()const
+    {return ef;}
+  bool isCenterDeleted()const
+    {return centerDeleted;}
+  bool isLazy()const
+    {return lazyEvaluation;}
+  MoveSwapMatrix(const TRANSPAIR&_ef, const alignment&_a);
+  void updateJ(WordIndex j, bool,double thisValue);
+  void updateI(WordIndex i,double thisValue);
+  void doMove(WordIndex _i, WordIndex _j);
+  void doSwap(WordIndex _j1, WordIndex _j2);
+  void delCenter()
+    {
+      centerDeleted=1;
+    }
+  void delMove(WordIndex x, WordIndex y)
+    {
+      delmove(x,y)=1;
+    }
+  void delSwap(WordIndex x, WordIndex y)
+    {
+      massert(y>x);
+      delswap(x,y)=1;
+      delswap(y,x)=1;
+    }
+  bool isDelMove(WordIndex x, WordIndex y)const
+    {
+      return DoViterbiTraining||delmove(x,y);
+    }
+  bool isDelSwap(WordIndex x, WordIndex y)const
+    {
+      massert(y>x);
+      return DoViterbiTraining||delswap(x,y);
+    }
+  LogProb cmove(WordIndex x, WordIndex y)const
+    {
+      massert( get_al(y)!=x );
+      massert( delmove(x,y)==0 );
+      if( lazyEvaluation )
+	return ef.scoreOfMove(*this,x,y);
+      else
+	{
+	  return _cmove(x, y);
+	}
+    }
+  LogProb cswap(WordIndex x, WordIndex y)const
+    {
+      massert(x<y);
+      massert(delswap(x,y)==0);
+      massert(get_al(x)!=get_al(y));
+      if( lazyEvaluation )
+	return ef.scoreOfSwap(*this,x,y);
+      else
+	{
+	  massert(y>x);
+	  return _cswap(x, y);
+	}
+    }
+  void printWrongs()const;
+  bool isRight()const;
+  friend ostream&operator<<(ostream&out, const MoveSwapMatrix<TRANSPAIR>&m)
+    {return out << (alignment)m << "\nEF:\n"<< m.ef << "\nCMOVE\n"<<m._cmove << "\nCSWAP\n" << m._cswap << endl;};
+};
+#endif

tools/giza-pp/GIZA++-v2/NTables.cpp

@@ -0,0 +1,93 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "NTables.h"
+#include <iostream>
+#include "defs.h"
+#include <fstream>
+#include "Parameter.h"
+
+GLOBAL_PARAMETER(double,NTablesFactorGraphemes,"nSmooth","smoothing for fertility parameters (good value: 64): weight for wordlength-dependent fertility parameters",PARLEV_SMOOTH,64.0);
+GLOBAL_PARAMETER(double,NTablesFactorGeneral,"nSmoothGeneral","smoothing for fertility parameters (default: 0): weight for word-independent fertility parameters",PARLEV_SMOOTH,0.0);
+
+template <class VALTYPE>
+void nmodel<VALTYPE>::printNTable(int noEW, const char* filename, 
+				  const Vector<WordEntry>& evlist, 
+				  bool actual) const
+     // prints the fertility table but with actual sourcce words (not their id)
+{
+  cerr << "Dumping nTable to: " << filename <<  '\n';  
+  ofstream of(filename);
+  VALTYPE p ;
+  WordIndex k, i ;
+  for(i=1; int(i) < noEW; i++){
+    if (evlist[i].freq > 0){
+      if (actual)
+	of << evlist[i].word << ' ' ;
+      else 
+	of << i << ' ' ;
+      for( k=0; k < MAX_FERTILITY; k++){
+	p = getValue(i, k);
+	if (p <= PROB_SMOOTH) 
+	  p = 0;
+	of << p << ' ';      
+      } 
+      of << '\n';
+    }
+  }
+}
+
+template <class VALTYPE>
+void nmodel<VALTYPE>::readNTable(const char *filename){
+  /* This function reads the n table from a file.
+     Each line is of the format:  source_word_id p0 p1 p2 ... pn
+     This is the inverse operation of the printTable function.
+     NAS, 7/11/99
+  */
+  ifstream inf(filename);
+  cerr << "Reading fertility table from " << filename << "\n";
+  if(!inf){
+    cerr << "\nERROR: Cannot open " << filename <<"\n";
+    return;
+  }
+
+  VALTYPE prob;
+  WordIndex tok, i;
+  int nFert=0;
+  while(!inf.eof()){
+    nFert++;
+    inf >> ws >> tok;
+    if (tok > MAX_VOCAB_SIZE){
+      cerr << "NTables:readNTable(): unrecognized token id: " << tok
+    <<'\n';
+    exit(-1);
+  }
+    for(i = 0; i < MAX_FERTILITY; i++){
+      inf >> ws >> prob;
+      getRef(tok, i)=prob;
+    }
+  }
+  cerr << "Read " << nFert << " entries in fertility table.\n";
+  inf.close();
+}
+
+template class nmodel<COUNT>;
+//template class nmodel<PROB>;

tools/giza-pp/GIZA++-v2/NTables.h

@@ -0,0 +1,145 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _ntables_h
+#define _ntables_h 1
+#include "Array2.h"
+#include "Vector.h"
+#include <cassert>
+#include "defs.h"
+#include "vocab.h"
+#include "myassert.h"
+#include "Globals.h"
+
+extern double NTablesFactorGraphemes,NTablesFactorGeneral;
+
+template <class VALTYPE>
+class nmodel
+{
+ private:
+  Array2<VALTYPE, Vector<VALTYPE> > ntab;
+ public:
+  nmodel(int maxw, int maxn)
+    : ntab(maxw, maxn, 0.0)
+    {}
+  VALTYPE getValue(int w, unsigned int n)const
+    {
+      massert(w!=0);
+      if(n>=ntab.getLen2())
+	return 0.0;
+      else
+	return max(ntab(w, n), VALTYPE(PROB_SMOOTH));
+    }
+  VALTYPE&getRef(int w, int n)
+    {
+      //massert(w!=0);
+      return ntab(w, n);
+    }
+  template<class COUNT>
+  void normalize(nmodel<COUNT>&write,const Vector<WordEntry>* _evlist)const
+{
+  int h1=ntab.getLen1(), h2=ntab.getLen2();
+  int nParams=0;
+  if( _evlist&&(NTablesFactorGraphemes||NTablesFactorGeneral) )
+    {
+      size_t maxlen=0;
+      const Vector<WordEntry>&evlist=*_evlist;
+      for(unsigned int i=1;i<evlist.size();i++)
+	maxlen=max(maxlen,evlist[i].word.length());
+      Array2<COUNT,Vector<COUNT> > counts(maxlen+1,MAX_FERTILITY+1,0.0);
+      Vector<COUNT> nprob_general(MAX_FERTILITY+1,0.0);
+      for(unsigned int i=1;i<min((unsigned int)h1,(unsigned int)evlist.size());i++)
+	{
+	  int l=evlist[i].word.length();
+	  for(int k=0;k<h2;k++)
+	    {
+	      counts(l,k)+=getValue(i,k);
+	      nprob_general[k]+=getValue(i,k);
+	    }
+	}
+      COUNT sum2=0; 
+      for(unsigned int i=1;i<maxlen+1;i++)
+	{
+	  COUNT sum=0.0;
+	  for(int k=0;k<h2;k++)
+	    sum+=counts(i,k);
+	  sum2+=sum;
+	  if( sum )
+	    {
+	      double average=0.0;
+	      //cerr << "l: " << i << " " << sum << " ";
+	      for(int k=0;k<h2;k++)
+		{
+		  counts(i,k)/=sum;
+		  //cerr << counts(i,k) << ' ';
+		  average+=k*counts(i,k);
+		}
+	      //cerr << "avg: " << average << endl;
+	      //cerr << '\n';
+	    }
+	}
+      for(unsigned int k=0;k<nprob_general.size();k++)
+	nprob_general[k]/=sum2;
+      
+      for(int i=1;i<h1;i++)
+	{
+	  int l=-1;
+	  if((unsigned int)i<evlist.size())
+	    l=evlist[i].word.length();
+	  COUNT sum=0.0;
+	  for(int k=0;k<h2;k++)
+	    sum+=getValue(i, k)+((l==-1)?0.0:(counts(l,k)*NTablesFactorGraphemes)) + NTablesFactorGeneral*nprob_general[k];
+	  assert(sum);
+	  for(int k=0;k<h2;k++)
+	    {
+	      write.getRef(i, k)=(getValue(i, k)+((l==-1)?0.0:(counts(l,k)*NTablesFactorGraphemes)))/sum + NTablesFactorGeneral*nprob_general[k];
+	      nParams++;
+	    }
+	}
+    }
+  else
+    for(int i=1;i<h1;i++)
+      {
+	COUNT sum=0.0;
+	for(int k=0;k<h2;k++)
+	  sum+=getValue(i, k);
+	assert(sum);
+	for(int k=0;k<h2;k++)
+	  {
+	    write.getRef(i, k)=getValue(i, k)/sum;
+	    nParams++;
+	  }
+      }
+  cerr << "NTable contains " << nParams << " parameter.\n";
+}
+
+  void clear()
+    {
+      int h1=ntab.getLen1(), h2=ntab.getLen2();
+      for(int i=0;i<h1;i++)for(int k=0;k<h2;k++)
+	ntab(i, k)=0;
+    }
+  void printNTable(int noEW, const char* filename, const Vector<WordEntry>& evlist, bool) const;
+  void readNTable(const char *filename);
+  
+};
+
+#endif

tools/giza-pp/GIZA++-v2/Parameter.cpp

@@ -0,0 +1,144 @@
+/*
+
+Copyright (C) 1997,1998,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "Parameter.h"
+#include <fstream>
+#include <unistd.h>
+#include <sstream>
+
+
+bool absolutePathNames=0;
+string ParameterPathPrefix;
+bool ParameterChangedFlag=0;
+
+bool writeParameters(ofstream&of,const ParSet&parset,int level)
+{
+  if(!of)return 0;
+  for(ParSet::const_iterator i=parset.begin();i!=parset.end();++i)
+    {
+      if(((*i)->getLevel()==level||level==-1)&&(*i)->onlyCopy==0)
+	{
+	  ostringstream os;
+	  (*i)->printValue(os);
+	  os << ends;
+	  string s(os.str());
+	  of << (*i)->getString() << " ";
+	  if( absolutePathNames&&(*i)->isFilename()&&s.length()&&s[0]!='/' )
+	    {
+	      char path[1024];
+	      getcwd(path,1024);
+	      of << path << '/';
+	    }
+	  if( ParameterPathPrefix.length()&&(*i)->isFilename()&&s.length()&&s[0]!='/' )
+	    of << ParameterPathPrefix << '/'; 
+	  (*i)->printValue(of);
+	  of << endl;
+	}
+    }
+  return 1;
+}
+
+bool readParameters(ifstream&f,const ParSet&parset,int verb,int level)
+{
+  string s;
+  if(!f)return 0;
+  while(getline(f,s))
+    {
+      istringstream eingabe(s);
+      string s1,s2;
+      eingabe>>s1>>s2;
+      if(makeSetCommand(s1,s2,parset,verb,level)==0)
+	cerr << "ERROR: could not set: (C) " << s1 << " " << s2 << endl;
+    }
+  return 1;
+}
+
+ 
+bool makeSetCommand(string _s1,string s2,const ParSet&parset,int verb,int level)
+{
+  ParPtr anf;
+  int anfset=0;
+  string s1=simpleString(_s1);
+  for(ParSet::const_iterator i=parset.begin();i!=parset.end();++i)
+    {
+      if( *(*i)==s1 )
+	{
+	  if( level==-1 || level==(*i)->getLevel() )
+	    (*i)->setParameter(s2,verb);
+	  else if(verb>1)
+	    cerr << "ERROR: Could not set: (A) " << s1 << " " << s2 << " " << level << " " << (*i)->getLevel() << endl;
+	  return 1;
+	}
+      else if( (*i)->getString().substr(0,s1.length())==s1 )
+	{
+	  anf=(*i);anfset++;
+	}
+    }
+  if(anfset==1)
+    {
+      if( level==-1 || level==anf->getLevel() )
+	anf->setParameter(s2,verb);
+      else if( verb>1 )
+	cerr << "ERROR: Could not set: (B) " << s1 << " " << s2 << " " << level << " " << anf->getLevel() << endl;
+      return 1;
+    }
+  if( anfset>1 )
+    cerr << "ERROR: ambiguous parameter '" << s1 << "'.\n";
+  if( anfset==0 )
+    cerr << "ERROR: parameter '" << s1 << "' does not exist.\n";
+  return 0;
+}
+
+ostream& printPars(ostream&of,const ParSet&parset,int level)
+{
+  if(!of)return of;
+  for(ParSet::const_iterator i=parset.begin();i!=parset.end();++i)
+    {
+      if(((*i)->getLevel()==level||level==-1)&&(*i)->onlyCopy==0)
+	{
+	  (*i)->printAt(of);
+	  of << endl;
+	}
+    }
+  return of;
+}
+
+string simpleString(const string s)
+{
+  string k;
+  for(unsigned int i=0;i<s.length();++i)
+    {
+      char c[2];
+      c[0]=tolower(s[i]);
+      c[1]=0;
+      if( (c[0]>='a'&&c[0]<='z')||(c[0]>='0'&&c[0]<='9') )
+	k += c;
+    }
+  return k;
+}
+
+
+ParSet&getGlobalParSet()
+{
+  static ParSet x;
+  return x;
+}

tools/giza-pp/GIZA++-v2/Parameter.h

@@ -0,0 +1,200 @@
+/*
+
+Copyright (C) 1997,1998,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef PARAMETER_H_DEFINED
+#define PARAMETER_H_DEFINED
+
+#include "mystl.h"
+#include <set>
+#include "Pointer.h"
+#include <string>
+#include "Globals.h"
+#include <fstream>
+#include <cstring>
+
+inline unsigned int mConvert(const string&s,unsigned int &i)
+{ 
+  if( strcasecmp(s.c_str(),"yes")==0 || strcasecmp(s.c_str(),"y")==0 || strcasecmp(s.c_str(),"true")==0 || strcasecmp(s.c_str(),"t")==0 ) { cerr << "TRUE\n";return i=1; }
+  if( strcasecmp(s.c_str(),"no")==0 ||  strcasecmp(s.c_str(),"n")==0 ||  strcasecmp(s.c_str(),"false")==0 ||  strcasecmp(s.c_str(),"f")==0 ) { cerr << "FALSE\n";return i=0;}
+  return i=atoi(s.c_str()); 
+}
+inline int mConvert(const string&s,int &i){ 
+  if( strcasecmp(s.c_str(),"yes")==0 ||  strcasecmp(s.c_str(),"y")==0 ||  strcasecmp(s.c_str(),"true")==0 ||  strcasecmp(s.c_str(),"t")==0 ) { cerr << "TRUE\n";return i=1;}
+  if( strcasecmp(s.c_str(),"no")==0 ||  strcasecmp(s.c_str(),"n")==0 ||  strcasecmp(s.c_str(),"false")==0 ||  strcasecmp(s.c_str(),"f")==0 ) { cerr << "FALSE\n";return i=0;}
+  return i=atoi(s.c_str()); 
+}
+inline double mConvert(const string&s,double &d) { return d=atof(s.c_str()); }
+inline double mConvert(const string&s,float &d) { return d=atof(s.c_str()); }
+inline string mConvert(const string&s,string&n) { return n=s; }
+inline bool mConvert(const string&s,bool&n) { 
+  if( strcasecmp(s.c_str(),"yes")==0 ||  strcasecmp(s.c_str(),"y")==0 ||  strcasecmp(s.c_str(),"true")==0 ||  strcasecmp(s.c_str(),"t")==0 ) { cerr << "TRUE\n";return n=1;}
+  if( strcasecmp(s.c_str(),"no")==0 ||  strcasecmp(s.c_str(),"n")==0 ||  strcasecmp(s.c_str(),"false")==0 ||  strcasecmp(s.c_str(),"f")==0 ) { cerr << "FALSE\n";return n=0;}
+  return n=atoi(s.c_str()); 
+}
+inline short mConvert(const string&s,short&n) { 
+  if( strcasecmp(s.c_str(),"yes")==0 ||  strcasecmp(s.c_str(),"y")==0 ||  strcasecmp(s.c_str(),"true")==0 ||  strcasecmp(s.c_str(),"t")==0 ) { cerr << "TRUE\n";return n=1;}
+  if( strcasecmp(s.c_str(),"no")==0 ||  strcasecmp(s.c_str(),"n")==0 ||  strcasecmp(s.c_str(),"false")==0 ||  strcasecmp(s.c_str(),"f")==0 ) { cerr << "FALSE\n";return n=0;}
+  return n=atoi(s.c_str()); 
+}
+inline unsigned short mConvert(const string&s,unsigned short&n) { 
+  if( strcasecmp(s.c_str(),"yes")==0 ||  strcasecmp(s.c_str(),"y")==0 ||  strcasecmp(s.c_str(),"true")==0 ||  strcasecmp(s.c_str(),"t")==0 ) { cerr << "TRUE\n";return n=1;}
+  if( strcasecmp(s.c_str(),"no")==0 ||  strcasecmp(s.c_str(),"n")==0 ||  strcasecmp(s.c_str(),"false")==0 ||  strcasecmp(s.c_str(),"f")==0 ) { cerr << "FALSE\n";return n=0;}
+  return n=atoi(s.c_str()); 
+}
+
+string simpleString(const string s);
+
+inline int Hashstring(const string& s)
+{
+  int sum=0;
+  string::const_iterator i=s.begin(),end=s.end();
+  for(;i!=end;i++)sum=5*sum+(*i);
+  return sum;
+}
+
+class _Parameter
+{
+ protected:
+  string name;
+  bool *ifChanged;
+  string description;
+  int level;
+  bool filename;
+ public:
+  int onlyCopy;
+  _Parameter(string n,bool&b,string desc,int _level,bool _onlyCopy)
+    : name(simpleString(n)),ifChanged(&b),description(desc),level(_level),filename(0),onlyCopy(_onlyCopy) {}
+  virtual ~_Parameter(){};
+  bool operator==(const string&s)const
+    { return name== simpleString(s); }
+  void setChanged() 
+    { *ifChanged=true; }
+  virtual bool setParameter(string s2,int)=0;
+  virtual ostream&printAt(ostream&out)=0;
+  virtual ostream&printValue(ostream&out)=0;
+  const string&getString() const { return name; }
+  int getLevel() const { return level;}
+  bool isFilename() { return filename;}
+  void setFilename(bool x=1) { filename=x;}
+  friend bool operator==(const _Parameter&a,const _Parameter&b)
+    { return a.name==b.name; }
+  friend bool operator<(const _Parameter&a,const _Parameter&b)
+    { return a.name<b.name; }
+  friend int Hash(const _Parameter&aaa)
+    { return Hashstring(aaa.name); }
+  friend ostream&operator<<(ostream&out,const _Parameter&p)
+    { return out<<"Parameter: "<<p.name <<endl;}
+};
+
+template<class T>
+class Parameter : public _Parameter
+{
+ private:
+  T*t;
+ public:
+  Parameter(string n,bool&b,string desc,T&_t,int level=0,bool onlyCopy=0)
+    : _Parameter(n,b,desc,level,onlyCopy),t(&_t) {}
+  virtual ~Parameter(){}
+  virtual bool setParameter(string s2,int verb)
+    { 
+      T x;  
+      if( !(*t==mConvert(s2,x)))
+	{
+	  bool printedFirst=0;
+	  if( verb>1 ) 
+	    {
+	      cout << "Parameter '"<<name <<"' changed from '"<<*t<<"' to '";
+	      printedFirst=1;
+	    }
+	  mConvert(s2,*t);
+	  if( printedFirst ) 
+	    cout << *t <<"'\n";
+	  setChanged();
+	  return 1;
+	} 
+      return 0;
+    }
+  virtual ostream&printAt(ostream&out)
+    {return out << name << " = " << *t << "  (" << description << ")";}
+  virtual ostream&printValue(ostream&out)
+    {return out << *t;}
+};
+
+typedef MP<_Parameter> ParPtr;
+
+class ParSet : public set<ParPtr>
+{
+ public:
+  void insert(const ParPtr&x)
+    {
+      if( count(x)!=0 )
+	cerr << "ERROR: element " << x->getString() << " already inserted.\n";
+      set<ParPtr>::insert(x);
+    }
+};
+
+bool makeSetCommand(string s1,string s2,const ParSet&pars,int verb=1,int level= -1);
+ostream&printPars(ostream&out,const ParSet&pars,int level=-1);
+bool writeParameters(ofstream&of,const ParSet&parset,int level=0);
+bool readParameters(ifstream&f,const ParSet&parset,int verb=2,int level=0);
+ParSet&getGlobalParSet();
+extern bool ParameterChangedFlag;
+template<class T>const T&addGlobalParameter(const char *name,const char *description,int level,T*adr,const T&init)
+{
+  *adr=init;
+  getGlobalParSet().insert(new Parameter<T>(name,ParameterChangedFlag,description,*adr,level));
+  return init;
+}
+template<class T>const T&addGlobalParameter(const char *name,const char *name2,const char *description,int level,T*adr,const T&init)
+{
+  *adr=init;
+  getGlobalParSet().insert(new Parameter<T>(name,ParameterChangedFlag,description,*adr,level));
+  getGlobalParSet().insert(new Parameter<T>(name2,ParameterChangedFlag,description,*adr,-1));
+  return init;
+}
+template<class T>const T&addGlobalParameter(const char *name,const char *name2,const char *name3,const char *description,int level,T*adr,const T&init)
+{
+  *adr=init;
+  getGlobalParSet().insert(new Parameter<T>(name,ParameterChangedFlag,description,*adr,level));
+  getGlobalParSet().insert(new Parameter<T>(name2,ParameterChangedFlag,description,*adr,-1));
+  getGlobalParSet().insert(new Parameter<T>(name3,ParameterChangedFlag,description,*adr,-1));
+  return init;
+}
+template<class T>const T&addGlobalParameter(const char *name,const char *name2,const char *name3,const char *name4,const char *description,int level,T*adr,const T&init)
+{
+  *adr=init;
+  getGlobalParSet().insert(new Parameter<T>(name,ParameterChangedFlag,description,*adr,level));
+  getGlobalParSet().insert(new Parameter<T>(name2,ParameterChangedFlag,description,*adr,-1));
+  getGlobalParSet().insert(new Parameter<T>(name3,ParameterChangedFlag,description,*adr,-1));
+  getGlobalParSet().insert(new Parameter<T>(name4,ParameterChangedFlag,description,*adr,-1));
+  return init;
+}
+void MakeParameterOptimizing(istream&file,string resultingParameters);
+
+#define GLOBAL_PARAMETER(TYP,VARNAME,NAME,DESCRIPTION,LEVEL,INIT) TYP VARNAME=addGlobalParameter< TYP >(NAME,DESCRIPTION,LEVEL,&VARNAME,INIT);
+#define GLOBAL_PARAMETER2(TYP,VARNAME,NAME,NAME2,DESCRIPTION,LEVEL,INIT) TYP VARNAME=addGlobalParameter< TYP >(NAME,NAME2,DESCRIPTION,LEVEL,&VARNAME,INIT);
+#define GLOBAL_PARAMETER3(TYP,VARNAME,NAME,NAME2,NAME3,DESCRIPTION,LEVEL,INIT) TYP VARNAME=addGlobalParameter< TYP >(NAME,NAME2,NAME3,DESCRIPTION,LEVEL,&VARNAME,INIT);
+#define GLOBAL_PARAMETER4(TYP,VARNAME,NAME,NAME2,NAME3,NAME4,DESCRIPTION,LEVEL,INIT) TYP VARNAME=addGlobalParameter< TYP >(NAME,NAME2,NAME3,NAME4,DESCRIPTION,LEVEL,&VARNAME,INIT);
+
+void setParameterLevelName(unsigned int i,string x);
+
+#endif

tools/giza-pp/GIZA++-v2/Perplexity.cpp

@@ -0,0 +1,40 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* Perplexity.cc
+ * =============
+ * Mike Jahr, 7/21/99
+ * Machine Translation group, WS99
+ * Center for Language and Speech Processing
+ * 
+ * Last Modified by: Yaser Al-Onaizan, August 17, 1999
+ *
+ * Simple class used to calculate cross entropy and perplexity
+ * of models.
+ */
+
+#include "Perplexity.h"
+
+void Perplexity::record(string model){
+  modelid.push_back(model);
+  perp.push_back(perplexity());
+  ce.push_back(cross_entropy());
+}

tools/giza-pp/GIZA++-v2/Perplexity.h

@@ -0,0 +1,108 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* Perplexity.h
+ * ============
+ * Mike Jahr, 7/15/99
+ * Machine Translation group, WS99
+ * Center for Language and Speech Processing
+ * 
+ * Last Modified by: Yaser Al-Onaizan, August 17, 1999
+ *
+ * Simple class used to calculate cross entropy and perplexity
+ * of models.
+ */
+
+#ifndef _PERPLEXITY_H
+#define _PERPLEXITY_H
+
+#include <cmath>
+#include <fstream>
+#include "Vector.h"
+#include "defs.h"
+#include "Array2.h"
+#include "Globals.h"
+
+#define CROSS_ENTROPY_BASE 2
+
+class Perplexity {
+ private:
+    double sum;
+    double wc;
+    Array2<double, Vector<double> > *E_M_L;
+    Vector<string> modelid;
+    Vector<double > perp;
+    Vector<double > ce;
+    Vector<string> name ;
+ public:
+    ~Perplexity() { delete E_M_L;}
+    Perplexity() {
+      E_M_L  = new Array2<double, Vector<double> >(MAX_SENTENCE_LENGTH,MAX_SENTENCE_LENGTH);
+      unsigned int l, m ;
+      Vector<double> fact(MAX_SENTENCE_LENGTH, 1.0);
+      for (m = 2 ; m < MAX_SENTENCE_LENGTH ; m++)
+	fact[m] = fact[m-1] * m ;
+      for (m = 1 ; m < MAX_SENTENCE_LENGTH ; m++)
+	for (l = 1 ; l < MAX_SENTENCE_LENGTH ; l++) {
+	  (*E_M_L)(l, m) = log (pow((LAMBDA * l), double(m)) * exp(-LAMBDA * double(l)) / 
+				(fact[m])) ;
+	}
+      sum = 0 ;
+      wc = 0;
+      perp.clear();
+      ce.clear();
+      name.clear();
+    }
+    inline void clear() {
+      sum = 0 ;
+      wc = 0 ;
+    }
+    size_t size() const {return(min(perp.size(), ce.size()));}
+    inline void addFactor(const double p, const double count, const int l, 
+			  const int m,bool withPoisson) {
+      wc += count * m ; // number of french words 
+      sum += count * ( (withPoisson?((*E_M_L)(l, m)):0.0) + p) ;
+    }
+    inline double perplexity() const {
+	return exp( -1*sum / wc);
+    }
+
+    inline double cross_entropy() const {
+      return (-1.0*sum / (log(double(CROSS_ENTROPY_BASE)) * wc)); 
+    }
+
+    inline double word_count() const {
+	return wc;
+    }
+    
+    inline double getSum() const {
+      return sum ;
+    }
+
+    void record(string model);
+    
+    friend void generatePerplexityReport(const Perplexity&, const Perplexity&, 
+					 const Perplexity&, const Perplexity&, 
+					 ostream&, int, int, bool); 
+};
+
+
+#endif

tools/giza-pp/GIZA++-v2/Pointer.h

@@ -0,0 +1,175 @@
+/*
+
+Copyright (C) 1997,1998,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef HEADER_Pointer_DEFINED
+#define HEADER_Pointer_DEFINED
+
+#include <cassert>
+#include <ostream>
+
+template<class T>
+class SmartPointer
+{
+ protected:
+  T*p;
+ public:
+  SmartPointer(T*_p=0) 
+    : p(_p) {}
+  inline T&operator*() const 
+    {return *p;}
+  inline T*operator->() const 
+    {return p;}
+  inline operator bool() const 
+    {return p!=0;}
+  inline T*ptr() const
+    { return p; }
+};
+template<class T> inline ostream &operator<<(ostream&out,const SmartPointer<T>&s)
+{if( s.ptr() )return out << *s;else return out <<"nullpointer";}
+
+
+template<class T>
+class SmartPointerConst
+{
+ protected:
+  const T*p;
+ public:
+  SmartPointerConst(const T*_p=0) 
+    : p(_p) {}
+  inline const T&operator*() const 
+    {return *p;}
+  inline const T*operator->() const 
+    {return p;}
+  inline operator bool() const
+    {return p!=0;}
+  inline const T*ptr() const
+    { return p; }
+};
+template<class T> inline ostream &operator<<(ostream&out,const SmartPointerConst<T>&s)
+{if( s.ptr() )return out << *s;else return out <<"nullpointer";}
+
+template <class T>
+class UP : public SmartPointer<T>
+{
+ public:
+  UP(T*_p=0) 
+    : SmartPointer<T>(_p) {}
+};
+template<class T> inline bool operator==(const UP<T>&s1,const UP<T>&s2)
+{return s1.ptr()==s2.ptr();}
+template<class T>  inline bool operator<(const UP<T>&s1,const UP<T>&s2)
+{return s1.ptr() < s2.ptr();}
+template<class T> inline int Hash(const UP<T> &wp)
+{if(wp.ptr())return Hash(*wp);else return 0;}
+
+
+template <class T>
+class UPConst : public SmartPointerConst<T>
+{
+ public:
+  UPConst(const T*_p=0) 
+    : SmartPointerConst<T>(_p) {}
+};
+template<class T> inline bool operator==(const UPConst<T>&s1,const UPConst<T>&s2)
+{return s1.ptr()==s2.ptr();}
+template<class T> inline bool operator<(const UPConst<T>&s1,const UPConst<T>&s2)
+{return s1.ptr()<s2.ptr();}
+template<class T> inline int Hash(const UPConst<T> &wp)
+{if(wp.ptr())return Hash(*wp);else return 0;}
+
+	
+template <class T>
+class MP : public SmartPointer<T>
+{
+ public:
+  MP(T*_p=0) 
+    : SmartPointer<T>(_p) {}
+};
+template <class T> inline bool operator==(const MP<T>&s1,const MP<T>&s2)
+{assert(s1);assert(s2);return *s1==*s2;}
+template <class T> inline bool operator<(const MP<T>&s1,const MP<T>&s2)
+{assert(s1);assert(s2);return *s1 < *s2;}
+template <class T> inline int Hash(const MP<T> &wp)
+{if(wp.ptr())return Hash(*wp);else return 0;}
+
+
+template <class T>
+class MPConst : public SmartPointerConst<T>
+{
+ public:
+  MPConst(const T*_p=0) 
+    : SmartPointerConst<T>(_p) {}
+};
+template <class T> inline bool operator==(const MPConst<T>&s1,const MPConst<T>&s2)
+{assert(s1);assert(s2);return *s1== *s2;}
+template <class T> inline bool operator<(const MPConst<T>&s1,const MPConst<T>&s2)
+{assert(s1);assert(s2);return *s1 < *s2;}
+template <class T> inline int Hash(const MPConst<T> &wp)
+{if(wp.ptr())return Hash(*wp);else return 0;}
+
+
+template <class T> 
+class DELP : public SmartPointer<T>
+{
+ private:
+  DELP(const DELP<T>&x);
+ public:
+  const DELP<T>&operator=(DELP<T>&x)
+  {
+    delete this->p;
+    this->p=x.p;x.p=0;
+    return *this;
+  }
+
+  ~DELP()
+    { delete this->p;this->p=0;}
+  DELP(T*_p=0) 
+    : SmartPointer<T>(_p) {}
+  void set(T*_p)
+    {
+      delete this->p;
+      this->p=_p;
+    }
+  friend bool operator==(const DELP<T>&s1,const DELP<T>&s2)
+    {
+      return *(s1.p)== *(s2.p);
+    }
+  friend bool operator<(const DELP<T>&s1,const DELP<T>&s2)
+    {
+      return *(s1.p) < *(s2.p);
+    }
+  friend inline int Hash(const DELP<T> &wp)
+    {
+      if(wp.p)
+	return Hash(*wp.p);
+      else 
+	return 0;
+    }
+};
+#endif
+
+
+
+
+
+
+

tools/giza-pp/GIZA++-v2/README

@@ -0,0 +1,508 @@
+========================================================================
+GIZA++ is an extension of the program GIZA.
+It is a program for learning statistical translation models from
+bitext. It is an implementation of the models described in
+(Brown et al., 1993), (Vogel et al., 1996), (Och et al., 2000a), 
+(Och et al., 2000b).
+========================================================================
+
+
+
+CONTENTS of this README file:
+
+Part I: GIZA Package Contents
+Part II: How To Compile GIZA
+Part III: How to Run GIZA
+Part IV: Input File Formats
+     A. VOCABULARY FILES
+     B. Bitext Files
+     C. Dictionary File (optional)
+Part V: Output File Formats:
+     A. PROBABILITY TABLES
+	1. T TABLE (translation table)
+	2. N TABLE (Fertility table)
+	3. P0 TABLE
+	4. A TABLE
+	5. D3 TABLE
+	6. D4 TABLE
+	7. D5 TABLE
+	8. HMM TABLE
+     B. ALIGNMENT FILE
+     C. Cross Entropy and Perplexity Files
+     D. Revised Vocabulary files
+Part VI: Literature
+Part VII: New features
+
+HISTORY of this README file:
+
+GIZA++:
+edited: 11 Jan. 2000, Franz Josef Och
+GIZA:
+edited: 16 Aug. 1999, Dan Melamed
+edited: 13 Aug. 1999, Yaser Al-Onaizan
+edited: 20 July 1999, Yaser Al-Onaizan
+edited: 15 July 1999, Yaser Al-Onaizan
+edited: 13 July 1999, Noah Smith
+========================================================================
+
+Part 0: What is GIZA++
+
+GIZA++ is an extension of the program GIZA (part of the SMT toolkit
+EGYPT - http://www.clsp.jhu.edu/ws99/projects/mt/toolkit/ ) which was
+developed by the Statistical Machine Translation team during the
+summer workshop in 1999 at the Center for Language and Speech
+Processing at Johns-Hopkins University (CLSP/JHU). GIZA++ includes a
+lot of additional features. The extensions of GIZA++ were designed and
+written by Franz Josef Och.
+
+Features of GIZA++ not in GIZA:
+
+- Implements full IBM-4 alignment model with a dependency of word
+classes as described in (Brown et al. 1993)
+
+- Implements IBM-5: dependency on word classes, smoothing, ...
+
+- Implements HMM alignment model: Baum-Welch training, Forward-Backward
+algorithm, empty word, dependency on word classes, transfer to
+fertility models, ...
+
+- Implementation of a variant of the IBM-3 and IBM-4
+(-deficientDistortionModel 1) models which allow the training of -p0
+
+- Smoothing for fertility, distortion/alignment parameters
+
+- Significant more efficient training of the fertility models 
+
+- Correct implementation of pegging as described in (Brown et
+al. 1993), implemented a series of heuristics in order to make pegging
+sufficiently efficient
+
+- Completely new parameter mechanism: allows to easily add additional
+parameters
+
+- Improved perplexity calculation for models IBM-1, IBM-2 and HMM (the
+parameter of the Poisson-distribution of the sentence lengths is
+computed automatically from the used training corpus)
+
+========================================================================
+Part I: GIZA++ Package Programs
+
+GIZA++: GIZA++ itself
+
+plain2snt.out: simple tool to transform plain text into GIZA text
+format
+
+snt2plain.out: simple tool to transform GIZA text format into plain
+text
+
+trainGIZA++.sh: Shell script to perform standard training given a
+corpus in GIZA text format
+
+========================================================================
+Part II: How To Compile GIZA++
+
+In order to compile GIZA++ you may need:
+- recent version of the GNU compiler (2.95 or higher)
+- recent version of assembler and linker which do not have restrictions
+  with respect to the length of symbol names
+
+There is a make file in the src directory that will take care of the
+compilation. The most important targets are:
+
+GIZA++: generates an optimized version 
+
+GIZA++.dbg: generates the debug version 
+
+depend: generates the "dependencies" file (make this whenever you add
+source or header files to the package.
+
+========================================================================
+Part III: How To run GIZA++
+
+It's simple:
+
+GIZA++ [config-file] [options]
+
+All options which expect a parameter could also be used in the
+parameter file. For example the command line options
+
+	GIZA++ -S S.vcb -T T.vcb -C ST.snt
+
+corresponds to the config file:
+
+	S: S.vcb
+	T: T.vcb
+	C: ST.snt
+
+If you call GIZA++ without a parameter you get a list of all the
+options. The option names form GIZA are normally still valid. The
+default values of the parameters typically are optimized with respect
+to the corpora I use and typically give good results. It is
+nevertheless important that these parameters are always optimized for
+every new task.
+
+==========================================================================
+Part IV: Input File Formats
+
+A. VOCABULARY FILES
+
+Each entry is stored on one line as follows:
+
+ uniq_id1 string1 no_occurrences1
+ uniq_id2 string2 no_occurrences2
+ uniq_id3 string3 no_occurrences3
+ ....
+
+Here is a sample from an English vocabulary file:
+
+627 abandon 10
+628 abandoned 17
+629 abandoning 2
+630 abandonment 12
+631 abatement 8
+632 abbotsford 2
+
+uniq_ids are sequential positive integer numbers.  0 is reserved for
+the special token NULL.
+
+
+B. Bitext Files
+
+Each sentence pair is stored in three lines. The first line
+is the number of times this sentence pair occurred. The second line is
+the source sentence where each token is replaced by its unique integer
+id from the vocabulary file and the third is the target sentence in
+the same format.
+
+Here's a sample of 3 sentences from English/french corpus:
+
+1
+1 1 226 5008 621 6492 226 6377 6813 226 9505 5100 6824 226 5100 5222 0 614 10243 613
+2769 155 7989 585 1 578 6503 585 8242 578 8142 8541 578 12328 6595 8550 578 6595 6710 1
+1
+1 1 226 6260 11856 11806 1293
+11 1 1 11 155 14888 2649 11447 9457 8488 4168
+1
+1 1 226 7652 1 226 5337 226 6940 12089 5582 8076 12050
+1 1 155 4140 6812 153 1 154 155 14668 15616 10524 9954 1392
+
+C. Dictionary File
+
+This is optional. The dictionary file is of the format:
+
+target_word_id source_word_id
+
+The list should be sorted by the target_word_id.
+
+C. Dictionary Files
+
+If you provide a dictionary and list it in the configuration file,
+GIZA++ will change the cooccurrence counting in the first iteration
+of model 1 to honor the so-called "Dictionary Constraint":
+
+	In parallel sentences "e1 ... en" and "f1 ... fm",
+	ei and fi are counted as a coocurrence pair if one of two
+	conditions is met:  1.) ei and fi occur as an entry in the
+	dictionary, or 2.) ei does not occur in the dictionary with
+	any fj (1 <= j <= m) and fi does not occur in the dictionary
+	with any ej (1 <= j <= n).
+
+The dictionary must a list of pairs, one per line:
+
+	F E
+
+where F is an integer of a target token, and E is the integer of a
+source token.  F may be listed with other Es, and vice versa.
+
+Important:  The dictionary must be sorted by the F integers!
+
+==========================================================================
+Part V: Output File Formats:
+
+For file names, we will use the prefix "prob_table".  This can be
+changed using the -o switch.  The default is a combination of user id
+and time stamp.
+
+
+A. PROBABILITY TABLES
+
+Normally, Model1 is trained first, and the result is used to start
+Model2 training.  Then Model2 is transfered to Model3.  Model3 viterbi
+training follows.  This sequence can be adjusted by the various
+options, either on the command line or in a config file.
+
+1. T TABLE ( *.t3.* )
+
+(translation table)
+
+ prob_table.t1.n = t table after n iterations of Model1 training
+ prob_table.t2.n = t table after n iterations of Model2 training
+ prob_table.t2to3 = t table after transfering Model2 to Model3
+ prob_table.t3.n = t table after n iterations of Model3 training
+ prob_table.4.n = t table after n iterations of Model4 training
+
+Each line is of the following format:
+
+s_id t_id P(t_id/s_id)
+
+where: 
+ s_id: is the unique id for the source token
+ t_id: is the unique id for the target token
+ P(t_id/s_id) the probability of translating s_id as t_id
+
+sample part of a file:
+
+3599 5697 0.0628115
+2056 10686 0.000259988
+8227 3738 3.57132e-13
+5141 13720 5.52332e-12
+10798 4102 6.53047e-06
+8227 3750 6.97502e-14
+7712 14080 6.0365e-20
+7712 14082 2.68323e-17
+7713 1083 3.94464e-15
+7712 14084 2.98768e-15
+
+Similar files will be generated (with the prefix
+"prob_table.actual.xxx" that has the actual tokens instead of their
+unique ids). This is also true for fertility tables. Also the inverse
+probability table will be generated for the final table and it will
+have the infix "ti" .
+
+2. N TABLE ( *.n3.* )
+
+(Fertility table)
+ 
+ prob_table.n2to3 = n table estimated during the transfer from M2 to M3
+ ptob_table.n3.X = n table after X iterations of model3
+
+Each line in this file is of the following format:
+
+source_token_id p0 p1 p2 .... pn
+
+where p0 is the probability that the source token has zero fertility;
+p1, fertility one, ...., and n is the maximum possible fertility as
+defined in the program.
+
+sample:
+
+1 0.475861 0.282418 0.133455 0.0653083 0.0329326 0.00844979 0.0014008
+10 0.249747 0.000107778 0.307767 0.192208 0.0641439 0.15016 0.0358886
+11 0.397111 0.390421 0.19925 0.013382 2.21286e-05 0 0
+12 0.0163432 0.560621 0.374745 0.00231588 0 0 0
+13 1.78045e-07 0.545694 0.299573 0.132127 0.0230494 9.00322e-05 0
+14 1.41918e-18 0.332721 0.300773 0.0334969 0 0 0
+15 0 5.98626e-10 0.47729 0.0230955 0 0 0
+17 0 1.66346e-07 0.895883 0.103948 0 0 0
+
+
+3. P0 TABLE ( *.p0* )
+
+(1 - P0 is the probability of inserting a null after a
+   source word.)
+ 
+This file contains only one line with one real number which is the
+value of P0, the probability of not inserting a NULL token.
+
+
+4. A TABLE ( *.a[23].* )
+
+The file names follow the naming conventions above. The format of each
+line is as follows:
+
+i j l m p(i | j, l, m)
+
+where i, j, l, m are all integers and
+ j = position in target sentence
+ i = position in source sentence
+ l = length of source sentence
+ m = length of target sentence
+and p(i/j,l,m) is the probability that a source word in position i is
+moved to position j in a pair of sentences of length l and m.
+
+sample:
+
+15 14 15 14 0.630798
+15 14 15 15 0.414137
+15 14 15 16 0.268919
+15 14 15 17 0.23171
+15 14 15 18 0.117311
+15 14 15 19 0.119202
+15 14 15 20 0.111369
+15 14 15 21 0.0358169
+
+
+5. D3 TABLE ( *.d3.* )
+
+distortion table
+
+The format is similar to the A table with a slight difference --- the
+position of i & j are switched:
+
+j i l m p(j/i,l,m)
+
+sample:
+
+15 14 14 15 0.286397
+15 14 14 16 0.138898
+15 14 14 17 0.109712
+15 14 14 18 0.0868322
+15 14 14 19 0.0535823
+
+6. D4 TABLE: (( *.d4.* )
+
+distortion table for IBM-4
+
+7. D5 TABLE: ( *.d5.* )
+
+distortion table for IBM-5
+
+8. HMM TABLE: ( *.hhmm.* )
+
+alignment probability table for HMM alignment model
+
+B. ALIGNMENT FILE ( *.A3.* )
+
+In each iteration of the training, and for each sentence pair in the
+training set, the best alignment (viterbi alignment) is written to the
+alignment file (if the dump parameters are set accordingly). The
+alignment file is named prob_table.An.i, where n is the model number
+({1,2, 2to3, 3 or 4}), and i is the iteration number. The format of
+the alignments file is illustrated in the following sample:
+
+# Sentence pair (1)
+il s' agit de la m�me soci�t� qui a chang� de propri�taires
+NULL ({ }) UNK ({ }) UNK ({ }) ( ({ }) this ({ 4 11 }) is ({ }) the ({ }) same ({ 6 }) agency ({ }) which ({ 8 }) has ({ }) undergone ({ 1 2 3 7 9 10 12 }) a ({ }) change ({ 5 }) of ({ }) UNK ({ })
+# Sentence pair (2)
+UNK UNK , le propri�taire , dit que cela s' est produit si rapidement qu' il n' en conna�t pas la cause exacte
+NULL ({ 4 }) UNK ({ 1 2 }) UNK ({ }) , ({ 3 }) the ({ }) owner ({ 5 22 23 }) , ({ 6 }) says ({ 7 8 }) it ({ }) happened ({ 10 11 12 }) so ({ 13 }) fast ({ 14 19 }) he ({ 16 }) is ({ }) not ({ 20 }) sure ({ 15 17 }) what ({ }) went ({ 18 21 }) wrong ({ 9 })
+
+The alignment file is represented by three lines for each sentence
+pair. The first line is a label that can be used, e.g., as a caption
+for alignment visualization tools.  It contains information about the
+sentence sequential number in the training corpus, sentence lengths,
+and alignment probability. The second line is the target sentence, the
+third line is the source sentence. Each token in the source sentence
+is followed by a set of zero or more numbers. These numbers represent
+the positions of the target words to which this source word is
+connected, according to the alignment.
+
+
+C. Perplexity File ( *.perp )
+
+This file will be generated at the end of training. It summarizes
+perplexity values for each training iteration.  Here is a sample
+perplexity file that illustrates the format. The format is the same
+for cross entropy. If no test corpus was provided, the values for it
+will be set to "N/A".
+
+# train-size test-size iter. model train-perplexity test-perplexity final(y/n) train-viterbi-perp test-viterbi-perp
+	447136 9625 0 1 187067 186722 n 3.34328e+06 3.35352e+06
+	447136 9625 1 1 192.88 248.763 n 909.879 1203.13
+	447136 9625 2 1 99.45 139.214 n 316.363 459.745
+	447136 9625 3 1 83.4746 126.046 n 214.612 341.27
+	447136 9625 4 1 78.6939 124.914 n 179.218 303.169
+	447136 9625 5 2 76.6848 125.986 n 161.874 286.226
+	447136 9625 6 2 50.7452 86.2273 n 84.7227 151.701
+	447136 9625 7 2 42.9178 74.5574 n 63.6644 116.034
+	447136 9625 8 2 40.0651 70.7444 n 56.3186 104.274
+	447136 9625 9 2 38.8471 69.4105 n 53.1277 99.6044
+	447136 9625 10 2to3 38.2561 68.9576 n 51.4856 97.4414
+	447136 9625 11 3 129.993 248.885 n 86.6675 165.012
+	447136 9625 12 3 79.2212 169.902 n 86.4842 171.367
+	447136 9625 13 3 75.0746 164.488 n 84.9647 172.639
+	447136 9625 14 3 73.412 162.765 n 83.5762 172.797
+	447136 9625 15 3 72.6107 162.254 y 82.4575 172.688
+
+
+D. Revised Vocabulary files (*.src.vcb, *.trg.vcb)
+
+The revised vocabulary files are similar in format to the original
+vocabulary files. The only exceptions is that the frequency for each
+token is calculated from the given corpus (i.e. it is exact), which is
+not required in the input.
+
+E. final parameter file: ( *.gizacfg )
+
+This file includes all the parameter settings that were used in order
+to perform this training. This means that starting GIZA using this
+parameter file produces (should produce) the same training.
+
+
+
+Part VI: LITERATURE
+-------------------
+
+The following two articles include a comparison of the alignment
+models implemented in GIZA++:
+
+@INPROCEEDINGS{och00:isa,
+	AUTHOR = {F.~J.~Och and H.~Ney},
+	TITLE ={Improved Statistical Alignment Models},
+	BOOKTITLE = ACL00 ,
+	PAGES ={440--447},
+	ADDRESS={ Hongkong, China},
+	MONTH = {October},
+	YEAR = 2000}
+
+@INPROCEEDINGS{och00:aco,
+	AUTHOR =  {F.~J.~Och and H.~Ney},
+	TITLE = {A Comparison of Alignment Models for Statistical Machine Translation},
+	BOOKTITLE = COLING00,
+	ADDRESS	= {Saarbr\"ucken, Germany},
+	YEAR = {2000},
+	MONTH = {August},
+        PAGES = {1086--1090}
+	}
+
+The following article describes the statistical machine translation
+toolkit EGYPT:
+
+@MISC{ alonaizan99:smt,
+AUTHOR = {Y. Al-Onaizan and J. Curin and M. Jahr and K. Knight and J. Lafferty and I. D. Melamed and F. J. Och and D. Purdy and N. A. Smith and D. Yarowsky},
+TITLE = {Statistical Machine Translation, Final Report, {JHU} Workshop},
+YEAR = {1999},
+ADDRESS = {Baltimore, Maryland, MD},
+NOTE={{\tt http://www.clsp.jhu.edu/ws99/projects/ mt/final\_report/mt-final-report.ps}}
+}
+
+
+The implemented alignment models IBM-1 to IBM-5 and HMM were originally described in:
+
+@ARTICLE{brown93:tmo,
+	AUTHOR	= {Brown, P. F. and Della Pietra, S. A. and Della Pietra, V. J. and Mercer, R. L.},
+	TITLE	= {The Mathematics of Statistical Machine Translation: Parameter Estimation},
+	JOURNAL	= {Computational Linguistics},
+	YEAR	= 1993,
+	VOLUME	= 19,
+	NUMBER	= 2,
+	PAGES	= {263--311}
+}
+
+@INPROCEEDINGS{ vogel96:hbw,
+	AUTHOR	= {Vogel, S. and Ney, H. and Tillmann, C.},
+	TITLE	= {{HMM}-Based Word Alignment in Statistical Translation},
+	YEAR	= 1996,
+	PAGES	= {836--841},
+	MONTH	= {August},
+	ADDRESS	= {Copenhagen},
+	BOOKTITLE	= COLING96
+}
+
+
+Part VII: New features
+======================
+
+2003-06-09: 
+
+- new parameter "-nbestalignments N": prints an N-best list of
+  alignments into a file *.NBEST
+
+- If program is compiled with "-DBINARY_SEARCH_FOR_TTABLE", it uses
+  more memory-efficient data structures for the t table (vector with
+  binary search instead of hash table). Then, the program expects a
+  parameter "-CoocurrenceFile FILE" which specifies a file which
+  includes all lexical coccurrences in the training corpus. This file
+  can be produced by the snt2cooc.out tool.
+
+

tools/giza-pp/GIZA++-v2/TTables.cpp

@@ -0,0 +1,323 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+ 
+*/
+#include "TTables.h"
+#include "Parameter.h"
+
+GLOBAL_PARAMETER(float,PROB_CUTOFF,"PROB CUTOFF","Probability cutoff threshold for lexicon probabilities",PARLEV_OPTHEUR,1e-7);
+GLOBAL_PARAMETER2(float, COUNTINCREASE_CUTOFF,"COUNTINCREASE CUTOFF","countCutoff","Counts increment cutoff threshold",PARLEV_OPTHEUR,1e-6);
+
+#ifdef BINARY_SEARCH_FOR_TTABLE
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printCountTable(const char *, 
+					 const Vector<WordEntry>&, 
+					 const Vector<WordEntry>&,
+					 const bool) const
+{
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printProbTable(const char *filename, 
+					 const Vector<WordEntry>& evlist, 
+					 const Vector<WordEntry>& fvlist,
+					 const bool actual) const
+{
+  ofstream of(filename);
+  /*  for(unsigned int i=0;i<es.size()-1;++i)
+    for(unsigned int j=es[i];j<es[i+1];++j)
+      {
+	const CPPair&x=fs[j].second;
+	WordIndex e=i,f=fs[j].first;
+	if( actual )
+	  of << evlist[e].word << ' ' << fvlist[f].word << ' ' << x.prob << '\n';
+	else
+	  of << e << ' ' << f << ' ' << x.prob << '\n';
+	  }*/
+  for(unsigned int i=0;i<lexmat.size();++i)
+    {
+      if( lexmat[i] )
+	for(unsigned int j=0;j<lexmat[i]->size();++j)
+	  {
+	    const CPPair&x=(*lexmat[i])[j].second;
+	    WordIndex e=i,f=(*lexmat[i])[j].first;
+	    if( x.prob>PROB_SMOOTH )
+	      if( actual )
+		of << evlist[e].word << ' ' << fvlist[f].word << ' ' << x.prob << '\n';
+	      else
+		of << e << ' ' << f << ' ' << x.prob << '\n';
+	  }
+    }
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printProbTableInverse(const char *, 
+				   const Vector<WordEntry>&, 
+				   const Vector<WordEntry>&, 
+				   const double, 
+				   const double, 
+				   const bool ) const
+{
+}
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::normalizeTable(const vcbList&, const vcbList&, int)
+{
+  for(unsigned int i=0;i<lexmat.size();++i)
+    {
+      double c=0.0;
+      if( lexmat[i] )
+	{
+	  unsigned int lSize=lexmat[i]->size();
+	  for(unsigned int j=0;j<lSize;++j)
+	    c+=(*lexmat[i])[j].second.count;
+	  for(unsigned int j=0;j<lSize;++j)
+	    {
+	      if( c==0 )
+		(*lexmat[i])[j].second.prob=1.0/(lSize);
+	      else
+		(*lexmat[i])[j].second.prob=(*lexmat[i])[j].second.count/c;
+	      (*lexmat[i])[j].second.count=0;
+	    }
+	}
+    }
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::readProbTable(const char *){
+}
+
+template class tmodel<COUNT,PROB> ; 
+#else
+/* ------------------ Method Definiotns for Class tmodel --------------------*/
+
+#
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printCountTable(const char *filename, 
+					 const Vector<WordEntry>& evlist, 
+					 const Vector<WordEntry>& fvlist,
+					 const bool actual) const
+     // this function dumps the t table. Each line is of the following format:
+     //
+     // c(target_word/source_word) source_word target_word
+{
+  ofstream of(filename);
+  typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i;
+  for(i = ef.begin(); i != ef.end();++i){
+    if ( ((*i).second).count >  COUNTINCREASE_CUTOFF)
+      if (actual)
+	of <<  ((*i).second).count << ' ' << evlist[ ((*i).first).first ].word << ' ' << fvlist[((*i).first).second].word << ' ' << (*i).second.prob << '\n';
+      else 
+	of << ((*i).second).count << ' ' <<  ((*i).first).first  << ' ' << ((*i).first).second << ' ' << (*i).second.prob << '\n';
+  }
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printProbTable(const char *filename, 
+					 const Vector<WordEntry>& evlist, 
+					 const Vector<WordEntry>& fvlist,
+					 const bool actual) const
+     // this function dumps the t table. Each line is of the following format:
+     //
+     // source_word target_word p(target_word/source_word)
+{
+  ofstream of(filename);
+  typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i;
+  for(i = ef.begin(); i != ef.end();++i)
+    if( actual )
+      of << evlist[((*i).first).first].word << ' ' << 
+	fvlist[((*i).first).second].word << ' ' << (*i).second.prob << '\n';
+    else
+      of << ((*i).first).first << ' ' << ((*i).first).second << ' ' << 
+	(*i).second.prob << '\n';
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::printProbTableInverse(const char *filename, 
+				   const Vector<WordEntry>& evlist, 
+				   const Vector<WordEntry>& fvlist, 
+				   const double, 
+				   const double, 
+				   const bool actual) const
+  // this function dumps the inverse t table. Each line is of the format:
+  //
+  // target_word_id source_word_id p(source_word/target_word)
+  //
+  // if flag "actual " is true then print actual word entries instead of 
+  // token ids
+{
+  cerr << "Dumping the t table inverse to file: " << filename << '\n';
+  ofstream of(filename);
+  typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i;
+  PROB p_inv = 0 ;
+  //  static const PROB ratio(double(fTotal)/eTotal);
+  WordIndex e, f ;
+  int no_errors(0);
+  vector<PROB> total(fvlist.size(),PROB(0)) ; // Sum over all e of P(f/e) * p(e) - needed for normalization
+ 
+  for(i = ef.begin(); i != ef.end(); i++){
+    e = ((*i).first).first ;
+    f = ((*i).first).second ;
+    total[f] += (PROB) evlist[e].freq * ((*i).second.prob); //add P(f/ei) * F(ei) 
+  }
+  
+  for(i = ef.begin(); i != ef.end(); i++){
+    e = ((*i).first).first ;
+    f = ((*i).first).second ;
+    p_inv = ((*i).second.prob) * (PROB) evlist[e].freq / total[f] ;
+    if (p_inv > 1.0001 || p_inv < 0){
+      no_errors++;
+      if (no_errors <= 10){
+	cerr << "printProbTableInverse(): Error - P("<<evlist[e].word<<"("<<
+	  e<<") / "<<fvlist[f].word << "("<<f<<")) = " << p_inv <<'\n';
+	cerr << "f(e) = "<<evlist[e].freq << " Sum(p(f/e).f(e)) = " << total[f] <<
+	  " P(f/e) = " <<((*i).second.prob)  <<'\n';
+	if (no_errors == 10)
+	  cerr<<"printProbTableInverse(): Too many P inverse errors ..\n";
+      }
+    }
+    if (actual)
+      of << fvlist[f].word << ' ' << evlist[e].word << ' ' << p_inv << '\n';
+    else 
+      of << f << ' ' << e << ' ' << p_inv <<  '\n';
+  }
+}
+/*
+
+
+
+{
+  cerr << "Dumping the t table inverse to file: " << filename << '\n';
+  ofstream of(filename);
+  hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i;
+  PROB p_inv = 0 ;
+  static const PROB ratio(double(fTotal)/eTotal);
+  WordIndex e, f ;
+  for(i = ef.begin(); i != ef.end(); i++){
+    e = ((*i).first).first ;
+    f = ((*i).first).second ;
+    p_inv = ((*i).second.prob) * ratio * (PROB) evlist[e].freq / 
+      (PROB) fvlist[f].freq ;
+    if (actual)
+      of << fvlist[f].word << ' ' << evlist[e].word << ' ' << p_inv << '\n';
+    else 
+      of << f << ' ' << e << ' ' << p_inv <<  '\n';
+  }
+}
+*/
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::normalizeTable(const vcbList&engl, const vcbList&french, int iter)
+  // normalize conditional probability P(fj/ei):
+  // i.e. make sure that Sum over all j of P(fj/e) = 1  
+  // this method reads the counts portion of the table and normalize into
+  // the probability portion. Then the counts are cleared (i.e. zeroed)
+  // if the resulting probability of an entry is below a threshold, then 
+  // remove it .
+{
+  if( iter==2 )
+    {
+      total2.resize(engl.uniqTokens());for(unsigned int i=0;i<total2.size();i++)total2[i]=0.0;
+    }
+  nFrench.resize(engl.uniqTokens());for(unsigned int i=0;i<nFrench.size();i++)nFrench[i]=0;
+  nEng.resize(french.uniqTokens());for(unsigned int i=0;i<nEng.size();i++)nEng[i]=0;
+  Vector<double> total(engl.uniqTokens(),0.0);
+  //Vector<int> nFrench(engl.uniqTokens(), 0);
+  //Vector<int> nEng(french.uniqTokens(), 0);
+
+  typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i;
+  for(i = ef.begin(); i != ef.end(); i++){ // for all possible source words e
+    if( iter==2 )
+      total2[((*i).first).first] += (*i).second.count;
+    total[((*i).first).first] += (*i).second.count;
+    nFrench[((*i).first).first]++;
+    nEng[((*i).first).second]++;
+  }
+  for(unsigned int k=0;k<engl.uniqTokens();++k)
+    if( nFrench[k] )
+      {
+	double probMass=(french.uniqTokensInCorpus()-nFrench[k])*PROB_SMOOTH;
+	if( probMass<0.0 )
+	  cout << k << " french.uniqTokensInCorpus(): " << french.uniqTokensInCorpus() << "  nFrench[k]:"<< nFrench[k] << '\n';
+	total[k]+= total[k]*probMass/(1-probMass);
+      }
+  typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::iterator j, k;
+  PROB p ;
+  int nParams=0;
+  for(j = ef.begin(); j != ef.end(); ){
+    k = j;
+    k++ ;
+    if( (total[((*j).first).first])>0.0 )
+      p = ((((*j).second).count) /(total[((*j).first).first])) ;
+    else
+      p= 0.0;
+    if (p > PROB_CUTOFF)
+      {
+	if( iter>0 )
+	  {
+	    ((*j).second).prob = 0 ;
+	    ((*j).second).count = p ;
+	  }
+	else
+	  {
+	    ((*j).second).prob = p ;
+	    ((*j).second).count = 0 ;
+	  }
+	nParams++;
+      }
+    else {
+      erase(((*j).first).first, ((*j).first).second);
+    }
+    j = k ;
+  }
+  if( iter>0 )
+    return normalizeTable(engl, french, iter-1);
+  else
+    {
+    }
+}
+
+template <class COUNT, class PROB>
+void tmodel<COUNT, PROB>::readProbTable(const char *filename){
+  /* This function reads the t table from a file.
+     Each line is of the format:  source_word_id target_word_id p(target_word|source_word)
+     This is the inverse operation of the printTable function.
+     NAS, 7/11/99
+  */
+  ifstream inf(filename);
+  cerr << "Reading t prob. table from " << filename << "\n";
+  if(!inf){
+    cerr << "\nERROR: Cannot open " << filename << "\n";
+    return;
+  }
+  WordIndex src_id, trg_id;
+  PROB prob;
+  int nEntry=0;
+  while(    inf >> src_id  >> trg_id  >> prob){
+    insert(src_id, trg_id, 0.0, prob);
+    nEntry++;
+  }
+  cerr << "Read " << nEntry << " entries in prob. table.\n";
+}
+
+template class tmodel<COUNT,PROB> ; 
+
+/* ---------------- End of Method Definitions of class tmodel ---------------*/
+
+
+#endif

tools/giza-pp/GIZA++-v2/TTables.h

@@ -0,0 +1,417 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* --------------------------------------------------------------------------*
+ *                                                                           *
+ * Module : TTables                                                          *
+ *                                                                           *
+ * Prototypes File: TTables.h                                               *
+ *                                                                           *
+ * Objective: Defines clases and methods for handling I/O for Probability &  *
+ *            Count tables and also alignment tables                         *
+ *****************************************************************************/
+
+#ifndef _ttables_h
+#define _ttables_h 1
+
+
+#include "defs.h"
+#include "vocab.h"  
+
+#include <cassert>
+ 
+#include <iostream>
+#include <algorithm>
+#include <functional>
+#include <map>
+#include <set>
+#include "Vector.h"
+#include <utility>
+
+#include <fstream>
+
+#include "Globals.h"
+
+
+/* The tables defined in the following classes are defined as hash tables. For
+   example. the t-table is a hash function of a word pair; an alignment is 
+   a hash function of a vector of integer numbers (sentence positions) and so
+   on   */
+
+
+/*----------- Defnition of Hash Function for class tmodel ------- -----------*/
+
+typedef pair<WordIndex, WordIndex> wordPairIds;
+
+
+class hashpair : public unary_function< pair<WordIndex, WordIndex>, size_t >
+{
+public:
+  size_t operator() (const pair<WordIndex, WordIndex>& key) const
+    {
+      return (size_t) MAX_W*key.first + key.second; /* hash function and it 
+						       is guarnteed to have 
+						       unique id for each 
+						       unique pair */
+    }
+};
+
+
+
+/* ------------------ Class Prototype Definitions ---------------------------*
+  Class Name: tmodel
+  Objective: This defines the underlying data structur for t Tables and t 
+  Count Tables. They are defined as a hash table. Each entry in the hash table
+  is the probability (P(fj/ei) ) or count collected for ( C(fj/ei)). The 
+  probability and the count are represented as log integer probability as 
+  defined by the class LogProb .  
+
+  This class is used to represents t Tables (probabiliity) and n (fertility 
+  Tables and also their corresponding count tables .
+ 
+ *---------------------------------------------------------------------------*/
+
+//typedef float COUNT ;
+//typedef LogProb PROB ;
+template <class COUNT, class PROB>
+class LpPair {
+ public:
+  COUNT count ;
+  PROB  prob ;
+ public: // constructor 
+  LpPair():count(0), prob(0){} ;
+  LpPair(COUNT c, PROB p):count(c), prob(p){};
+} ;
+
+#ifdef BINARY_SEARCH_FOR_TTABLE
+
+
+template<class T>
+T*mbinary_search(T*x,T*y,unsigned int val)
+{
+  if( y-x==0 )
+    return 0;
+  if( x->first==val)
+    return x;
+  if( y-x<2 )
+    return 0;
+  T*mid=x+(y-x)/2;
+  if( val < mid->first )
+    return mbinary_search(x,mid,val);
+  else
+    return mbinary_search(mid,y,val);
+  
+}
+
+template<class T>
+const T*mbinary_search(const T*x,const T*y,unsigned int val)
+{
+  if( y-x==0 )
+    return 0;
+  if( x->first==val)
+    return x;
+  if( y-x<2 )
+    return 0;
+  const T*mid=x+(y-x)/2;
+  if( val < mid->first )
+    return mbinary_search(x,mid,val);
+  else
+    return mbinary_search(mid,y,val);
+  
+}
+
+template <class COUNT, class PROB>
+class tmodel{
+  typedef LpPair<COUNT, PROB> CPPair;
+ public:
+  int noEnglishWords;  // total number of unique source words
+  int noFrenchWords;   // total number of unique target words
+  //vector<pair<unsigned int,CPPair> > fs;
+  //vector<unsigned int> es;
+  vector< vector<pair<unsigned int,CPPair> >* > lexmat;
+
+  void erase(WordIndex e, WordIndex f)
+    {
+      CPPair *p=find(e,f);
+      if(p)
+	*p=CPPair(0,0);
+    };
+  CPPair*find(int e,int f)
+    {
+      //pair<unsigned int,CPPair> *be=&(fs[0])+es[e];
+      //pair<unsigned int,CPPair> *en=&(fs[0])+es[e+1];
+      pair<unsigned int,CPPair> *be=&(*lexmat[e])[0];
+      pair<unsigned int,CPPair> *en=&(*lexmat[e])[0]+(*lexmat[e]).size();
+      pair<unsigned int,CPPair> *x= mbinary_search(be,en,f);
+      if( x==0 )
+	{
+	  //cerr << "A:DID NOT FIND ENTRY: " << e << " " << f << '\n';
+	  //abort();
+	  return 0;
+	}
+      return &(x->second);
+    }
+  const CPPair*find(int e,int f)const
+    {
+      const pair<unsigned int,CPPair> *be=&(*lexmat[e])[0];
+      const pair<unsigned int,CPPair> *en=&(*lexmat[e])[0]+(*lexmat[e]).size();
+      //const pair<unsigned int,CPPair> *be=&(fs[0])+es[e];
+      //const pair<unsigned int,CPPair> *en=&(fs[0])+es[e+1];
+      const pair<unsigned int,CPPair> *x= mbinary_search(be,en,f);
+      if( x==0 )
+	{
+	  //cerr << "B:DID NOT FIND ENTRY: " << e << " " << f << '\n';
+	  //abort();
+	  return 0;
+	}
+
+      return &(x->second);
+    }
+public:
+  void insert(WordIndex e, WordIndex f, COUNT cval=0.0, PROB pval = 0.0){
+    *find(e,f)=CPPair(cval,pval);
+  }
+  CPPair*getPtr(int e,int f){return find(e,f);}
+  tmodel(const string&fn)
+    {
+      int count=0,count2=0;
+      ifstream infile2(fn.c_str());
+      int e,f,olde=-1,oldf=-1;
+      pair<unsigned int,CPPair> cp;
+      vector< pair<unsigned int,CPPair> > cps;
+      while(infile2>>e>>f)
+	{
+	  cp.first=f;
+	  assert(e>=olde);
+	  assert(e>olde ||f>oldf);
+	  if( e!=olde&&olde>=0 )
+	    {
+	      int oldsize=lexmat.size();
+	      lexmat.resize(olde+1);
+	      for(unsigned int i=oldsize;i<lexmat.size();++i)
+		lexmat[i]=0;
+	      lexmat[olde]=new vector< pair<unsigned int,CPPair> > (cps);
+	      cps.clear();
+	      if( !((*lexmat[olde]).size()==(*lexmat[olde]).capacity()) )
+		cerr << "eRROR: waste of memory: " << (*lexmat[olde]).size() << " " << (*lexmat[olde]).capacity() << endl;
+	      count2+=lexmat[olde]->capacity();
+	    }
+	  cps.push_back(cp);
+	  olde=e;
+	  oldf=f;
+	  count++;
+	}
+      lexmat.resize(olde+1);
+      lexmat[olde]=new vector< pair<unsigned int,CPPair> > (cps);
+      count2+=lexmat[olde]->capacity();      
+      cout << "There are " << count << " " << count2 << " entries in table" << '\n';
+    }
+
+
+  /*  tmodel(const string&fn)
+    {
+      size_t count=0;
+      {
+	ifstream infile1(fn.c_str());
+	if( !infile1 )
+	  {
+	    cerr << "ERROR: can't read coocurrence file " << fn << '\n';
+	    abort();
+	  }
+	int e,f;
+	while(infile1>>e>>f)
+	  count++;
+      }
+      cout << "There are " << count << " entries in table" << '\n';
+      ifstream infile2(fn.c_str());
+      fs.resize(count);
+      int e,f,olde=-1,oldf=-1;
+      pair<unsigned int,CPPair> cp;
+      count=0;
+      while(infile2>>e>>f)
+	{
+	  assert(e>=olde);
+	  assert(e>olde ||f>oldf);
+	  if( e!=olde )
+	    {
+	      es.resize(e+1);
+	      for(unsigned int i=olde+1;int(i)<=e;++i)
+		es[i]=count;
+	    }
+	  cp.first=f;
+	  assert(count<fs.size());
+	  fs[count]=cp;
+	  //fs.push_back(cp);
+	  olde=e;
+	  oldf=f;
+	  count++;
+	}
+      assert(count==fs.size());
+      es.push_back(fs.size());
+      cout << fs.size() << " " << count << " coocurrences read" << '\n';
+      }*/
+  void incCount(WordIndex e, WordIndex f, COUNT inc) 
+    {
+      if( inc )
+	{
+	  CPPair *p=find(e,f);
+	  if( p )
+	    p->count += inc ;
+	}
+    }
+
+  PROB getProb(WordIndex e, WordIndex f) const
+    {
+      const CPPair *p=find(e,f);
+      if( p )
+	return max(p->prob, PROB_SMOOTH);
+      else
+	return PROB_SMOOTH;
+    }
+
+  COUNT getCount(WordIndex e, WordIndex f) const
+    {
+      const CPPair *p=find(e,f);
+      if( p )
+	return p->count;
+      else
+	return 0.0;
+    }
+
+  void printProbTable(const char* filename, const Vector<WordEntry>&, const Vector<WordEntry>&,bool actual) const;
+  void printCountTable(const char* filename, const Vector<WordEntry>&, const Vector<WordEntry>&,bool actual) const;
+  void printProbTableInverse(const char *filename, 
+			     const Vector<WordEntry>& evlist, 
+			     const Vector<WordEntry>& fvlist, 
+			     const double eTotal, 
+			     const double fTotal, 
+			     const bool actual = false ) const;
+  void normalizeTable(const vcbList&engl, const vcbList&french, int iter=2);
+  void readProbTable(const char *filename); 
+};
+
+
+#else
+
+
+template <class COUNT, class PROB>
+class tmodel{
+  typedef LpPair<COUNT, PROB> CPPair;
+ public:
+  int noEnglishWords;  // total number of unique source words
+  int noFrenchWords;   // total number of unique target words
+  hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> > ef;
+  void erase(WordIndex e, WordIndex f)
+  // In: a source and a target token ids.
+  // removes the entry with that pair from table
+    {
+      ef.erase(wordPairIds(e, f));
+    };
+
+public:
+  Vector<PROB> total2;
+  Vector<int> nFrench;
+  Vector<int> nEng;
+
+ 
+  // methods;
+
+  // insert: add entry P(fj/ei) to the hash function, Default value is 0.0 
+  void insert(WordIndex e, WordIndex f, COUNT cval=0.0, PROB pval = 0.0){
+    ef[wordPairIds(e, f)].count = cval ;
+    ef[wordPairIds(e, f)].prob = pval ;
+  }
+
+  // returns a reference to the word pair, if does not exists, it creates it.
+  CPPair&getRe(WordIndex e, WordIndex f)
+    {return ef[wordPairIds(e, f)];}
+
+  // returns a pointer to an existing word pair. if pair does not exists, 
+  // the method returns the zero pointer (NULL)
+
+  CPPair*getPtr(WordIndex e, WordIndex f) 
+    {      
+      // look up this pair and return its position
+      typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::iterator i = ef.find(wordPairIds(e, f)); 
+      if(i != ef.end())  // if it exists, return a pointer to it.
+	return(&((*i).second));
+      else return(0) ; // else return NULL pointer
+    }
+
+  void incCount(WordIndex e, WordIndex f, COUNT inc) 
+    // increments the count of the given word pair. if the pair does not exist, 
+    // it creates it with the given value.
+    {
+      if( inc )
+	ef[wordPairIds(e, f)].count += inc ;
+    }
+
+  PROB getProb(WordIndex e, WordIndex f) const
+    // read probability value for P(fj/ei) from the hash table 
+    // if pair does not exist, return floor value PROB_SMOOTH
+    {
+      typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i= ef.find(wordPairIds(e, f));
+      if(i == ef.end())  
+	return PROB_SMOOTH; 
+      else
+	return max(((*i).second).prob, PROB_SMOOTH);
+    }
+
+  COUNT getCount(WordIndex e, WordIndex f) const
+    /* read count value for entry pair (fj/ei) from the hash table */
+    {
+      typename hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >::const_iterator i= ef.find(wordPairIds(e, f));
+      if(i == ef.end())
+	return 0; 
+      else
+	return ((*i).second).count;
+    }
+
+  inline const hash_map<wordPairIds, CPPair, hashpair, equal_to<wordPairIds> >& getHash(void) const {return ef;};
+  /* get a refernece to the hash table */
+  //inline void resize(WordIndex n) {ef.resize(n);};
+  // to resize he hash table 
+
+  void printProbTable(const char* filename, const Vector<WordEntry>&, const Vector<WordEntry>&,bool actual) const;
+  void printCountTable(const char* filename, const Vector<WordEntry>&, const Vector<WordEntry>&,bool actual) const;
+  // print the t table to the given file but this time print actual source and
+  //     target words instead of thier token ids 
+  
+  void printProbTableInverse(const char *filename, 
+			     const Vector<WordEntry>& evlist, 
+			     const Vector<WordEntry>& fvlist, 
+			     const double eTotal, 
+			     const double fTotal, 
+			     const bool actual = false ) const;
+  // dump  inverse of t table (i.e P(ei/fj)) to the given file name, 
+  //  if the given flag is true then actual words are printed not token ids 
+  
+  void normalizeTable(const vcbList&engl, const vcbList&french, int iter=2);
+  // to norlmalize the table i.e. make sure P(fj/ei) for all j is equal to 1 
+
+  void readProbTable(const char *filename); 
+  //  void readAsFertilityTable(const char *filename);
+};
+/*--------------- End of Class Definition for tmodel -----------------------*/ 
+
+#endif
+
+#endif

tools/giza-pp/GIZA++-v2/Vector.h

@@ -0,0 +1,427 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/*--
+Vector: checked vector implementation
+
+Franz Josef Och (30/07/99)
+--*/
+#ifndef ARRAY_H_DEFINED
+#define ARRAY_H_DEFINED
+#include "mystl.h"
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <functional>
+#include <cassert>
+
+
+#ifdef NDEBUG
+
+#include <vector>
+#define Vector vector
+template<class T> ostream& operator<<(ostream&o, const Vector<T>&a)
+{
+  o << "Vector(" << a.size() << "){ ";
+  for(unsigned int iii=0;iii<a.size();iii++)
+    o << " " << iii<< ": " << a[iii]<<" ;";
+  return o << "}\n";
+}
+
+#else
+
+#define ARRAY_DEBUG
+#define memo_del(a, b)
+#define memo_new(a)
+
+template<class T> class Vector
+{
+ private:
+  T *p;                  	
+  int realSize;
+  int maxWritten;
+
+  void copy(T *a, const T *b, int n);
+  void copy(T *a, T *b, int n);
+  void _expand();
+		
+ public:
+  Vector() 
+    : p(0), realSize(0), maxWritten(-1)
+    {
+#ifdef VERY_ARRAY_DEBUG
+      cout << "MAKE ARRAY: " << this<<" "<<(void*)p << '\n';
+#endif
+    }
+  Vector(const Vector<T> &x)
+    : p(new T[x.maxWritten+1]), realSize(x.maxWritten+1), maxWritten(x.maxWritten)
+    {
+      memo_new(p);
+      copy(p, x.p, realSize);
+#ifdef VERY_ARRAY_DEBUG
+      cout << "MAKE ARRAY copy: " << this << " " << realSize <<" "<<(void*)p<< '\n';
+#endif
+    }
+  explicit Vector(int n)
+    : p(new T[n]), realSize(n), maxWritten(n-1)
+    {
+      memo_new(p);
+#ifdef VERY_ARRAY_DEBUG
+      cout << "MAKE ARRAY with parameter n: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+    }
+  Vector(int n, const T&_init)
+    : p(new T[n]), realSize(n), maxWritten(n-1)
+    {
+      memo_new(p);
+      for(int iii=0;iii<n;iii++)p[iii]=_init;
+#ifdef VERY_ARRAY_DEBUG
+      cout << "MAKE ARRAY with parameter n and init: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+    }
+  
+  ~Vector() 
+    { 
+#ifdef VERY_ARRAY_DEBUG
+      cout << "FREE ARRAY: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+      delete [] p;
+      memo_del(p, 1);
+#ifndef NDEBUG
+      p=0;realSize=-1;maxWritten=-1;
+#endif
+    }
+  
+  Vector<T>& operator=(const Vector<T>&x)
+    {
+      if( this!= &x )
+	{
+#ifdef VERY_ARRAY_DEBUG
+	  cout << "FREE ARRAY because of operator=: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif	
+	  delete [] p;
+	  memo_del(p, 1);
+	  realSize = x.maxWritten+1;
+	  maxWritten = x.maxWritten;
+	  p = new T[realSize]; 
+	  memo_new(p);
+	  copy(p, x.p, realSize);
+#ifdef VERY_ARRAY_DEBUG
+	  cout << "NEW ARRAY because of operator=: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+	}
+      return *this;
+    }
+  
+  Vector<T>& operator=(Vector<T>&x)
+    {
+      if( this!= &x )
+	{
+#ifdef VERY_ARRAY_DEBUG
+	  cout << "FREE ARRAY because of operator=: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+	  delete [] p;
+	  memo_del(p, 1);
+	  realSize = x.maxWritten+1;
+	  maxWritten = x.maxWritten;
+	  p = new T[realSize]; 
+	  memo_new(p);
+	  copy(p, x.p, realSize);
+#ifdef VERY_ARRAY_DEBUG
+	  cout << "NEW ARRAY because of operator=: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+	}
+      return *this;
+    }
+  
+  void allowAccess(int n) 
+    { 
+      while( realSize<=n )
+	_expand(); 
+      maxWritten=max(maxWritten, n);
+      assert( maxWritten<realSize );
+    }
+  void resize(int n)
+    {
+      while( realSize<n ) 
+	_expand(); 
+      maxWritten=n-1;
+    }
+  void clear()
+    {
+      resize(0);
+    }
+  void reserve(int n)
+    {
+      int maxOld=maxWritten;
+      resize(n);
+      maxWritten=maxOld;
+    }
+  void sort(int until=-1)
+    {
+      if( until== -1 ) until=size();
+      std::sort(p, p+until);
+    }
+  void invsort(int until=-1)
+    {
+      if( until== -1 ) until=size();
+      std::sort(p, p+until, greater<T>());      
+    }
+  void init(int n, const T&_init)
+    {
+#ifdef VERY_ARRAY_DEBUG
+      cout << "FREE ARRAY because of init: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+      delete []p;
+      memo_del(p, 1);
+      p=new T[n];
+      memo_new(p);
+      realSize=n;
+      maxWritten=n-1;
+      for(int iii=0;iii<n;iii++)p[iii]=_init;
+#ifdef VERY_ARRAY_DEBUG
+      cout << "NEW ARRAY because of init: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+    }
+  inline unsigned int size() const
+    {assert( maxWritten<realSize );
+    return maxWritten+1;}
+  inline int low() const
+    { return 0; }
+  inline int high() const
+    { return maxWritten; }
+  int findMax() const;
+  int findMin() const;
+  void errorAccess(int n) const;
+  inline T*getPointerToData(){return p;}
+  inline T*begin(){return p;}
+  inline T*end(){return p+maxWritten+1;}
+  inline T& operator[](int n)
+    { 
+#ifndef NDEBUG
+      if( n<0 || n>maxWritten )
+	errorAccess(n);
+#endif
+      return p[n];
+    }
+  inline const T& operator[](int n) const 
+    { 
+#ifndef NDEBUG
+      if(n<0 || n>maxWritten )
+	errorAccess(n);
+#endif
+      return p[n]; 
+    }
+  inline const T& get(int n) const 
+    { 
+#ifndef NDEBUG
+      if(n<0 || n>maxWritten )
+	errorAccess(n);
+#endif      
+      return p[n]; 
+    }
+  const T&top(int n=0) const
+    {return (*this)[maxWritten-n];}
+  T&top(int n=0)
+    {return (*this)[maxWritten-n];}
+  const T&back(int n=0) const
+    {return (*this)[maxWritten-n];}
+  T&back(int n=0)
+    {return (*this)[maxWritten-n];}
+  T&push_back(const T&x)
+    {     
+      allowAccess(maxWritten+1);
+      (*this)[maxWritten]=x;
+      return top();
+    }
+    /*
+  bool writeTo(ostream&out) const
+    {
+      out << "Vector ";
+      out << size() << " ";
+      out << a << '\n';
+      for(int iv=0;iv<=maxWritten;iv++)
+	{
+	  writeOb(out, (*this)[iv]);
+	  out << '\n';
+	}
+      return 1;
+    }
+    */
+    
+  bool readFrom(istream&in)
+    {
+      string s;
+      if( !in )
+	{
+	  cerr << "ERROR(Vector): file cannot be opened.\n";
+	  return 0;
+	}
+      in >> s;
+      if( !(s=="Vector") )
+	{
+	  cerr << "ERROR(Vector): Vector!='"<<s<<"'\n";
+	  return 0;
+	}
+      int biggest;
+      in >> biggest;
+      in >> a;
+      resize(biggest);
+      for(int iv=0;iv<size();iv++)
+	{
+	  readOb(in, (*this)[iv]);
+	}
+      return 1;
+    }
+};
+
+template<class T> bool operator==(const Vector<T> &x, const Vector<T> &y)
+{
+  if( &x == &y )
+    return 1;
+  else
+    {
+      if( y.size()!=x.size() )
+	return 0;
+      else
+	{
+	  for(unsigned int iii=0;iii<x.size();iii++)
+	    if( !(x[iii]==y[iii]) )
+	      return 0;
+	  return 1;
+	}
+    }
+}
+template<class T> bool operator!=(const Vector<T> &x, const Vector<T> &y)
+{
+  return !(x==y);
+}
+
+template<class T> bool operator<(const Vector<T> &x, const Vector<T> &y)
+{
+  if( &x == &y )
+    return 0;
+  else
+    {
+      if( y.size()<x.size() )
+	return !(y<x);
+      for(int iii=0;iii<x.size();iii++)
+	{
+	  assert( iii!=y.size() );
+	  if( x[iii]<y[iii] )
+	    return 1;
+	  else if( y[iii]<x[iii] )
+	    return 0;
+	}
+      return x.size()!=y.size();//??
+    }
+}
+
+
+template<class T> void Vector<T>:: errorAccess(int n) const
+{
+  cerr 	<< "ERROR: Access to array element " << n 
+	<< " (" << maxWritten << ", " << realSize << ", " << (void*)p << ")\n";
+  cout <<  "ERROR: Access to array element " << n 
+       << " (" << maxWritten << ", " << realSize << ", " << (void*)p << ")\n";
+  assert(0);
+#ifndef DEBUG
+  abort();
+#endif
+}
+
+template<class T> ostream& operator<<(ostream&o, const Vector<T>&a)
+{
+  o << "Vector(" << a.size() << "){ ";
+  for(unsigned int iii=0;iii<a.size();iii++)
+    o << " " << iii<< ": " << a[iii]<<" ;";
+  return o << "}\n";
+}
+
+template<class T> istream& operator>>(istream&in, Vector<T>&)
+{return in;}
+
+template<class T> int Hash(const Vector<T>&a)
+{
+  int n=0;
+  for(int iii=0;iii<a.size();iii++)
+    n+=Hash(a[iii])*(iii+1);
+  return n+a.size()*47;
+}
+template<class T> void Vector<T>::copy(T *aa, const T *bb, int n)
+{
+  for(int iii=0;iii<n;iii++)
+    aa[iii]=bb[iii];
+}
+template<class T> void Vector<T>::copy(T *aa, T *bb, int n)
+{
+  for(int iii=0;iii<n;iii++)
+    aa[iii]=bb[iii];
+}
+
+template<class T> void Vector<T>::_expand()
+{
+#ifdef VERY_ARRAY_DEBUG
+  cout << "FREE ARRAY because of _expand: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+  T *oldp=p;
+  int oldsize=realSize;
+  realSize=realSize*2+1;
+  p=new T[realSize];
+  memo_new(p);
+  copy(p, oldp, oldsize);
+  delete [] oldp;
+  memo_del(oldp, 1);
+#ifdef VERY_ARRAY_DEBUG
+  cout << "NEW ARRAY because of _expand: " << this << " " << realSize<<" "<<(void*)p << '\n';
+#endif			
+}
+
+template<class T> int Vector<T>::findMax() const
+{
+  if( size()==0 )
+    return -1;
+  else
+    {
+      int maxPos=0;
+      for(int iii=1;iii<size();iii++)
+	if( (*this)[maxPos]<(*this)[iii] )
+	  maxPos=iii;
+      return maxPos;
+    }
+}
+template<class T> int Vector<T>::findMin() const
+{
+  if( size()==0 )
+    return -1;
+  else
+    {
+      int minPos=0;
+      for(int iii=1;iii<size();iii++)
+	if( (*this)[iii]<(*this)[minPos] )
+	  minPos=iii;
+      return minPos;
+    }
+}
+
+#endif
+
+#endif

tools/giza-pp/GIZA++-v2/WordClasses.h

@@ -0,0 +1,96 @@
+/*
+
+Copyright (C) 2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef WordClasses_h_DEFINED
+#define WordClasses_h_DEFINED
+#include <map>
+#include <string>
+#include <set>
+
+class WordClasses
+{
+ private:
+  map<string,string> Sw2c;
+  map<string,int> Sc2int;
+  Vector<string> Sint2c;
+  Vector<int> w2c;
+  unsigned int classes;
+ public:
+  WordClasses() 
+    : classes(1) 
+    {
+      Sint2c.push_back("0");
+      Sc2int["0"]=0;
+    }
+  template<class MAPPER> bool read(istream&in,const MAPPER&m)
+    {
+      string sline;
+      int maxword=0;
+      while(getline(in,sline))
+	{
+	  string word,wclass;
+	  //istringstream iline(sline.c_str());
+	  istringstream iline(sline);
+	  iline>>word>>wclass;
+	  maxword=max(m(word),maxword);
+	  assert(Sw2c.count(word)==0);
+	  Sw2c[word]=wclass;
+	  if( !Sc2int.count(wclass) )
+	    {
+	      Sc2int[wclass]=classes++;
+	      Sint2c.push_back(wclass);
+	      assert(classes==Sint2c.size());
+	    }
+	}
+      w2c=Vector<int>(maxword+1,0);
+      for(map<string,string>::const_iterator i=Sw2c.begin();i!=Sw2c.end();++i)
+	w2c[m(i->first)]=Sc2int[i->second];
+      cout << "Read classes: #words: " << maxword << " " << " #classes: "<< classes <<endl;
+      return 1;
+    }
+  int getClass(int w)const
+    {
+      if(w>=0&&int(w)<int(w2c.size()) )
+	return w2c[w];
+      else
+	return 0;
+    }
+  int operator()(const string&x)const
+    {
+      if( Sc2int.count(x) )
+	return Sc2int.find(x)->second;
+      else
+	{
+	  cerr << "WARNING:  class " << x << " not found.\n";
+	  return 0;
+	}
+    }
+  string classString(unsigned int cnr)const
+    {
+      if( cnr<Sint2c.size())
+	return Sint2c[cnr];
+      else
+	return string("0");
+    }
+};
+
+#endif

tools/giza-pp/GIZA++-v2/alignment.cpp

@@ -0,0 +1,38 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/*--
+alignment: 'checked' alignment representation with automatic calculation
+           of fertilities
+Franz Josef Och (30/07/99)
+--*/
+#include "alignment.h"
+
+ostream&operator<<(ostream&out, const alignment&a)
+{
+  int m=a.a.size()-1,l=a.f.size()-1;
+  out << "AL(l:"<<l<<",m:"<<m<<")(a: ";
+  for(int j=1;j<=m;j++)out << a(j) << ' ';
+  out << ")(fert: ";
+  for(int i=0;i<=l;i++)out << a.fert(i) << ' ';
+  return out << ")  c:"<<"\n";
+}
+

tools/giza-pp/GIZA++-v2/alignment.h

@@ -0,0 +1,227 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/*--
+alignment: 'checked' alignment representation with autom. calc. of fertilities
+Franz Josef Och (30/07/99)
+--*/
+#ifndef alignment_h_fjo_defined
+#define alignment_h_fjo_defined
+#include "Vector.h"
+#include <cassert>
+#include "defs.h"
+#include "myassert.h"
+
+class al_struct
+{
+ public:
+  al_struct()
+    : prev(0),next(0){}
+  PositionIndex prev,next;
+};
+
+
+class alignment
+{
+ private:
+  Vector<PositionIndex> a;
+  Vector<PositionIndex> positionSum,f;
+ public:
+  Vector<PositionIndex> als_i;
+  Vector<al_struct>  als_j;
+  PositionIndex l,m;
+  alignment()
+    {}
+  alignment(PositionIndex _l, PositionIndex _m)
+    : a(_m+1, (PositionIndex)0),
+    positionSum(_l+1, (PositionIndex)0), f(_l+1, (PositionIndex)0), als_i(_l+1,0),als_j(_m+1),l(_l), m(_m)
+    {
+      f[0]=m;
+      for(PositionIndex j=1;j<=m;j++)
+	{
+	  if( j>1 )
+	    als_j[j].prev= j-1;
+	  if( j<m )
+	    als_j[j].next= j+1;
+	}
+      als_i[0]=1;
+    }
+  PositionIndex get_l()const
+    {return l;}
+  PositionIndex get_m()const
+    {return m;}
+  void doMove(int i,int j)
+    {
+      set(j,i);
+    }
+  void doSwap(int j1,int j2)
+    {
+      int aj1=a[j1],aj2=a[j2];
+      set(j1,aj2);
+      set(j2,aj1);
+    }
+  void set(PositionIndex j, PositionIndex aj)
+    {
+      PositionIndex old_aj=a[j];
+      massert(j<a.size());massert(aj<f.size());
+      massert(old_aj<f.size());massert(f[old_aj]>0);
+      massert(j>0);
+      positionSum[old_aj]-=j;
+      // ausfuegen
+      PositionIndex prev=als_j[j].prev;
+      PositionIndex next=als_j[j].next;
+      if( next )
+	als_j[next].prev=prev;
+      if( prev )
+	als_j[prev].next=next;
+      else
+	als_i[old_aj]=next;
+      
+      // neue Position suchen
+      PositionIndex lfd=als_i[aj],llfd=0;
+      while( lfd && lfd<j )
+	lfd = als_j[llfd=lfd].next;
+
+      // einfuegen
+      als_j[j].prev=llfd;
+      als_j[j].next=lfd;
+      if( llfd )
+	als_j[llfd].next=j;
+      else
+	als_i[aj]=j;
+      if( lfd )
+	als_j[lfd].prev=j;
+
+      f[old_aj]--;
+      positionSum[aj]+=j;
+      f[aj]++;
+      a[j]=aj;
+    }
+  const Vector<PositionIndex>& getAlignment() const 
+    {return a ;}
+  PositionIndex get_al(PositionIndex j)const
+    {
+      massert(j<a.size());
+      return a[j];
+    }
+  PositionIndex operator()(PositionIndex j)const
+    {
+      massert(j<a.size());
+      return a[j];
+    }
+  PositionIndex fert(PositionIndex i)const
+    {
+      massert(i<f.size());
+      return f[i];
+    }
+  PositionIndex get_head(PositionIndex i)const
+    {
+      massert( als_i[i]==_get_head(i) );
+      return als_i[i];
+    }
+  PositionIndex get_center(PositionIndex i)const
+    {
+      if( i==0 )return 0;
+      massert(((positionSum[i]+f[i]-1)/f[i]==_get_center(i)));
+      return (positionSum[i]+f[i]-1)/f[i];
+    }
+  PositionIndex _get_head(PositionIndex i)const
+    {
+      if( fert(i)==0 )return 0;
+      for(PositionIndex j=1;j<=m;j++)
+	if( a[j]==i )
+	  return j;
+      return 0;
+    }
+  PositionIndex _get_center(PositionIndex i)const
+    {
+      if( i==0 )return 0;
+      massert(fert(i));
+      PositionIndex sum=0;
+      for(PositionIndex j=1;j<=m;j++)
+	if( a[j]==i )
+	  sum+=j;
+      return (sum+fert(i)-1)/fert(i);
+    }
+  PositionIndex prev_cept(PositionIndex i)const
+    {
+      if( i==0 )return 0;
+      PositionIndex k=i-1;
+      while(k&&fert(k)==0)
+	k--;
+      return k;
+    }
+  PositionIndex next_cept(PositionIndex i)const
+    {
+      PositionIndex k=i+1;
+      while(k<l+1&&fert(k)==0)
+	k++;
+      return k;
+    }
+  PositionIndex prev_in_cept(PositionIndex j)const
+    {
+      //PositionIndex k=j-1;
+      //while(k&&a[k]!=a[j])
+      //k--;
+      //assert( als_j[j].prev==k );
+      //assert(k);
+      //return k;
+      massert(als_j[j].prev==0||a[als_j[j].prev]==a[j]);
+      return als_j[j].prev;
+    }
+  friend ostream &operator<<(ostream&out, const alignment&a);
+  friend bool operator==(const alignment&a, const alignment&b)
+    {
+      massert(a.a.size()==b.a.size());
+      for(PositionIndex j=1;j<=a.get_m();j++)
+	if(a(j)!=b(j))
+	  return 0;
+      return 1;
+    }
+  friend bool operator<(const alignment&x, const alignment&y)
+    {
+      massert(x.get_m()==y.get_m());
+      for(PositionIndex j=1;j<=x.get_m();j++)
+	if( x(j)<y(j) ) 
+	  return 1;
+	else if( y(j)<x(j) )
+	  return 0;
+      return 0;
+    }
+  friend int differences(const alignment&x, const alignment&y){
+    int count=0;
+    massert(x.get_m()==y.get_m());
+    for(PositionIndex j=1;j<=x.get_m();j++)
+      count += (x(j)!=y(j));
+    return count;
+  }
+  bool valid()const
+    {
+      if( 2*f[0]>m )
+	return 0;
+      for(unsigned int i=1;i<=l;i++)
+	if( f[i]>=MAX_FERTILITY )
+	  return 0;
+      return 1;
+    }
+  friend class transpair_model5;
+};
+#endif

tools/giza-pp/GIZA++-v2/collCounts.cpp

@@ -0,0 +1,293 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "alignment.h"
+#include "transpair_model3.h"
+#include <map>
+#include "collCounts.h"
+#include "MoveSwapMatrix.h"
+#include "D5Tables.h"
+#include "transpair_model5.h"
+#include "transpair_modelhmm.h"
+#include "Parameter.h"
+
+extern float COUNTINCREASE_CUTOFF_AL;
+// unifies collectCountsOverAlignments and findAlignmentNeighborhood FJO-20/07/99
+template<class TRANSPAIR>
+int collectCountsOverNeighborhood(const MoveSwapMatrix<TRANSPAIR>&msc,LogProb ascore,Array2<LogProb,Vector<LogProb> >&dtcount,Array2<LogProb,Vector<LogProb> >&ncount,LogProb&p1count,LogProb&p0count,LogProb&total_count)
+{
+  int nAl=0;
+  const PositionIndex l=msc.get_l(),m=msc.get_m();
+  Array2<LogProb,Vector<LogProb> > cmove(l+1,m+1),cswap(l+1,m+1);
+  Vector<LogProb> negmove(m+1),negswap(m+1),plus1fert(l+1),minus1fert(l+1);
+  LogProb total_move,total_swap;
+  if( msc.isCenterDeleted()==0 )
+    {
+      total_move+=ascore;
+      nAl++;
+    }
+  for(PositionIndex j=1;j<=m;j++)
+      for(PositionIndex i=0;i<=l;i++)
+	if( msc(j)!=i && !msc.isDelMove(i,j) )
+	  {
+	    LogProb newscore=ascore*msc.cmove(i,j);
+	    total_move+=newscore;
+	    nAl++;
+	    cmove(i,j)+=newscore;
+	    negmove[j]+=newscore;
+	    plus1fert[i]+=newscore;
+	    minus1fert[msc(j)]+=newscore;
+	  }
+  for(PositionIndex j1=1;j1<=m;j1++)
+    for(PositionIndex j2=j1+1;j2<=m;j2++)
+      if( msc(j1)!=msc(j2) && !msc.isDelSwap(j1,j2) )
+	{
+	  LogProb newscore=ascore*msc.cswap(j1,j2);
+	  total_swap+=newscore;
+	  nAl++;
+	  cswap(msc(j1),j2)+=newscore;
+	  cswap(msc(j2),j1)+=newscore;
+	  negswap[j1]+=newscore; 
+	  negswap[j2]+=newscore;
+	}
+  total_count+=total_move+total_swap;
+  for(PositionIndex j=1;j<=m;j++)
+    for(PositionIndex i=0;i<=l;i++)
+      dtcount(i,j) += ((i==msc(j)) ? (total_count-(negmove[j]+negswap[j])) : (cswap(i,j)+cmove(i,j)));
+  for(PositionIndex i=1;i<=l;i++)
+    {
+      LogProb temp=minus1fert[i]+plus1fert[i];
+      if( msc.fert(i)<MAX_FERTILITY )
+	ncount(i,msc.fert(i))+=total_count-temp;
+      if(msc.fert(i)>0&&msc.fert(i)-1<MAX_FERTILITY)
+	ncount(i,msc.fert(i)-1)+=minus1fert[i];
+      else
+	if( minus1fert[i]!=0.0 )
+	  cerr << "ERROR: M1Fa: " << minus1fert[i] << ' ' << i << ' ' << msc.fert(i)<< endl;
+      if(msc.fert(i)+1<MAX_FERTILITY) 
+	ncount(i,msc.fert(i)+1)+=plus1fert[i];
+    }
+  LogProb temp=minus1fert[0]+plus1fert[0];
+  p1count += (total_count-temp)*(LogProb)msc.fert(0);
+  p0count += (total_count-temp)*(LogProb)(m-2*msc.fert(0));
+  if( msc.fert(0)>0 )
+    {
+      p1count += (minus1fert[0])*(LogProb)(msc.fert(0)-1);
+      p0count += (minus1fert[0])*(LogProb)(m-2*(msc.fert(0)-1));
+    }
+  else 
+    if( minus1fert[0]!=0.0 )
+      cerr << "ERROR: M1Fb: " << minus1fert[0] << endl;
+  if(int(m)-2*(int(msc.fert(0))+1)>=0)
+    {
+      p1count += (plus1fert[0])*(LogProb)(msc.fert(0)+1);
+      p0count += (plus1fert[0])*(LogProb)(m-2*(msc.fert(0)+1));
+    }
+  msc.check();
+  return nAl;
+};
+
+template<class TRANSPAIR>
+double collectCountsOverNeighborhoodForSophisticatedModels(const MoveSwapMatrix<TRANSPAIR>&,LogProb,void*)
+{
+  return 0.0;
+}
+
+template<class TRANSPAIR>
+void _collectCountsOverNeighborhoodForSophisticatedModels(const MoveSwapMatrix<TRANSPAIR>&Mmsc,const alignment&msc,const TRANSPAIR&ef,LogProb normalized_ascore,d4model*d4Table)
+{
+  Mmsc.check();
+  const PositionIndex m=msc.get_m(),l=msc.get_l();
+  for(PositionIndex j=1;j<=m;++j)
+    if( msc(j)!=0 )
+      if( msc.get_head(msc(j))==j)
+	{
+	  int ep=msc.prev_cept(msc(j));
+	  //massert( &d4Table->getCountRef_first(j,msc.get_center(ep),d4Table->ewordclasses.getClass(ef.get_es(ep)),d4Table->fwordclasses.getClass(ef.get_fs(j)),l,m) ==  ef.getCountFirst(ep,j,msc.get_center(ep)));
+	  d4Table->getCountRef_first(j,msc.get_center(ep),d4Table->ewordclasses.getClass(ef.get_es(ep)),d4Table->fwordclasses.getClass(ef.get_fs(j)),l,m)+=normalized_ascore;
+	}
+      else
+	{
+	  //massert( &d4Table->getCountRef_bigger(j,msc.prev_in_cept(j),0,d4Table->fwordclasses.getClass(ef.get_fs(j)),l,m) == ef.getCountSecond(j,msc.prev_in_cept(j) ));
+	  d4Table->getCountRef_bigger(j,msc.prev_in_cept(j),0,d4Table->fwordclasses.getClass(ef.get_fs(j)),l,m)+=normalized_ascore;
+	}
+}
+
+template<class TRANSPAIR>
+void _collectCountsOverNeighborhoodForSophisticatedModels(const MoveSwapMatrix<TRANSPAIR>&Mmsc,const alignment&msc,const TRANSPAIR&ef,LogProb normalized_ascore,d5model*d5Table)
+{
+  Mmsc.check();
+  _collectCountsOverNeighborhoodForSophisticatedModels(Mmsc,msc,ef,normalized_ascore,&d5Table->d4m);
+  Mmsc.check();
+  const PositionIndex m=msc.get_m(),l=msc.get_l();
+  PositionIndex prev_cept=0;
+  PositionIndex vac_all=m;
+  Vector<char> vac(m+1,0);
+  for(PositionIndex i=1;i<=l;i++)
+    {
+      PositionIndex cur_j=msc.als_i[i]; 
+      PositionIndex prev_j=0;
+      PositionIndex k=0;
+      if(cur_j) { // process first word of cept
+	k++;
+	d5Table->getCountRef_first(vacancies(vac,cur_j),vacancies(vac,msc.get_center(prev_cept)),
+				   d5Table->fwordclasses.getClass(ef.get_fs(cur_j)),l,m,vac_all-msc.fert(i)+k)+=normalized_ascore;
+	vac_all--;
+	assert(vac[cur_j]==0);
+	vac[cur_j]=1;
+	Mmsc.check();
+	prev_j=cur_j;
+	cur_j=msc.als_j[cur_j].next;
+      }
+      while(cur_j) { // process following words of cept
+	k++;
+	int vprev=vacancies(vac,prev_j);
+	d5Table->getCountRef_bigger(vacancies(vac,cur_j),vprev,d5Table->fwordclasses.getClass(ef.get_fs(cur_j)),l,m,vac_all-vprev/*war weg*/-msc.fert(i)+k)+=normalized_ascore;
+	vac_all--;
+	vac[cur_j]=1;
+	Mmsc.check();
+	prev_j=cur_j;
+	cur_j=msc.als_j[cur_j].next;
+      }
+      assert(k==msc.fert(i));
+      if( k )
+	prev_cept=i;
+    }
+  assert(vac_all==msc.fert(0));
+}
+
+extern int NumberOfAlignmentsInSophisticatedCountCollection;
+
+template<class TRANSPAIR,class MODEL>
+double collectCountsOverNeighborhoodForSophisticatedModels(const MoveSwapMatrix<TRANSPAIR>&msc,LogProb normalized_ascore,MODEL*d5Table)
+{
+  const PositionIndex m=msc.get_m(),l=msc.get_l();
+  alignment x(msc);
+  double sum=0;
+  msc.check();
+  if( !msc.isCenterDeleted() )
+    {
+      _collectCountsOverNeighborhoodForSophisticatedModels<TRANSPAIR>(msc,x,msc.get_ef(),normalized_ascore,d5Table);
+      NumberOfAlignmentsInSophisticatedCountCollection++;
+      sum+=normalized_ascore;
+    }
+  msc.check();
+  for(WordIndex j=1;j<=m;j++)for(WordIndex i=0;i<=l;i++)
+    {
+      WordIndex old=x(j);
+      if( i!=old&& !msc.isDelMove(i,j) )
+	{
+	  msc.check();
+	  double c=msc.cmove(i,j)*normalized_ascore;
+	  if(c > COUNTINCREASE_CUTOFF_AL )
+	    {
+	      x.set(j,i);
+	      _collectCountsOverNeighborhoodForSophisticatedModels<TRANSPAIR>(msc,x,msc.get_ef(),c,d5Table);
+	      NumberOfAlignmentsInSophisticatedCountCollection++;
+	      x.set(j,old);
+	      sum+=c;
+	    }
+	  msc.check();
+	}
+    }      
+  for(PositionIndex j1=1;j1<=m;j1++)
+    for(PositionIndex j2=j1+1;j2<=m;j2++)
+      if( msc(j1)!=msc(j2) && !msc.isDelSwap(j1,j2) )
+	{
+	  double c=msc.cswap(j1,j2)*normalized_ascore;
+	  msc.check();
+	  if(c > COUNTINCREASE_CUTOFF_AL )
+	    {
+	      int old1=msc(j1),old2=msc(j2);
+	      x.set(j1,old2);
+	      x.set(j2,old1);
+	      _collectCountsOverNeighborhoodForSophisticatedModels<TRANSPAIR>(msc,x,msc.get_ef(),c,d5Table);
+	      NumberOfAlignmentsInSophisticatedCountCollection++;
+	      x.set(j1,old1);
+	      x.set(j2,old2);
+	      sum+=c;
+	    }
+	  msc.check();
+	}
+  msc.check();
+  return sum;
+}
+
+template<class TRANSPAIR,class MODEL>
+int collectCountsOverNeighborhood(const Vector<pair<MoveSwapMatrix<TRANSPAIR>*,LogProb> >&smsc,Vector<WordIndex>&es,Vector<WordIndex>&fs,tmodel<COUNT,PROB>&tTable,amodel<COUNT>&aCountTable,amodel<COUNT>&dCountTable,nmodel<COUNT>&nCountTable,double&p1count,double&p0count,LogProb&_total,float count,bool addCounts,MODEL*d4Table)
+{
+  int nAl=0;
+  const PositionIndex l=es.size()-1,m=fs.size()-1;
+  Array2<LogProb,Vector<LogProb> > dtcount(l+1,m+1),ncount(l+1,MAX_FERTILITY+1);
+  LogProb p0=0,p1=0,all_total=0;
+  for(unsigned int i=0;i<smsc.size();++i)
+    {
+      LogProb this_total=0;
+      nAl+=collectCountsOverNeighborhood(*smsc[i].first,smsc[i].second,dtcount,ncount,p1,p0,this_total);
+      all_total+=this_total;
+    }
+  _total=all_total;
+  all_total/=(double)count;
+  double sum2=0;
+  if( addCounts && d4Table )
+    {
+      for(unsigned int i=0;i<smsc.size();++i)
+	{
+	  //for(WordIndex j=1;j<=m;j++)for(WordIndex ii=0;ii<=l;ii++)
+	  //  (*smsc[i].first).cmove(ii,j);
+	  sum2+=collectCountsOverNeighborhoodForSophisticatedModels(*smsc[i].first,smsc[i].second/all_total,d4Table);    
+	}
+      if(!(fabs(count-sum2)<0.05))
+	cerr << "WARNING: DIFFERENT SUMS: (" << count << ") (" << sum2 << ")\n";
+    }
+ if( addCounts )
+    {
+      for(PositionIndex i=0;i<=l;i++) 
+	{
+	  for(PositionIndex j=1;j<=m;j++)
+	    {
+	      LogProb ijadd=dtcount(i,j)/all_total;
+	      if( ijadd>COUNTINCREASE_CUTOFF_AL )
+		{
+		  tTable.incCount(es[i],fs[j],ijadd);
+		  dCountTable.getRef(j,i,l,m)+=ijadd;
+		  aCountTable.getRef(i,j,l,m)+=ijadd;
+		}
+	    }
+	  if( i>0 )
+	    for(PositionIndex n=0;n<MAX_FERTILITY;n++)
+	      nCountTable.getRef(es[i],n)+=ncount(i,n)/all_total;
+	}
+      p0count+=p0/all_total;
+      p1count+=p1/all_total;
+    }
+ return nAl;
+}
+
+
+
+
+
+
+
+
+

tools/giza-pp/GIZA++-v2/collCounts.h

@@ -0,0 +1,80 @@
+/*
+
+Copyright (C) 1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef collCounts_h_defined
+#define collCounts_h_defined
+#include "alignment.h"
+#include "transpair_model3.h"
+#include <map>
+#include "MoveSwapMatrix.h"
+#include "D4Tables.h"
+#include "transpair_model4.h"
+
+class OneMoveSwap
+{
+ public:
+  short type;
+  short a,b;
+  OneMoveSwap(short _type,short _a,short _b)
+    : type(_type),a(_a),b(_b)
+    {}
+  OneMoveSwap()
+    : type(0){}
+};
+
+inline bool operator<(const OneMoveSwap&a,const OneMoveSwap&b)
+{
+  if(a.type<b.type)return 1;
+  else if(b.type<a.type)return 0;
+  else if(a.a<b.a)return 1;
+  else if(b.a<a.a)return 0;
+  else return a.b<b.b;
+}
+
+inline bool operator==(const OneMoveSwap&a,const OneMoveSwap&b)
+{
+  return a.type==b.type&&a.a==b.a&&a.b==b.b;
+}
+
+inline ostream&operator<<(ostream&out,const OneMoveSwap&o)
+{
+  return out << '(' << o.type << "," << o.a << "," << o.b << ")";
+}
+
+inline ostream &operator<<(ostream &out,const set<OneMoveSwap>&s)
+{
+  for(set<OneMoveSwap>::const_iterator i=s.begin();i!=s.end();++i)
+    cout << *i << ' ';
+  return out;
+}
+
+bool makeOneMoveSwap(const alignment&a,const alignment&b,set<OneMoveSwap>&oms);
+
+template<class TRANSPAIR,class MODEL>
+int collectCountsOverNeighborhood(const Vector<pair<MoveSwapMatrix<TRANSPAIR>*,LogProb> >&smsc,
+				   Vector<WordIndex>&es,
+				   Vector<WordIndex>&fs,tmodel<COUNT,PROB>&tTable,
+				   amodel<COUNT>&aCountTable,amodel<COUNT>&dCountTable,
+				   nmodel<COUNT>&nCountTable,double&p1count,double&p0count,
+				   LogProb&_total,float count,bool addCounts,MODEL*d4Table=0);
+
+#endif

tools/giza-pp/GIZA++-v2/defs.h

@@ -0,0 +1,78 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _defs_h
+#define _defs_h 1
+#include <string>
+#include <math.h>
+#include <limits.h>
+
+const int TRANSFER_SIMPLE=1;
+const int TRANSFER=0;
+
+const unsigned int MAX_SENTENCE_LENGTH_ALLOWED=101;
+const int TRAIN_BUFFER_SIZE= 50000;
+//#ifdef WORDINDEX_WITH_4_BYTE
+typedef unsigned int WordIndex;
+const unsigned int MAX_VOCAB_SIZE=UINT_MAX;
+typedef unsigned int PositionIndex;
+//#else
+//typedef unsigned short WordIndex;
+//const unsigned int MAX_VOCAB_SIZE=USHRT_MAX;
+//typedef unsigned short PositionIndex;
+//#endif
+extern WordIndex MAX_FERTILITY;
+
+const int MAX_W=457979;
+extern double LAMBDA; // Lambda that is used to scale cross_entropy factor
+
+typedef float PROB ;
+typedef float COUNT ;
+ 
+class LogProb {
+ private:
+  double x ;
+ public:
+  LogProb():x(0){}
+  LogProb(double y):x(y){}
+  LogProb(float y):x(y){}
+  LogProb(int y):x(y){}
+  LogProb(WordIndex y):x(y){}
+  operator double() const {return x;}
+  LogProb operator *= (double y) { x *= y ; return *this;}
+  LogProb operator *= (LogProb y) { x *= y.x ; return *this;}
+  LogProb operator /= (double y) { x /= y ; return *this;}
+  LogProb operator /= (LogProb y) { x /= y.x ; return *this;}
+  LogProb operator += (double y) { x += y ; return *this;}
+  LogProb operator += (LogProb y) { x += y.x ; return *this;}
+};
+
+const int PARLEV_ITER=1;
+const int PARLEV_OPTHEUR=2;
+const int PARLEV_OUTPUT=3;
+const int PARLEV_SMOOTH=4;
+const int PARLEV_EM=5;
+const int PARLEV_MODELS=6;
+const int PARLEV_SPECIAL=7;
+const int PARLEV_INPUT=8;
+
+#endif
+  

tools/giza-pp/GIZA++-v2/dependencies

@@ -0,0 +1,635 @@
+#Automatically generated dependecy list
+optimized/alignment.o: alignment.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h
+optimized/AlignTables.o: AlignTables.cpp AlignTables.h defs.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h transpair_model1.h NTables.h vocab.h \
+  ATables.h Array4.h TTables.h Globals.h alignment.h
+optimized/ATables.o: ATables.cpp ATables.h defs.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h Array4.h Globals.h Parameter.h Pointer.h
+optimized/collCounts.o: collCounts.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  collCounts.h MoveSwapMatrix.h D4Tables.h WordClasses.h \
+  transpair_model4.h D5Tables.h transpair_model5.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h Parameter.h Pointer.h
+optimized/Dictionary.o: Dictionary.cpp Dictionary.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h
+optimized/ForwardBackward.o: ForwardBackward.cpp ForwardBackward.h myassert.h \
+  Array.h Vector.h mystl.h mymath.h Array2.h Globals.h defs.h HMMTables.h \
+  FlexArray.h
+optimized/getSentence.o: getSentence.cpp getSentence.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h vocab.h Globals.h Parameter.h Pointer.h
+optimized/hmm.o: hmm.cpp hmm.h Vector.h mystl.h myassert.h mymath.h Array2.h \
+  TTables.h defs.h vocab.h Globals.h ATables.h Array4.h getSentence.h \
+  model2.h model1.h Perplexity.h Dictionary.h WordClasses.h HMMTables.h \
+  FlexArray.h Array.h ForwardBackward.h utility.h Parameter.h Pointer.h \
+  HMMTables.cpp
+optimized/HMMTables.o: HMMTables.cpp HMMTables.h FlexArray.h Array.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h Globals.h defs.h Parameter.h \
+  Pointer.h
+optimized/logprob.o: logprob.cpp logprob.h
+optimized/main.o: main.cpp getSentence.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h defs.h vocab.h Globals.h TTables.h model1.h Perplexity.h \
+  Dictionary.h model2.h ATables.h Array4.h model3.h MoveSwapMatrix.h \
+  alignment.h transpair_model3.h NTables.h transpair_model2.h \
+  transpair_model1.h transpair_modelhmm.h ForwardBackward.h Array.h hmm.h \
+  WordClasses.h HMMTables.h FlexArray.h D4Tables.h AlignTables.h \
+  file_spec.h utility.h Parameter.h Pointer.h D5Tables.h \
+  transpair_model4.h transpair_model5.h
+optimized/model1.o: model1.cpp model1.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h vocab.h defs.h TTables.h Globals.h getSentence.h Perplexity.h \
+  Dictionary.h utility.h Parameter.h Pointer.h
+optimized/model2.o: model2.cpp model2.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h TTables.h defs.h vocab.h Globals.h ATables.h Array4.h \
+  getSentence.h model1.h Perplexity.h Dictionary.h utility.h Parameter.h \
+  Pointer.h
+optimized/model2to3.o: model2to3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h utility.h
+optimized/model345-peg.o: model345-peg.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h collCounts.h transpair_model4.h
+optimized/model3.o: model3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h collCounts.h transpair_model4.h utility.h D5Tables.h \
+  transpair_model5.h Parameter.h Pointer.h
+optimized/model3_viterbi.o: model3_viterbi.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h utility.h
+optimized/model3_viterbi_with_tricks.o: model3_viterbi_with_tricks.cpp mystl.h \
+  myassert.h mymath.h Array2.h model3.h Vector.h MoveSwapMatrix.h \
+  alignment.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  getSentence.h model2.h model1.h Perplexity.h Dictionary.h \
+  transpair_modelhmm.h ForwardBackward.h Array.h hmm.h WordClasses.h \
+  HMMTables.h FlexArray.h D4Tables.h AlignTables.h collCounts.h \
+  transpair_model4.h utility.h D5Tables.h transpair_model5.h Parameter.h \
+  Pointer.h collCounts.cpp
+optimized/MoveSwapMatrix.o: MoveSwapMatrix.cpp MoveSwapMatrix.h alignment.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h transpair_model4.h D4Tables.h \
+  WordClasses.h transpair_model5.h D5Tables.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h
+optimized/myassert.o: myassert.cpp mystl.h myassert.h mymath.h Array2.h
+optimized/NTables.o: NTables.cpp NTables.h Array2.h mystl.h myassert.h mymath.h \
+  Vector.h defs.h vocab.h Parameter.h Pointer.h Globals.h
+optimized/Parameter.o: Parameter.cpp Parameter.h mystl.h myassert.h mymath.h \
+  Array2.h Pointer.h Globals.h defs.h Vector.h
+optimized/parse.o: parse.cpp defs.h utility.h Perplexity.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h TTables.h vocab.h getSentence.h \
+  D4Tables.h WordClasses.h D5Tables.h ATables.h Array4.h Parameter.h \
+  Pointer.h
+optimized/Perplexity.o: Perplexity.cpp Perplexity.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h Globals.h
+optimized/plain2snt.o: plain2snt.cpp
+optimized/reports.o: reports.cpp defs.h vocab.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h Perplexity.h Globals.h getSentence.h TTables.h Parameter.h \
+  Pointer.h
+optimized/snt2cooc.o: snt2cooc.cpp
+optimized/snt2plain.o: snt2plain.cpp
+optimized/transpair_model3.o: transpair_model3.cpp transpair_model3.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h transpair_model2.h \
+  transpair_model1.h
+optimized/transpair_model4.o: transpair_model4.cpp transpair_model4.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D4Tables.h WordClasses.h \
+  transpair_model3.h transpair_model2.h transpair_model1.h Parameter.h \
+  Pointer.h
+optimized/transpair_model5.o: transpair_model5.cpp transpair_model5.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D5Tables.h D4Tables.h \
+  WordClasses.h transpair_model4.h transpair_model3.h transpair_model2.h \
+  transpair_model1.h Parameter.h Pointer.h
+optimized/TTables.o: TTables.cpp TTables.h defs.h vocab.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h Parameter.h Pointer.h
+optimized/utility.o: utility.cpp mymath.h
+optimized/vocab.o: vocab.cpp vocab.h defs.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h
+#Automatically generated dependecy list
+debug/alignment.o: alignment.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h
+debug/AlignTables.o: AlignTables.cpp AlignTables.h defs.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h transpair_model1.h NTables.h vocab.h \
+  ATables.h Array4.h TTables.h Globals.h alignment.h
+debug/ATables.o: ATables.cpp ATables.h defs.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h Array4.h Globals.h Parameter.h Pointer.h
+debug/collCounts.o: collCounts.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  collCounts.h MoveSwapMatrix.h D4Tables.h WordClasses.h \
+  transpair_model4.h D5Tables.h transpair_model5.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h Parameter.h Pointer.h
+debug/Dictionary.o: Dictionary.cpp Dictionary.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h
+debug/ForwardBackward.o: ForwardBackward.cpp ForwardBackward.h myassert.h \
+  Array.h Vector.h mystl.h mymath.h Array2.h Globals.h defs.h HMMTables.h \
+  FlexArray.h
+debug/getSentence.o: getSentence.cpp getSentence.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h vocab.h Globals.h Parameter.h Pointer.h
+debug/hmm.o: hmm.cpp hmm.h Vector.h mystl.h myassert.h mymath.h Array2.h \
+  TTables.h defs.h vocab.h Globals.h ATables.h Array4.h getSentence.h \
+  model2.h model1.h Perplexity.h Dictionary.h WordClasses.h HMMTables.h \
+  FlexArray.h Array.h ForwardBackward.h utility.h Parameter.h Pointer.h \
+  HMMTables.cpp
+debug/HMMTables.o: HMMTables.cpp HMMTables.h FlexArray.h Array.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h Globals.h defs.h Parameter.h \
+  Pointer.h
+debug/logprob.o: logprob.cpp logprob.h
+debug/main.o: main.cpp getSentence.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h defs.h vocab.h Globals.h TTables.h model1.h Perplexity.h \
+  Dictionary.h model2.h ATables.h Array4.h model3.h MoveSwapMatrix.h \
+  alignment.h transpair_model3.h NTables.h transpair_model2.h \
+  transpair_model1.h transpair_modelhmm.h ForwardBackward.h Array.h hmm.h \
+  WordClasses.h HMMTables.h FlexArray.h D4Tables.h AlignTables.h \
+  file_spec.h utility.h Parameter.h Pointer.h D5Tables.h \
+  transpair_model4.h transpair_model5.h
+debug/model1.o: model1.cpp model1.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h vocab.h defs.h TTables.h Globals.h getSentence.h Perplexity.h \
+  Dictionary.h utility.h Parameter.h Pointer.h
+debug/model2.o: model2.cpp model2.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h TTables.h defs.h vocab.h Globals.h ATables.h Array4.h \
+  getSentence.h model1.h Perplexity.h Dictionary.h utility.h Parameter.h \
+  Pointer.h
+debug/model2to3.o: model2to3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h utility.h
+debug/model345-peg.o: model345-peg.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h collCounts.h transpair_model4.h
+debug/model3.o: model3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h collCounts.h transpair_model4.h utility.h D5Tables.h \
+  transpair_model5.h Parameter.h Pointer.h
+debug/model3_viterbi.o: model3_viterbi.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h utility.h
+debug/model3_viterbi_with_tricks.o: model3_viterbi_with_tricks.cpp mystl.h \
+  myassert.h mymath.h Array2.h model3.h Vector.h MoveSwapMatrix.h \
+  alignment.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  getSentence.h model2.h model1.h Perplexity.h Dictionary.h \
+  transpair_modelhmm.h ForwardBackward.h Array.h hmm.h WordClasses.h \
+  HMMTables.h FlexArray.h D4Tables.h AlignTables.h collCounts.h \
+  transpair_model4.h utility.h D5Tables.h transpair_model5.h Parameter.h \
+  Pointer.h collCounts.cpp
+debug/MoveSwapMatrix.o: MoveSwapMatrix.cpp MoveSwapMatrix.h alignment.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h transpair_model4.h D4Tables.h \
+  WordClasses.h transpair_model5.h D5Tables.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h
+debug/myassert.o: myassert.cpp mystl.h myassert.h mymath.h Array2.h
+debug/NTables.o: NTables.cpp NTables.h Array2.h mystl.h myassert.h mymath.h \
+  Vector.h defs.h vocab.h Parameter.h Pointer.h Globals.h
+debug/Parameter.o: Parameter.cpp Parameter.h mystl.h myassert.h mymath.h \
+  Array2.h Pointer.h Globals.h defs.h Vector.h
+debug/parse.o: parse.cpp defs.h utility.h Perplexity.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h TTables.h vocab.h getSentence.h \
+  D4Tables.h WordClasses.h D5Tables.h ATables.h Array4.h Parameter.h \
+  Pointer.h
+debug/Perplexity.o: Perplexity.cpp Perplexity.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h Globals.h
+debug/plain2snt.o: plain2snt.cpp
+debug/reports.o: reports.cpp defs.h vocab.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h Perplexity.h Globals.h getSentence.h TTables.h Parameter.h \
+  Pointer.h
+debug/snt2cooc.o: snt2cooc.cpp
+debug/snt2plain.o: snt2plain.cpp
+debug/transpair_model3.o: transpair_model3.cpp transpair_model3.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h transpair_model2.h \
+  transpair_model1.h
+debug/transpair_model4.o: transpair_model4.cpp transpair_model4.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D4Tables.h WordClasses.h \
+  transpair_model3.h transpair_model2.h transpair_model1.h Parameter.h \
+  Pointer.h
+debug/transpair_model5.o: transpair_model5.cpp transpair_model5.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D5Tables.h D4Tables.h \
+  WordClasses.h transpair_model4.h transpair_model3.h transpair_model2.h \
+  transpair_model1.h Parameter.h Pointer.h
+debug/TTables.o: TTables.cpp TTables.h defs.h vocab.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h Parameter.h Pointer.h
+debug/utility.o: utility.cpp mymath.h
+debug/vocab.o: vocab.cpp vocab.h defs.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h
+#Automatically generated dependecy list
+vdebug/alignment.o: alignment.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h
+vdebug/AlignTables.o: AlignTables.cpp AlignTables.h defs.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h transpair_model1.h NTables.h vocab.h \
+  ATables.h Array4.h TTables.h Globals.h alignment.h
+vdebug/ATables.o: ATables.cpp ATables.h defs.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h Array4.h Globals.h Parameter.h Pointer.h
+vdebug/collCounts.o: collCounts.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  collCounts.h MoveSwapMatrix.h D4Tables.h WordClasses.h \
+  transpair_model4.h D5Tables.h transpair_model5.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h Parameter.h Pointer.h
+vdebug/Dictionary.o: Dictionary.cpp Dictionary.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h
+vdebug/ForwardBackward.o: ForwardBackward.cpp ForwardBackward.h myassert.h \
+  Array.h Vector.h mystl.h mymath.h Array2.h Globals.h defs.h HMMTables.h \
+  FlexArray.h
+vdebug/getSentence.o: getSentence.cpp getSentence.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h vocab.h Globals.h Parameter.h Pointer.h
+vdebug/hmm.o: hmm.cpp hmm.h Vector.h mystl.h myassert.h mymath.h Array2.h \
+  TTables.h defs.h vocab.h Globals.h ATables.h Array4.h getSentence.h \
+  model2.h model1.h Perplexity.h Dictionary.h WordClasses.h HMMTables.h \
+  FlexArray.h Array.h ForwardBackward.h utility.h Parameter.h Pointer.h \
+  HMMTables.cpp
+vdebug/HMMTables.o: HMMTables.cpp HMMTables.h FlexArray.h Array.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h Globals.h defs.h Parameter.h \
+  Pointer.h
+vdebug/logprob.o: logprob.cpp logprob.h
+vdebug/main.o: main.cpp getSentence.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h defs.h vocab.h Globals.h TTables.h model1.h Perplexity.h \
+  Dictionary.h model2.h ATables.h Array4.h model3.h MoveSwapMatrix.h \
+  alignment.h transpair_model3.h NTables.h transpair_model2.h \
+  transpair_model1.h transpair_modelhmm.h ForwardBackward.h Array.h hmm.h \
+  WordClasses.h HMMTables.h FlexArray.h D4Tables.h AlignTables.h \
+  file_spec.h utility.h Parameter.h Pointer.h D5Tables.h \
+  transpair_model4.h transpair_model5.h
+vdebug/model1.o: model1.cpp model1.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h vocab.h defs.h TTables.h Globals.h getSentence.h Perplexity.h \
+  Dictionary.h utility.h Parameter.h Pointer.h
+vdebug/model2.o: model2.cpp model2.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h TTables.h defs.h vocab.h Globals.h ATables.h Array4.h \
+  getSentence.h model1.h Perplexity.h Dictionary.h utility.h Parameter.h \
+  Pointer.h
+vdebug/model2to3.o: model2to3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h utility.h
+vdebug/model345-peg.o: model345-peg.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h collCounts.h transpair_model4.h
+vdebug/model3.o: model3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h collCounts.h transpair_model4.h utility.h D5Tables.h \
+  transpair_model5.h Parameter.h Pointer.h
+vdebug/model3_viterbi.o: model3_viterbi.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h utility.h
+vdebug/model3_viterbi_with_tricks.o: model3_viterbi_with_tricks.cpp mystl.h \
+  myassert.h mymath.h Array2.h model3.h Vector.h MoveSwapMatrix.h \
+  alignment.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  getSentence.h model2.h model1.h Perplexity.h Dictionary.h \
+  transpair_modelhmm.h ForwardBackward.h Array.h hmm.h WordClasses.h \
+  HMMTables.h FlexArray.h D4Tables.h AlignTables.h collCounts.h \
+  transpair_model4.h utility.h D5Tables.h transpair_model5.h Parameter.h \
+  Pointer.h collCounts.cpp
+vdebug/MoveSwapMatrix.o: MoveSwapMatrix.cpp MoveSwapMatrix.h alignment.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h transpair_model4.h D4Tables.h \
+  WordClasses.h transpair_model5.h D5Tables.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h
+vdebug/myassert.o: myassert.cpp mystl.h myassert.h mymath.h Array2.h
+vdebug/NTables.o: NTables.cpp NTables.h Array2.h mystl.h myassert.h mymath.h \
+  Vector.h defs.h vocab.h Parameter.h Pointer.h Globals.h
+vdebug/Parameter.o: Parameter.cpp Parameter.h mystl.h myassert.h mymath.h \
+  Array2.h Pointer.h Globals.h defs.h Vector.h
+vdebug/parse.o: parse.cpp defs.h utility.h Perplexity.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h TTables.h vocab.h getSentence.h \
+  D4Tables.h WordClasses.h D5Tables.h ATables.h Array4.h Parameter.h \
+  Pointer.h
+vdebug/Perplexity.o: Perplexity.cpp Perplexity.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h Globals.h
+vdebug/plain2snt.o: plain2snt.cpp
+vdebug/reports.o: reports.cpp defs.h vocab.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h Perplexity.h Globals.h getSentence.h TTables.h Parameter.h \
+  Pointer.h
+vdebug/snt2cooc.o: snt2cooc.cpp
+vdebug/snt2plain.o: snt2plain.cpp
+vdebug/transpair_model3.o: transpair_model3.cpp transpair_model3.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h transpair_model2.h \
+  transpair_model1.h
+vdebug/transpair_model4.o: transpair_model4.cpp transpair_model4.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D4Tables.h WordClasses.h \
+  transpair_model3.h transpair_model2.h transpair_model1.h Parameter.h \
+  Pointer.h
+vdebug/transpair_model5.o: transpair_model5.cpp transpair_model5.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D5Tables.h D4Tables.h \
+  WordClasses.h transpair_model4.h transpair_model3.h transpair_model2.h \
+  transpair_model1.h Parameter.h Pointer.h
+vdebug/TTables.o: TTables.cpp TTables.h defs.h vocab.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h Parameter.h Pointer.h
+vdebug/utility.o: utility.cpp mymath.h
+vdebug/vocab.o: vocab.cpp vocab.h defs.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h
+#Automatically generated dependecy list
+norm/alignment.o: alignment.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h
+norm/AlignTables.o: AlignTables.cpp AlignTables.h defs.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h transpair_model1.h NTables.h vocab.h \
+  ATables.h Array4.h TTables.h Globals.h alignment.h
+norm/ATables.o: ATables.cpp ATables.h defs.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h Array4.h Globals.h Parameter.h Pointer.h
+norm/collCounts.o: collCounts.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  collCounts.h MoveSwapMatrix.h D4Tables.h WordClasses.h \
+  transpair_model4.h D5Tables.h transpair_model5.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h Parameter.h Pointer.h
+norm/Dictionary.o: Dictionary.cpp Dictionary.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h
+norm/ForwardBackward.o: ForwardBackward.cpp ForwardBackward.h myassert.h \
+  Array.h Vector.h mystl.h mymath.h Array2.h Globals.h defs.h HMMTables.h \
+  FlexArray.h
+norm/getSentence.o: getSentence.cpp getSentence.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h vocab.h Globals.h Parameter.h Pointer.h
+norm/hmm.o: hmm.cpp hmm.h Vector.h mystl.h myassert.h mymath.h Array2.h \
+  TTables.h defs.h vocab.h Globals.h ATables.h Array4.h getSentence.h \
+  model2.h model1.h Perplexity.h Dictionary.h WordClasses.h HMMTables.h \
+  FlexArray.h Array.h ForwardBackward.h utility.h Parameter.h Pointer.h \
+  HMMTables.cpp
+norm/HMMTables.o: HMMTables.cpp HMMTables.h FlexArray.h Array.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h Globals.h defs.h Parameter.h \
+  Pointer.h
+norm/logprob.o: logprob.cpp logprob.h
+norm/main.o: main.cpp getSentence.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h defs.h vocab.h Globals.h TTables.h model1.h Perplexity.h \
+  Dictionary.h model2.h ATables.h Array4.h model3.h MoveSwapMatrix.h \
+  alignment.h transpair_model3.h NTables.h transpair_model2.h \
+  transpair_model1.h transpair_modelhmm.h ForwardBackward.h Array.h hmm.h \
+  WordClasses.h HMMTables.h FlexArray.h D4Tables.h AlignTables.h \
+  file_spec.h utility.h Parameter.h Pointer.h D5Tables.h \
+  transpair_model4.h transpair_model5.h
+norm/model1.o: model1.cpp model1.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h vocab.h defs.h TTables.h Globals.h getSentence.h Perplexity.h \
+  Dictionary.h utility.h Parameter.h Pointer.h
+norm/model2.o: model2.cpp model2.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h TTables.h defs.h vocab.h Globals.h ATables.h Array4.h \
+  getSentence.h model1.h Perplexity.h Dictionary.h utility.h Parameter.h \
+  Pointer.h
+norm/model2to3.o: model2to3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h utility.h
+norm/model345-peg.o: model345-peg.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h collCounts.h transpair_model4.h
+norm/model3.o: model3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h collCounts.h transpair_model4.h utility.h D5Tables.h \
+  transpair_model5.h Parameter.h Pointer.h
+norm/model3_viterbi.o: model3_viterbi.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h utility.h
+norm/model3_viterbi_with_tricks.o: model3_viterbi_with_tricks.cpp mystl.h \
+  myassert.h mymath.h Array2.h model3.h Vector.h MoveSwapMatrix.h \
+  alignment.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  getSentence.h model2.h model1.h Perplexity.h Dictionary.h \
+  transpair_modelhmm.h ForwardBackward.h Array.h hmm.h WordClasses.h \
+  HMMTables.h FlexArray.h D4Tables.h AlignTables.h collCounts.h \
+  transpair_model4.h utility.h D5Tables.h transpair_model5.h Parameter.h \
+  Pointer.h collCounts.cpp
+norm/MoveSwapMatrix.o: MoveSwapMatrix.cpp MoveSwapMatrix.h alignment.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h transpair_model4.h D4Tables.h \
+  WordClasses.h transpair_model5.h D5Tables.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h
+norm/myassert.o: myassert.cpp mystl.h myassert.h mymath.h Array2.h
+norm/NTables.o: NTables.cpp NTables.h Array2.h mystl.h myassert.h mymath.h \
+  Vector.h defs.h vocab.h Parameter.h Pointer.h Globals.h
+norm/Parameter.o: Parameter.cpp Parameter.h mystl.h myassert.h mymath.h \
+  Array2.h Pointer.h Globals.h defs.h Vector.h
+norm/parse.o: parse.cpp defs.h utility.h Perplexity.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h TTables.h vocab.h getSentence.h \
+  D4Tables.h WordClasses.h D5Tables.h ATables.h Array4.h Parameter.h \
+  Pointer.h
+norm/Perplexity.o: Perplexity.cpp Perplexity.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h Globals.h
+norm/plain2snt.o: plain2snt.cpp
+norm/reports.o: reports.cpp defs.h vocab.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h Perplexity.h Globals.h getSentence.h TTables.h Parameter.h \
+  Pointer.h
+norm/snt2cooc.o: snt2cooc.cpp
+norm/snt2plain.o: snt2plain.cpp
+norm/transpair_model3.o: transpair_model3.cpp transpair_model3.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h transpair_model2.h \
+  transpair_model1.h
+norm/transpair_model4.o: transpair_model4.cpp transpair_model4.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D4Tables.h WordClasses.h \
+  transpair_model3.h transpair_model2.h transpair_model1.h Parameter.h \
+  Pointer.h
+norm/transpair_model5.o: transpair_model5.cpp transpair_model5.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D5Tables.h D4Tables.h \
+  WordClasses.h transpair_model4.h transpair_model3.h transpair_model2.h \
+  transpair_model1.h Parameter.h Pointer.h
+norm/TTables.o: TTables.cpp TTables.h defs.h vocab.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h Parameter.h Pointer.h
+norm/utility.o: utility.cpp mymath.h
+norm/vocab.o: vocab.cpp vocab.h defs.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h
+#Automatically generated dependecy list
+profile/alignment.o: alignment.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h
+profile/AlignTables.o: AlignTables.cpp AlignTables.h defs.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h transpair_model1.h NTables.h vocab.h \
+  ATables.h Array4.h TTables.h Globals.h alignment.h
+profile/ATables.o: ATables.cpp ATables.h defs.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h Array4.h Globals.h Parameter.h Pointer.h
+profile/collCounts.o: collCounts.cpp alignment.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  collCounts.h MoveSwapMatrix.h D4Tables.h WordClasses.h \
+  transpair_model4.h D5Tables.h transpair_model5.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h Parameter.h Pointer.h
+profile/Dictionary.o: Dictionary.cpp Dictionary.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h
+profile/ForwardBackward.o: ForwardBackward.cpp ForwardBackward.h myassert.h \
+  Array.h Vector.h mystl.h mymath.h Array2.h Globals.h defs.h HMMTables.h \
+  FlexArray.h
+profile/getSentence.o: getSentence.cpp getSentence.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h vocab.h Globals.h Parameter.h Pointer.h
+profile/hmm.o: hmm.cpp hmm.h Vector.h mystl.h myassert.h mymath.h Array2.h \
+  TTables.h defs.h vocab.h Globals.h ATables.h Array4.h getSentence.h \
+  model2.h model1.h Perplexity.h Dictionary.h WordClasses.h HMMTables.h \
+  FlexArray.h Array.h ForwardBackward.h utility.h Parameter.h Pointer.h \
+  HMMTables.cpp
+profile/HMMTables.o: HMMTables.cpp HMMTables.h FlexArray.h Array.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h Globals.h defs.h Parameter.h \
+  Pointer.h
+profile/logprob.o: logprob.cpp logprob.h
+profile/main.o: main.cpp getSentence.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h defs.h vocab.h Globals.h TTables.h model1.h Perplexity.h \
+  Dictionary.h model2.h ATables.h Array4.h model3.h MoveSwapMatrix.h \
+  alignment.h transpair_model3.h NTables.h transpair_model2.h \
+  transpair_model1.h transpair_modelhmm.h ForwardBackward.h Array.h hmm.h \
+  WordClasses.h HMMTables.h FlexArray.h D4Tables.h AlignTables.h \
+  file_spec.h utility.h Parameter.h Pointer.h D5Tables.h \
+  transpair_model4.h transpair_model5.h
+profile/model1.o: model1.cpp model1.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h vocab.h defs.h TTables.h Globals.h getSentence.h Perplexity.h \
+  Dictionary.h utility.h Parameter.h Pointer.h
+profile/model2.o: model2.cpp model2.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h TTables.h defs.h vocab.h Globals.h ATables.h Array4.h \
+  getSentence.h model1.h Perplexity.h Dictionary.h utility.h Parameter.h \
+  Pointer.h
+profile/model2to3.o: model2to3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h utility.h
+profile/model345-peg.o: model345-peg.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h collCounts.h transpair_model4.h
+profile/model3.o: model3.cpp model3.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h MoveSwapMatrix.h alignment.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h transpair_modelhmm.h ForwardBackward.h \
+  Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h D4Tables.h \
+  AlignTables.h collCounts.h transpair_model4.h utility.h D5Tables.h \
+  transpair_model5.h Parameter.h Pointer.h
+profile/model3_viterbi.o: model3_viterbi.cpp model3.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h MoveSwapMatrix.h alignment.h defs.h \
+  transpair_model3.h NTables.h vocab.h ATables.h Array4.h TTables.h \
+  Globals.h transpair_model2.h transpair_model1.h getSentence.h model2.h \
+  model1.h Perplexity.h Dictionary.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h WordClasses.h HMMTables.h FlexArray.h \
+  D4Tables.h AlignTables.h utility.h
+profile/model3_viterbi_with_tricks.o: model3_viterbi_with_tricks.cpp mystl.h \
+  myassert.h mymath.h Array2.h model3.h Vector.h MoveSwapMatrix.h \
+  alignment.h defs.h transpair_model3.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h transpair_model2.h transpair_model1.h \
+  getSentence.h model2.h model1.h Perplexity.h Dictionary.h \
+  transpair_modelhmm.h ForwardBackward.h Array.h hmm.h WordClasses.h \
+  HMMTables.h FlexArray.h D4Tables.h AlignTables.h collCounts.h \
+  transpair_model4.h utility.h D5Tables.h transpair_model5.h Parameter.h \
+  Pointer.h collCounts.cpp
+profile/MoveSwapMatrix.o: MoveSwapMatrix.cpp MoveSwapMatrix.h alignment.h Vector.h \
+  mystl.h myassert.h mymath.h Array2.h defs.h transpair_model3.h \
+  NTables.h vocab.h ATables.h Array4.h TTables.h Globals.h \
+  transpair_model2.h transpair_model1.h transpair_model4.h D4Tables.h \
+  WordClasses.h transpair_model5.h D5Tables.h transpair_modelhmm.h \
+  ForwardBackward.h Array.h hmm.h getSentence.h model2.h model1.h \
+  Perplexity.h Dictionary.h HMMTables.h FlexArray.h
+profile/myassert.o: myassert.cpp mystl.h myassert.h mymath.h Array2.h
+profile/NTables.o: NTables.cpp NTables.h Array2.h mystl.h myassert.h mymath.h \
+  Vector.h defs.h vocab.h Parameter.h Pointer.h Globals.h
+profile/Parameter.o: Parameter.cpp Parameter.h mystl.h myassert.h mymath.h \
+  Array2.h Pointer.h Globals.h defs.h Vector.h
+profile/parse.o: parse.cpp defs.h utility.h Perplexity.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h TTables.h vocab.h getSentence.h \
+  D4Tables.h WordClasses.h D5Tables.h ATables.h Array4.h Parameter.h \
+  Pointer.h
+profile/Perplexity.o: Perplexity.cpp Perplexity.h Vector.h mystl.h myassert.h \
+  mymath.h Array2.h defs.h Globals.h
+profile/plain2snt.o: plain2snt.cpp
+profile/reports.o: reports.cpp defs.h vocab.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h Perplexity.h Globals.h getSentence.h TTables.h Parameter.h \
+  Pointer.h
+profile/snt2cooc.o: snt2cooc.cpp
+profile/snt2plain.o: snt2plain.cpp
+profile/transpair_model3.o: transpair_model3.cpp transpair_model3.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h transpair_model2.h \
+  transpair_model1.h
+profile/transpair_model4.o: transpair_model4.cpp transpair_model4.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D4Tables.h WordClasses.h \
+  transpair_model3.h transpair_model2.h transpair_model1.h Parameter.h \
+  Pointer.h
+profile/transpair_model5.o: transpair_model5.cpp transpair_model5.h Array2.h \
+  mystl.h myassert.h mymath.h defs.h Vector.h NTables.h vocab.h ATables.h \
+  Array4.h TTables.h Globals.h alignment.h D5Tables.h D4Tables.h \
+  WordClasses.h transpair_model4.h transpair_model3.h transpair_model2.h \
+  transpair_model1.h Parameter.h Pointer.h
+profile/TTables.o: TTables.cpp TTables.h defs.h vocab.h Vector.h mystl.h \
+  myassert.h mymath.h Array2.h Globals.h Parameter.h Pointer.h
+profile/utility.o: utility.cpp mymath.h
+profile/vocab.o: vocab.cpp vocab.h defs.h Vector.h mystl.h myassert.h mymath.h \
+  Array2.h

tools/giza-pp/GIZA++-v2/file_spec.h

@@ -0,0 +1,60 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef FILE_SPEC_H
+#define FILE_SPEC_H
+
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+/* This function returns a string, locally called file_spec. This
+   string is the concatenation of the date and time of execution
+   and the user who is performing the execution */
+/* Originally implemented in C by Yaser Al-Onaizan; 
+   editions for C++ and formatting by Noah A. Smith, 9 July 1999 */
+
+char *Get_File_Spec (){
+  struct tm *local;
+  time_t t;
+  const char *user;
+  char time_stmp[19];
+  char *file_spec = 0;
+  
+  t = time(NULL);
+  local = localtime(&t);
+  
+  sprintf(time_stmp, "%02d-%02d-%02d.%02d%02d%02d.", local->tm_year, 
+	  (local->tm_mon + 1), local->tm_mday, local->tm_hour, 
+	  local->tm_min, local->tm_sec);
+  user = getenv("USER");
+  if (!user) { user = "no_user"; }
+
+  file_spec = (char *)malloc(sizeof(char) * 
+			     (strlen(time_stmp) + strlen(user) + 1));
+  file_spec[0] = '\0';
+  strcat(file_spec, time_stmp) ;
+  strcat(file_spec, user);
+  return file_spec;
+}
+
+#endif

tools/giza-pp/GIZA++-v2/getSentence.cpp

@@ -0,0 +1,340 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* --------------------------------------------------------------------------*
+ *                                                                           *
+ * Module : getSentece                                                       *
+ *                                                                           *
+ * Method Definitions File: getSentence.cc                                   *
+ *                                                                           *
+ * Objective: Defines clases and methods for handling I/O for the parallel   *
+ *            corpus.                                                        *
+ *****************************************************************************/
+
+
+#include "getSentence.h"
+#include <iostream>
+#include <sstream>
+#include "Parameter.h"
+#include "errno.h"
+
+int PrintedTooLong=0;
+
+/* -------------- Method Defnitions for Class sentenceHandler ---------------*/
+
+GLOBAL_PARAMETER(double,ManlexMAX_MULTIPLICITY,"manlexMAX_MULTIPLICITY","",PARLEV_EM,20.0);
+GLOBAL_PARAMETER(double,Manlexfactor1,"manlexfactor1","",PARLEV_EM,0.0);
+GLOBAL_PARAMETER(double,Manlexfactor2,"manlexfactor2","",PARLEV_EM,0.0);
+
+sentenceHandler::sentenceHandler(const char*  filename, vcbList* elist, 
+				 vcbList*  flist) : realCount(0)
+  // This method is the constructor of the class, it also intitializes the 
+  // sentence pair sequential number (count) to zero.
+
+{
+  readflag = false ;
+  allInMemory = false ;
+  inputFilename = filename ;
+  inputFile = new ifstream(filename);
+  pair_no = 0 ;
+  if(!(*inputFile)){
+    cerr << "\nERROR:(a) Cannot open " << filename;
+    exit(1);
+  }
+  currentSentence = 0;
+  totalPairs1 = 0 ;
+  totalPairs2 =0;
+  pair_no = 0 ;
+  noSentInBuffer = 0 ;
+  Buffer.clear();
+  bool isNegative=0;
+  if (elist && flist){
+    cout << "Calculating vocabulary frequencies from corpus " << filename << '\n';
+    sentPair s ;
+    while (getNextSentence(s, elist, flist))
+      {
+	totalPairs1++;
+	totalPairs2+=s.realCount; 
+	// NOTE: this value might change during training 
+	// for words from the manual dictionary, yet this is ignored!
+
+	if( s.noOcc<0 )
+	  isNegative=1;
+      }
+  }
+  if( isNegative==1 )
+    {
+      cerr << "WARNING: corpus contains negative occurrency frequencies => these are interpreted as entries of a manual dictionary.\n";
+      realCount=new Vector<double>(totalPairs1,1.0);
+    }
+  else
+    realCount=0;
+}
+
+void sentenceHandler::rewind()
+{
+  currentSentence = 0;
+  readflag = false ;
+  if (!allInMemory || 
+      !(Buffer.size() >= 1 && Buffer[currentSentence].sentenceNo == 1)){
+    // check if the buffer doe not already has the first chunk of pairs 
+    if (Buffer.size() > 0)
+      cerr << ' ' <<  Buffer[currentSentence].sentenceNo << '\n';
+    //    totalPairs = 0 ;
+    pair_no = 0 ;
+    noSentInBuffer = 0 ;
+    Buffer.clear();
+  }
+  if (!allInMemory){
+    delete inputFile;
+    inputFile = new ifstream(inputFilename);
+    if(!(*inputFile)){
+      cerr << "\nERROR:(b) Cannot open " << inputFilename << " " << (int)errno;
+    }
+  }
+}
+
+  
+bool sentenceHandler::getNextSentence(sentPair& sent, vcbList* elist, vcbList* flist)
+{
+  sentPair s ;
+  if (readflag){
+    cerr << "Attempting to read from the end of corpus, rewinding\n";
+    rewind();
+    return(false);
+  } 
+  if (currentSentence >= noSentInBuffer){
+    if (allInMemory)
+      return(false);
+    /* no more sentences in buffer */
+    noSentInBuffer = 0 ;
+    currentSentence = 0 ;
+    Buffer.clear();
+    cout << "Reading more sentence pairs into memory ... \n";
+    while((noSentInBuffer < TRAIN_BUFFER_SIZE) && readNextSentence(s)){
+      if ((s.fSent.size()-1) > (MAX_FERTILITY-1) * (s.eSent.size()-1)){
+	cerr << "WARNING: The following sentence pair has source/target sentence length ration more than\n"<<
+	  "the maximum allowed limit for a source word fertility\n"<<
+	  " source length = " << s.eSent.size()-1 << " target length = " << s.fSent.size()-1 <<
+	  " ratio " << double(s.fSent.size()-1)/  (s.eSent.size()-1) << " ferility limit : " <<
+	  MAX_FERTILITY-1 << '\n';
+	cerr << "Shortening sentence \n";
+	cerr << s;
+	s.eSent.resize(min(s.eSent.size(),s.fSent.size()));
+	s.fSent.resize(min(s.eSent.size(),s.fSent.size()));
+      }
+      Buffer.push_back(s) ;
+      if (elist && flist){
+	if ((*elist).size() > 0)
+	  for (WordIndex i= 0 ; i < s.eSent.size() ; i++){
+	    if (s.eSent[i] >= (*elist).uniqTokens()){
+	      if( PrintedTooLong++<100)
+		cerr << "ERROR: source word " << s.eSent[i] << " is not in the vocabulary list \n";
+	      exit(-1);
+	    }
+	    (*elist).incFreq(s.eSent[i], s.realCount);
+	  }
+	if ((*flist).size() > 0)
+	  for (WordIndex j= 1 ; j < s.fSent.size() ; j++){
+	    if (s.fSent[j] >= (*flist).uniqTokens()){
+	      cerr << "ERROR: target word " << s.fSent[j] << " is not in the vocabulary list \n";
+	      exit(-1);
+	    }
+	    (*flist).incFreq(s.fSent[j], s.realCount);
+	  }
+      }
+      noSentInBuffer++;
+    }
+    if (inputFile->eof()){
+      allInMemory = (Buffer.size() >= 1 && 
+		     Buffer[currentSentence].sentenceNo == 1) ;
+      if (allInMemory)
+	cout << "Corpus fits in memory, corpus has: " << Buffer.size() <<
+	  " sentence pairs.\n";
+    }
+  }
+  if(noSentInBuffer <= 0 ){
+    //cerr << "# sent in buffer " << noSentInBuffer << '\n';
+    readflag = true ;
+    return(false);
+  }
+  sent = Buffer[currentSentence++] ;
+  if( sent.noOcc<0 && realCount )
+    {
+      if( Manlexfactor1 && sent.noOcc==-1.0 )
+	sent.realCount=Manlexfactor1;
+      else if( Manlexfactor2 && sent.noOcc==-2.0 )
+	sent.realCount=Manlexfactor2;
+      else
+	sent.realCount=(*realCount)[sent.getSentenceNo()-1];
+    }
+  return true ;
+}
+bool sentenceHandler::readNextSentence(sentPair& sent)
+  /* This method reads in a new pair of sentences, each pair is read from the 
+     corpus file as line triples. The first line the no of times this line 
+     pair occured in the corpus, the second line is the source sentence and 
+     the third is the target sentence. The sentences are represented by a space
+     separated positive integer token ids. */
+{
+
+  string line;
+  bool fail(false) ;
+  
+  sent.clear();
+  if (getline(*inputFile, line)){
+    istringstream buffer(line);
+    buffer >> sent.noOcc;
+    if( sent.noOcc<0 )
+      {
+	if( realCount )
+	  {
+	    if( Manlexfactor1 && sent.noOcc==-1.0 )
+	      sent.realCount=Manlexfactor1;
+	    else if( Manlexfactor2 && sent.noOcc==-2.0 )
+	      sent.realCount=Manlexfactor2;
+	    else
+	      {
+		sent.realCount=(*realCount)[pair_no];
+	      }
+	  }
+	else
+	  sent.realCount=1.0;
+      }
+    else
+      sent.realCount=sent.noOcc;
+  }
+  else {
+    fail = true ;;
+  }
+  if (getline(*inputFile, line)){
+    istringstream buffer(line);
+    WordIndex w;  // w is a local variabe for token id
+    sent.eSent.push_back(0); // each source word is assumed to have 0 == 
+    // a null word (id 0) at the begining of the sentence. 
+    while(buffer>>w){ // read source sentece , word by word .
+      if (sent.eSent.size() < MAX_SENTENCE_LENGTH)
+	sent.eSent.push_back(w);
+      else {
+	if( PrintedTooLong++<100)
+	  cerr << "{WARNING:(a)truncated sentence "<<pair_no<<"}";
+	//cerr << "ERROR: getSentence.cc:getNextSentence(): sentence exceeds preset length limit of : " << MAX_SENTENCE_LENGTH << '\n';
+	//cerr << "The following sentence will be truncated\n" << line;
+	break ;
+      }
+    }
+  }
+  else {
+    fail = true ;
+  }
+  if (getline(*inputFile, line)){
+    istringstream buffer(line);
+    WordIndex w;  // w is a local variabe for token id
+    sent.fSent.push_back(0); //0 is inserted for program uniformity
+    while(buffer>>w){ // read target sentece , word by word .
+      if (sent.fSent.size() < MAX_SENTENCE_LENGTH)
+	sent.fSent.push_back(w);
+      else {
+	if( PrintedTooLong++<100)
+	  cerr << "{WARNING:(b)truncated sentence "<<pair_no<<"}";
+	//cerr << "ERROR: getSentence.cc:getNextSentence(): sentence exceeds preset length limit of : " << MAX_SENTENCE_LENGTH << '\n';
+	//cerr << "The following sentence will be truncated\n" << line;
+	break ;
+      }
+    }
+  }
+  else {
+    fail = true ;
+  }
+  if (fail){
+    sent.eSent.clear();
+    sent.fSent.clear();
+    sent.sentenceNo = 0 ;
+    sent.noOcc = 0 ;
+    sent.realCount=0;
+    return(false);
+  }  
+  if( sent.eSent.size()==1||sent.fSent.size()==1 )
+    cerr << "ERROR: Forbidden zero sentence length " << sent.sentenceNo << endl;
+  sent.sentenceNo = ++pair_no;
+  if(pair_no % 100000 == 0) 
+    cout << "[sent:" << sent.sentenceNo  << "]"<< '\n';
+  return true;
+}
+
+double optimize_lambda(Vector<double>&vd)
+{
+  Vector<double> l;
+  for(double lambda=1.0;lambda<ManlexMAX_MULTIPLICITY;lambda+=0.33)
+    {
+      double prod=0.0;
+      for(unsigned int i=0;i<vd.size();++i)
+	{
+	  prod += vd[i]*exp(lambda*vd[i])/(exp(lambda*vd[i])-1.0);
+	}
+      l.push_back(fabs(prod-1.0));
+    }
+  double lam=double(min_element(l.begin(),l.end())-l.begin())*0.33+1.0;
+  if( lam<1.0 )
+    {
+      cerr << "ERROR: lambda is smaller than one: " << lam << endl;
+      for(unsigned int i=0;i<vd.size();++i)
+	cerr << vd[i] << ' ';
+      cerr << endl;
+    }
+  return lam;
+}
+
+void sentenceHandler::setProbOfSentence(const sentPair&s,double d)
+{
+  if( realCount==0 )
+    return;
+  else
+    {
+      if( s.noOcc<=0 )
+	{
+	  double ed=exp(d);
+	  if( oldPairs.size()>0&&(oldPairs.back().get_eSent()!=s.get_eSent()||oldPairs.back().getSentenceNo()>=s.getSentenceNo()) )
+	    {
+	      double lambda=optimize_lambda(oldProbs);
+	      for(unsigned int i=0;i<oldPairs.size();++i)
+		{
+		  if( oldProbs[i]<1e-5 )
+		    (*realCount)[oldPairs[i].getSentenceNo()-1]=1.0;
+		  else
+		    (*realCount)[oldPairs[i].getSentenceNo()-1]=lambda*oldProbs[i]/(1-exp(-lambda*oldProbs[i]));		    
+		}
+	      oldPairs.clear();
+	      oldProbs.clear();
+	    }
+	  oldPairs.push_back(s);
+	  oldProbs.push_back(ed);
+	}
+    }
+}
+
+/* ------------- End of Method Definition of Class sentenceHandler ----------*/
+
+
+
+
+
+

tools/giza-pp/GIZA++-v2/getSentence.h

@@ -0,0 +1,123 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+/* --------------------------------------------------------------------------*
+ *                                                                           *
+ * Module : getSentence                                                      *
+ *                                                                           *
+ * Prototypes File: getSentence.h                                            *
+ *                                                                           *
+ * Objective: Defines clases and methods for handling I/O for the parallel   *
+ *            corpus.                                                        *
+ *****************************************************************************/
+
+
+
+
+
+#ifndef _sentenceHandler_h
+#define _sentenceHandler_h 1
+
+
+#include <iostream>
+#include <fstream>
+#include <string>
+#include "Vector.h"
+#include "defs.h"
+#include "vocab.h"
+#include "Globals.h"
+/*----------------------- Class Prototype Definition ------------------------*
+ Class Name: sentenceHandleer 
+ Objective: This class is defined to handle training sentece pairs from the 
+ parallel corpus. Each pair has: a target sentece, called here French; a 
+ source sentece, called here English sentece; and an integer number denoting
+ the number of times this pair occured in trining corpus. Both source and 
+ target senteces are represented as integer vector (variable size arrays), 
+ each entry is a numeric value which is the token id for the particular token
+ in the sentece.
+
+ *---------------------------------------------------------------------------*/
+
+class sentPair{
+ public:
+  int sentenceNo ;
+  float noOcc;
+  float realCount;
+  Vector<WordIndex> eSent ;
+  Vector<WordIndex> fSent;
+
+ public:
+  sentPair(){};
+  void clear(){ eSent.clear(); fSent.clear(); noOcc=0; realCount=0; sentenceNo=0;};
+  const Vector<WordIndex>&get_eSent()const
+    { return eSent; }
+  const Vector<WordIndex>&get_fSent()const
+    { return fSent; }
+  int getSentenceNo()const
+    { return sentenceNo; }
+  double getCount()const
+    { return realCount; }
+};
+
+inline ostream&operator<<(ostream&of,const sentPair&s)
+{
+  of << "Sent No: " << s.sentenceNo << " , No. Occurrences: " << s.noOcc << '\n';
+  if( s.noOcc!=s.realCount )
+    of << " Used No. Occurrences: " << s.realCount << '\n';
+  unsigned int i;
+  for(i=0; i < s.eSent.size(); i++)
+    of << s.eSent[i] << ' ';
+  of <<  '\n';
+  for(i=1; i < s.fSent.size(); i++)
+    of << s.fSent[i] << ' ';
+  of << '\n';
+  return of;
+}
+
+class sentenceHandler{
+public:
+  const char * inputFilename;   // parallel corpus file name, similar for all 
+                            // sentence pair objects
+  ifstream *inputFile;     // parallel corpus file handler
+  Vector<sentPair> Buffer;
+  int noSentInBuffer ;
+  int currentSentence ;
+  int totalPairs1 ;
+  double totalPairs2;
+  bool readflag ; // true if you reach the end of file
+  bool allInMemory ;
+  int pair_no ;
+  Vector<double> *realCount;
+
+  Vector<sentPair> oldPairs;
+  Vector<double> oldProbs;
+  sentenceHandler(const char* filename, vcbList* elist=0, vcbList* flist=0);
+  void rewind();
+  bool getNextSentence(sentPair&, vcbList* = 0, vcbList* = 0);  // will be defined in the definition file, this
+  int getTotalNoPairs1()const {return totalPairs1;};
+  double getTotalNoPairs2()const {return totalPairs2;};
+  // method will read the next pair of sentence from memory buffer
+  bool readNextSentence(sentPair&);  // will be defined in the definition file, this
+  void setProbOfSentence(const sentPair&s,double d);
+};
+
+#endif
+  

tools/giza-pp/GIZA++-v2/hmm.cpp

@@ -0,0 +1,405 @@
+/*
+
+Copyright (C) 1998,1999,2000,2001  Franz Josef Och (RWTH Aachen - Lehrstuhl fuer Informatik VI)
+
+This file is part of GIZA++ ( extension of GIZA ).
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#include "hmm.h"
+#include "Globals.h"
+#include "utility.h"
+#include "HMMTables.h"
+#include "ForwardBackward.h"
+#include "Parameter.h"
+
+#define CLASSIFY(i,empty,ianf) bool empty=(i>=l); unsigned int ianf=(i%l);
+#define CLASSIFY2(i,ianf) unsigned int ianf=(i%l);
+
+
+short PredictionInAlignments=0;
+short UniformEntryExit=3;
+short HMMTrainingSpecialFlags=0;
+
+GLOBAL_PARAMETER2(int,ModelH_Dump_Freq,"HMM DUMP FREQUENCY","th","dump frequency of HMM",PARLEV_OUTPUT,0);
+
+GLOBAL_PARAMETER(short,CompareAlDeps,"emAlignmentDependencies",
+		 "lextrain: dependencies in the HMM alignment model. "
+		 " &1: sentence length; &2: previous class; &4: previous position; "
+		 " &8: French position; &16: French class"
+		 ,PARLEV_MODELS,2);
+GLOBAL_PARAMETER(double,GLOBALProbabilityForEmpty,"emProbForEmpty",
+		 "f-b-trn: probability for empty word",PARLEV_MODELS,0.4);
+GLOBAL_PARAMETER(short,SmoothHMM,"emSmoothHMM",
+		 "f-b-trn: smooth HMM model &1: modified counts; &2:perform smoothing with -emAlSmooth",PARLEV_SPECIAL,2);
+GLOBAL_PARAMETER(double,HMMAlignmentModelSmoothFactor,"emAlSmooth",
+		 "f-b-trn: smoothing factor for HMM alignment model (can be ignored by -emSmoothHMM)",PARLEV_SMOOTH,0.2);
+
+
+/*template<class T>
+void smooth_standard(T*a,T*b,double p)
+{
+  int n=b-a;
+  if( n==0 ) 
+    return;
+  double pp=p/n;
+  for(T*i=a;i!=b;++i)
+    *i = (1.0-p)*(*i)+pp;
+}*/
+
+
+hmm::hmm(model2& m)
+  : model2(m),counts(GLOBALProbabilityForEmpty,ewordclasses,fwordclasses),
+    probs(GLOBALProbabilityForEmpty,ewordclasses,fwordclasses)
+{  }
+
+void hmm::initialize_table_uniformly(sentenceHandler&){}
+
+int hmm::em_with_tricks(int noIterations)
+{
+  double minErrors=1.0;int minIter=0;
+  string modelName="Hmm",shortModelName="hmm";
+  int dumpFreq=ModelH_Dump_Freq;
+  time_t it_st, st, it_fn, fn;
+  string tfile, afile,afileh, number, alignfile, test_alignfile;
+  int pair_no = 0;
+  bool dump_files = false ;
+  ofstream of2 ;
+  st = time(NULL) ;
+  sHandler1.rewind();
+  cout << "\n==========================================================\n";
+  cout << modelName << " Training Started at: " << ctime(&st);
+  for(int it=1; it <= noIterations ; it++){
+    pair_no = 0;
+    it_st = time(NULL) ;
+    cout << endl << "-----------\n" << modelName << ": Iteration " << it << '\n';
+    dump_files = (dumpFreq != 0) && ((it % dumpFreq) == 0) && !NODUMPS;
+    number = "";
+    int n = it;
+    do{
+      number.insert((size_t)0, 1, (char)(n % 10 + '0'));
+    } while((n /= 10) > 0);
+    tfile = Prefix + ".t" + shortModelName + "." + number ;
+    afile = Prefix + ".a" + shortModelName + "." + number ;
+    afileh = Prefix + ".h" + shortModelName + "." + number ;
+    alignfile = Prefix + ".A" + shortModelName + "." + number ;
+    test_alignfile = Prefix + ".tst.A" + shortModelName + "." + number ;
+    counts=HMMTables<int,WordClasses>(GLOBALProbabilityForEmpty,ewordclasses,fwordclasses);
+    aCountTable.clear();
+    initAL();
+    em_loop(perp, sHandler1,  dump_files , alignfile.c_str(), trainViterbiPerp, false,it==1,it);
+    if( errorsAL()<minErrors )
+      {
+	minErrors=errorsAL();
+        minIter=it;
+      }
+    if (testPerp && testHandler)
+      em_loop(*testPerp, *testHandler, dump_files, test_alignfile.c_str(), *testViterbiPerp,  true,it==1,it); 
+    if (dump_files&&OutputInAachenFormat==1)
+      tTable.printCountTable(tfile.c_str(),Elist.getVocabList(),Flist.getVocabList(),1);
+    tTable.normalizeTable(Elist, Flist);
+    aCountTable.normalize(aTable);
+    probs=counts;
+    cout << modelName << ": ("<<it<<") TRAIN CROSS-ENTROPY " << perp.cross_entropy()
+	 << " PERPLEXITY " << perp.perplexity() << '\n';
+     if (testPerp && testHandler)
+       cout << modelName << ": ("<<it<<") TEST CROSS-ENTROPY " << (*testPerp).cross_entropy()
+	    << " PERPLEXITY " << (*testPerp).perplexity() 
+	    << '\n';
+     cout << modelName << ": ("<<it<<") VITERBI TRAIN CROSS-ENTROPY " << trainViterbiPerp.cross_entropy()
+	 << " PERPLEXITY " << trainViterbiPerp.perplexity() << '\n';
+    if (testPerp && testHandler)
+       cout << modelName << ": ("<<it<<") VITERBI TEST CROSS-ENTROPY " << testViterbiPerp->cross_entropy()
+	    << " PERPLEXITY " << testViterbiPerp->perplexity() 
+	    << '\n';
+    if (dump_files){
+      if( OutputInAachenFormat==0)
+	tTable.printProbTable(tfile.c_str(),Elist.getVocabList(),Flist.getVocabList(),OutputInAachenFormat);
+      ofstream afilestream(afileh.c_str());
+      probs.writeJumps(afilestream);
+      aCountTable.printTable(afile.c_str());
+    }
+    it_fn = time(NULL) ;
+    cout << "\n" << modelName << " Iteration: " << it<< " took: " <<
+      difftime(it_fn, it_st) << " seconds\n";
+  } // end of iterations 
+  fn = time(NULL) ;
+  cout << endl << "Entire " << modelName << " Training took: " << difftime(fn, st) << " seconds\n";
+  //cout << "tTable contains " << tTable.getHash().bucket_count() 
+  //     << " buckets and  " << tTable.getHash().size() << " entries." ;
+  cout << "==========================================================\n";
+  return minIter;
+}
+
+/*template<class T>
+T normalize_if_possible_with_increment(T*a,T*b,int increment)
+{
+  T sum=0;
+  for(T*i=a;i!=b;i+=increment)
+    sum+=*i;
+  if( sum )
+    for(T*i=a;i!=b;i+=increment)
+      *i/=sum;
+  else
+    {
+      T factor=increment/(b-a);
+      for(T*i=a;i!=b;i+=increment)
+	*i=factor;
+    }
+  return sum;
+}*/
+
+void hmm::load_table(const char* aname){
+  cout << "Hmm: loading a table not implemented.\n";
+  abort();
+  ifstream anamefile(aname);
+  probs.readJumps(anamefile);
+}
+
+HMMNetwork *hmm::makeHMMNetwork(const Vector<WordIndex>& es,const Vector<WordIndex>&fs,bool doInit)const
+{
+  unsigned int i,j;
+  unsigned int l = es.size() - 1;
+  unsigned int m = fs.size() - 1;
+  unsigned int I=2*l,J=m;
+  int IJ=I*J;
+  bool DependencyOfJ=(CompareAlDeps&(16|8))||(PredictionInAlignments==2);
+  bool DependencyOfPrevAJ=(CompareAlDeps&(2|4))||(PredictionInAlignments==0);
+  HMMNetwork *net = new HMMNetwork(I,J);
+  fill(net->alphainit.begin(),net->alphainit.end(),0.0);
+  fill(net->betainit.begin(),net->betainit.end(),0.0);
+  for(j=1;j<=m;j++)
+    {
+      for(i=1;i<=l;i++)
+	net->n(i-1,j-1)=tTable.getProb(es[i], fs[j]) ;
+      double emptyContribution=0;  
+      emptyContribution=tTable.getProb(es[0],fs[j]) ;
+      for(i=1;i<=l;i++)
+	net->n(i+l-1,j-1)=emptyContribution;
+      net->finalMultiply*=max(normalize_if_possible_with_increment(&net->n(0,j-1),&net->n(0,j-1)+IJ,J),double(1e-12));
+    }
+  if( DependencyOfJ )
+    net->e.resize(m-1);
+  else
+    net->e.resize(J>1);
+  for(j=0;j<net->e.size();j++)
+    {
+      int frenchClass=fwordclasses.getClass(fs[1+min(int(m)-1,int(j)+1)]);
+      net->e[j].resize(I,I,0);
+      for(unsigned int i1=0;i1<I;++i1) {
+	Array<double> al(l);
+	CLASSIFY2(i1,i1real);
+	for(unsigned int i2=0;i2<l;i2++)
+	  al[i2]=probs.getAlProb(i1real,i2,l,m,ewordclasses.getClass(es[1+i1real]),frenchClass
+				 ,j+1);
+	normalize_if_possible(conv<double>(al.begin()),conv<double>(al.end()));
+	if( SmoothHMM&2 )
+	  smooth_standard(conv<double>(al.begin()),conv<double>(al.end()),HMMAlignmentModelSmoothFactor);
+	for(unsigned int i2=0;i2<I;i2++) {
+	  CLASSIFY(i2,empty_i2,i2real);
+	  net->e[j](i1,i2)	    = al[i2real];
+	  
+	  if( empty_i2 )
+	    if(i1real!=i2real)
+	      {
+		net->e[j](i1,i2)=0;
+	      }
+	    else
+	      {
+		net->e[j](i1,i2)=doInit?al[0]:(probs.getProbabilityForEmpty()); // make first HMM iteration like IBM-1
+	      }
+	}
+	normalize_if_possible(&net->e[j](i1,0),&net->e[j](i1,0)+I);
+      }
+    }
+  if( doInit )
+    {
+      for(unsigned int i=0;i<I;++i)
+	{
+	  net->alphainit[i]=net->betainit[i]=(i<I/2)?1:(2.0/I);
+	  net->betainit[i]=1.0;
+	}
+    }
+  else
+    {
+      if( DependencyOfPrevAJ==0 )
+	{
+	  for(i=0;i<I;i++)
+	    {
+	      CLASSIFY2(i,ireal);
+	      net->alphainit[i]=probs.getAlProb(-1,ireal,l,m,0,fwordclasses.getClass(fs[1+0]),0);
+	    }
+	}
+      else
+	{
+	  if( UniformEntryExit&2 )probs.getBetaInit(I,net->betainit);
+	  if( UniformEntryExit&1 )probs.getAlphaInit(I,net->alphainit);
+	}
+    }
+  massert( net->alphainit.size()==I );massert( net->betainit.size()==I );
+  normalize_if_possible(conv<double>(net->alphainit.begin()),conv<double>(net->alphainit.end()));
+  normalize_if_possible(conv<double>(net->betainit.begin()),conv<double>(net->betainit.end()));
+  transform(net->betainit.begin(),net->betainit.end(),net->betainit.begin(),bind1st(multiplies<double>(),2*l));
+  return net;
+}
+extern float MINCOUNTINCREASE;
+
+void hmm::em_loop(Perplexity& perp, sentenceHandler& sHandler1, 
+		  bool dump_alignment, const char* alignfile, Perplexity& viterbi_perp, 
+		     bool test,bool doInit,int 
+)
+{
+  WordIndex i, j, l, m ;
+  double cross_entropy;
+  int pair_no=0 ;
+  perp.clear();
+  viterbi_perp.clear();
+  ofstream of2;
+  // for each sentence pair in the corpus
+  if (dump_alignment||FEWDUMPS )
+    of2.open(alignfile);
+  sentPair sent ;
+  sHandler1.rewind();
+  while(sHandler1.getNextSentence(sent)){
+    const Vector<WordIndex>& es = sent.get_eSent();
+    const Vector<WordIndex>& fs = sent.get_fSent();
+    const float so  = sent.getCount();
+    l = es.size() - 1;
+    m = fs.size() - 1;
+    cross_entropy = log(1.0);
+    Vector<WordIndex> viterbi_alignment(fs.size());
+
+    unsigned int I=2*l,J=m;
+    bool DependencyOfJ=(CompareAlDeps&(16|8))||(PredictionInAlignments==2);
+    bool DependencyOfPrevAJ=(CompareAlDeps&(2|4))||(PredictionInAlignments==0);
+    HMMNetwork *net=makeHMMNetwork(es,fs,doInit);
+    Array<double> gamma;
+    Array<Array2<double> > epsilon(DependencyOfJ?(m-1):1);
+    double trainProb;
+      trainProb=ForwardBackwardTraining(*net,gamma,epsilon);
+    if( !test )
+      {
+	double *gp=conv<double>(gamma.begin());
+	for(unsigned int i2=0;i2<J;i2++)for(unsigned int i1=0;i1<I;++i1,++gp)
+	  if( *gp>MINCOUNTINCREASE )
+	    {
+	      COUNT add= *gp*so;
+	      if( i1>=l )
+		{
+		  tTable.incCount(es[0],fs[1+i2],add);
+		  aCountTable.getRef(0,i2+1,l,m)+=add;
+		}
+	      else
+		{
+		  tTable.incCount(es[1+i1],fs[1+i2],add);
+		  aCountTable.getRef(1+i1,1+i2,l,m)+=add;
+		}
+	    }
+	double p0c=0.0,np0c=0.0;
+	for(unsigned int jj=0;jj<epsilon.size();jj++)
+	  {
+	    int frenchClass=fwordclasses.getClass(fs[1+min(int(m)-1,int(jj)+1)]);
+	    double *ep=epsilon[jj].begin();
+	    if( ep )
+	      {
+		//for(i=0;i<I;i++)
+		//  normalize_if_possible_with_increment(ep+i,ep+i+I*I,I);
+		//		for(i=0;i<I*I;++i)
+		//  ep[i] *= I;
+		//if( DependencyOfJ )
+		//  if( J-1 )
+		//    for(i=0;i<I*I;++i)
+		//      ep[i] /= (J-1);
+		double mult=1.0;
+		mult*=l;
+		//if( DependencyOfJ && J-1)
+		//  mult/=(J-1);
+		for(i=0;i<I;i++)
+		  {
+		    for(unsigned int i_bef=0;i_bef<I;i_bef++,ep++)
+		      {
+			CLASSIFY(i,i_empty,ireal);
+			CLASSIFY2(i_bef,i_befreal);
+			if( i_empty )
+			  p0c+=*ep * mult;
+			else
+			  {
+			    counts.addAlCount(i_befreal,ireal,l,m,ewordclasses.getClass(es[1+i_befreal]),
+					      frenchClass ,jj+1,*ep * mult,0.0);
+			    np0c+=*ep * mult; 
+			  }
+			massert( &epsilon[jj](i,i_bef)== ep);
+		      }
+		  }
+	      }
+	  }
+	double *gp1=conv<double>(gamma.begin()),*gp2=conv<double>(gamma.end())-I;
+	Array<double>&ai=counts.doGetAlphaInit(I);
+	Array<double>&bi=counts.doGetBetaInit(I);
+	int firstFrenchClass=(fs.size()>1)?(fwordclasses.getClass(fs[1+0])):0;
+	for(i=0;i<I;i++,gp1++,gp2++)
+	  {
+	    CLASSIFY(i,i_empty,ireal);
+	    ai[i]+= *gp1;
+	    bi[i]+= *gp2;
+	    if( DependencyOfPrevAJ==0 )
+	      {
+		if( i_empty )
+		  p0c+=*gp1;
+		else
+		  {
+		    counts.addAlCount(-1,ireal,l,m,0,firstFrenchClass,0,*gp1,0.0);
+		    np0c+=*gp1;
+		  }
+	      }
+	  }
+    if( Verbose )
+      cout << "l: " << l << "m: " << m << " p0c: " << p0c << " np0c: " << np0c << endl;
+      }
+    cross_entropy+=log(max(trainProb,1e-100))+log(max(net->finalMultiply,1e-100));
+    Array<int>vit;
+    double viterbi_score=1.0;
+    if( (HMMTrainingSpecialFlags&1) )
+      HMMViterbi(*net,gamma,vit);
+    else
+      viterbi_score=HMMRealViterbi(*net,vit);
+    for(j=1;j<=m;j++)
+      {
+	viterbi_alignment[j]=vit[j-1]+1;
+	if( viterbi_alignment[j]>l)
+	  viterbi_alignment[j]=0;
+      }
+    sHandler1.setProbOfSentence(sent,cross_entropy);
+    perp.addFactor(cross_entropy, so, l, m,1);
+    viterbi_perp.addFactor(log(viterbi_score)+log(max(net->finalMultiply,1e-100)), so, l, m,1);
+    if( Verbose )
+      cout << "Viterbi-perp: " << log(viterbi_score) << ' ' << log(max(net->finalMultiply,1e-100)) << ' ' << viterbi_score << ' ' << net->finalMultiply << ' ' << *net << "gamma: " << gamma << endl;
+    delete net;net=0;
+    if (dump_alignment||(FEWDUMPS&&sent.getSentenceNo()<1000) )
+      printAlignToFile(es, fs, Elist.getVocabList(), Flist.getVocabList(), of2, viterbi_alignment, sent.getSentenceNo(), viterbi_score);
+    addAL(viterbi_alignment,sent.getSentenceNo(),l);    
+    pair_no++;
+  } /* of while */
+  sHandler1.rewind();
+  perp.record("HMM");
+  viterbi_perp.record("HMM");
+  errorReportAL(cout,"HMM");
+}
+
+#include "HMMTables.cpp"
+template class HMMTables<int,WordClasses>;
+ 

tools/giza-pp/GIZA++-v2/hmm.h

@@ -0,0 +1,82 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+#ifndef _hmm_h
+#define _hmm_h 1
+
+#include <cassert>
+ 
+#include <iostream>
+#include <algorithm>
+#include <functional>
+#include <map>
+#include <set>
+#include "Vector.h"
+#include <utility>
+
+#include <fstream>
+#include <cmath>
+#include <ctime>
+
+#include "TTables.h"
+#include "ATables.h" 
+#include "getSentence.h"
+#include "defs.h"
+#include "model2.h"
+#include "Perplexity.h"
+#include "vocab.h"
+#include "WordClasses.h"
+#include "HMMTables.h"
+#include "ForwardBackward.h"
+
+class hmm : public model2
+{
+ private:
+  WordClasses ewordclasses;
+  WordClasses fwordclasses;
+  HMMTables<int,WordClasses> counts,probs;
+ public:
+  template<class MAPPER>
+  void makeWordClasses(const MAPPER&m1,const MAPPER&m2,string efile,string ffile)
+    {
+      ifstream estrm(efile.c_str()),fstrm(ffile.c_str());
+      if( !estrm )
+	{
+	  cerr << "ERROR: can not read " << efile << endl;
+	}
+      else
+	ewordclasses.read(estrm,m1);
+      if( !fstrm )
+	cerr << "ERROR: can not read " << ffile << endl;
+      else
+	fwordclasses.read(fstrm,m2);
+    }
+  hmm(model2&m2);
+  void initialize_table_uniformly(sentenceHandler&);
+  int em_with_tricks(int);
+  void load_table(const char* aname);
+  void em_loop(Perplexity& perp, sentenceHandler& sHandler1, bool dump_files, 
+	       const char* alignfile, Perplexity&, bool test,bool doInit,int iter);
+  HMMNetwork *makeHMMNetwork(const Vector<WordIndex>& es,const Vector<WordIndex>&fs,bool doInit)const;
+  friend class model3;
+};
+
+#endif

tools/giza-pp/GIZA++-v2/logprob.cpp

@@ -0,0 +1,154 @@
+/*
+
+EGYPT Toolkit for Statistical Machine Translation
+Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, 
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 
+USA.
+
+*/
+
+//  Routines to perform integer exponential arithmetic.
+//  A number x is represented as n, where x = b**n.
+//  It is assumed that b > 1, something like b = 1.001;
+
+#include "logprob.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <iostream>
+#include <fstream>
+#include <string>
+double *LogProb::ntof  = NULL; // Tables will be initialized
+int   *LogProb::addtbl = NULL; // in Initialize function.
+int   *LogProb::subtbl = NULL; //
+
+const int    LogProb::max_2byte_integer =  32767;
+const int    LogProb::min_2byte_integer = -32768;
+const double LogProb::b      = 1.001;   // a logarithm basis
+const double LogProb::logb2  = log(b);
+//const int    LogProb::nmax   = round(78.0E0 * log(1.0E1) / logb2);
+const int    LogProb::nmax   = round(300.0E0 * log(1.0E1) / logb2);
+const int    LogProb::nmin   = -nmax;
+const int    LogProb::tblbnd = round(log((b-1.0E0)/2.0E0)/logb2);
+const int    LogProb::zeron  = round(pow(-2, 23));
+const int    LogProb::onen   = 0;
+const int    LogProb::infn   = onen - zeron;
+
+const int LogProb::initialized = LogProb::Initialize();
+const LogProb LogProb::zero(0);
+const LogProb LogProb::one(1);
+const LogProb LogProb::minus2(1e-2);
+const LogProb LogProb::minus4(1e-4);
+const LogProb LogProb::minus6(1e-6);
+const LogProb LogProb::minus8(1e-8);
+const LogProb LogProb::minus10(1e-10);
+const LogProb LogProb::minus12(1e-12);
+const LogProb LogProb::minus14(1e-14);
+const LogProb LogProb::minus16(1e-16);
+
+// static table initialization function
+int LogProb::Initialize()
+{
+  int nbytes = sizeof(double)*(nmax-nmin+1) + sizeof(int)*(0-tblbnd+1);
+  std::cerr << nbytes << " bytes used for LogProb tables (C++ version)\n";
+  ntof   = new double[nmax-nmin+1];
+  addtbl = new int[-tblbnd+1];
+  subtbl = new int[-tblbnd+1];
+
+  //  char filename[257];
+  //  string filename ;
+  //  ifstream ifs;
+  //  ifs.open(filename.c_str());
+  //  if (!ifs)
+  //    {
+  int i;
+  std::cerr << "Building integer logs conversion tables\n";
+  ntof[0] = 0 ;
+  
+  for (i=nmin+1; i<=nmax; ++i) 
+    {
+      double x = i;
+      ntof[i-nmin] = exp(x*logb2);
+      
+    }
+  for (i=tblbnd; i<=0; ++i) 
+    {
+      double x = 1.0 + pow(b, i);
+      addtbl[i-tblbnd] = round(log(x)/logb2);
+    }
+  double sqrtb = exp(0.5*logb2);
+  for (i=0; i<=-tblbnd; ++i) 
+    {
+      double x = sqrtb * pow(b, i) - 1.0;
+      subtbl[i] = round(log(x)/logb2);
+    }
+  //      if (toolsRoot)
+  //	{
+  //      ofstream ofs(filename.c_str());
+  //      if (!ofs)
+  //	cerr << "Could not write LogProb data to " << filename << endl;
+  //      else
+  //	{
+  //	  ofs.write((const char *)ntof, sizeof(double) * (nmax-nmin+1));
+  //	  ofs.write((const char *)addtbl, sizeof(int) * (-tblbnd+1));
+  //	  ofs.write((const char *)subtbl, sizeof(int) * (-tblbnd+1));
+  //	}
+  //    }
+  //    }
+  //  else
+  //    {
+  //      ifs.read((char *)ntof,   sizeof(double) * (nmax - nmin + 1));
+  //      ifs.read((char *)addtbl, sizeof(int) * (-tblbnd+1));
+  //      ifs.read((char *)subtbl, sizeof(int) * (-tblbnd+1));
+  //    }
+  return 1;
+}
+
+void LogProb::FreeTables()
+{
+   delete [] addtbl;
+   delete [] subtbl;
+   delete [] ntof;
+}
+
+//---------------------------------------------------------------------------
+//            Aritmetic operators
+//---------------------------------------------------------------------------
+
+
+// Subtract two logarithm numbers. Use the following method:
+// b**n - b**m = b**m( b**(n-m) - 1 ), assuming n >= m. 
+LogProb& LogProb::operator-=(const LogProb &subs) 
+{
+  if (subs.logr == zeron)
+    return *this;
+  int a = logr - subs.logr;
+  if (a <= 0)
+    {
+      if (a < 0)
+	{
+	  std::cerr << "WARNING(logprob): Invalid arguments to nsub" <<(*this)<< " " << subs << std::endl;
+	  //abort();
+	}
+      logr = zeron;
+      return *this;
+    }
+  if (a > -tblbnd)
+    return *this;
+  logr = subs.logr + subtbl[a];
+  return *this;
+}
+
+