tools/mosesdecoder/phrase-extract/consolidate-direct-main.cpp

/***********************************************************************
  Moses - factored phrase-based language decoder
  Copyright (C) 2009 University of Edinburgh

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include <string.h>
#include <fstream>
#include <vector>
#include <string>
#include <iostream>
#include <cstdlib>
#include "InputFileStream.h"
#include "OutputFileStream.h"
#include "util/tokenize.hh"

using namespace std;

vector< string > splitLine(const char *line)
{
  vector< string > item;
  int start=0;
  int i=0;
  for(; line[i] != '\0'; i++) {
    if (line[i] == ' ' &&
        line[i+1] == '|' &&
        line[i+2] == '|' &&
        line[i+3] == '|' &&
        line[i+4] == ' ') {
      if (start > i) start = i; // empty item
      item.push_back( string( line+start, i-start ) );
      start = i+5;
      i += 3;
    }
  }
  item.push_back( string( line+start, i-start ) );

  return item;
}

bool getLine( istream &fileP, vector< string > &item )
{
  if (fileP.eof())
    return false;

  string line;
  if (getline(fileP, line)) {
    item = splitLine(line.c_str());
    return true;
  } else {
    return false;
  }
}


int main(int argc, char* argv[])
{
  cerr << "Starting..." << endl;

  char* &fileNameDirect = argv[1];
  Moses::InputFileStream fileDirect(fileNameDirect);


  //fileDirect.open(fileNameDirect);
  if (fileDirect.fail()) {
    cerr << "ERROR: could not open extract file " << fileNameDirect << endl;
    exit(1);
  }
  istream &fileDirectP = fileDirect;

  char* &fileNameConsolidated = argv[2];
  ostream *fileConsolidated;

  if (strcmp(fileNameConsolidated, "-") == 0) {
    fileConsolidated = &cout;
  } else {
    Moses::OutputFileStream *outputFile = new Moses::OutputFileStream();
    bool success = outputFile->Open(fileNameConsolidated);
    if (!success) {
      cerr << "ERROR: could not open file phrase table file "
           << fileNameConsolidated << endl;
      exit(1);
    }
    fileConsolidated = outputFile;
  }

  int i=0;
  while(true) {
    i++;
    if (i%1000 == 0) cerr << "." << flush;
    if (i%10000 == 0) cerr << ":" << flush;
    if (i%100000 == 0) cerr << "!" << flush;

    vector< string > itemDirect;
    if (! getLine(fileDirectP,  itemDirect  ))
      break;

    const vector< string > count = util::tokenize( itemDirect[4] );
    float countEF = atof(count[0].c_str());
    float countF = atof(count[1].c_str());
    float prob = countF/countEF;

    (*fileConsolidated) << itemDirect[0] << " ||| "        // source
                        << itemDirect[1] << " ||| "        // target
                        << prob << " ||| "                 // prob
                        << itemDirect[2] << "||| "        // alignment
                        << itemDirect[4] << " " << countEF // counts
                        << " ||| " << endl;
  }

  fileConsolidated->flush();
  if (fileConsolidated != &cout) {
    delete fileConsolidated;
  }

  cerr << "Finished" << endl;
}