mosesdecoder/moses/TranslationModel/fuzzy-match/FuzzyMatchWrapper.cpp

//
//  FuzzyMatchWrapper.cpp
//  moses
//
//  Created by Hieu Hoang on 26/07/2012.
//  Copyright 2012 __MyCompanyName__. All rights reserved.
//

#include <iostream>
#include "FuzzyMatchWrapper.h"
#include "SentenceAlignment.h"
#include "Match.h"
#include "create_xml.h"
#include "moses/Util.h"
#include "moses/StaticData.h"
#include "util/file.hh"

using namespace std;

namespace tmmt
{

FuzzyMatchWrapper::FuzzyMatchWrapper(const std::string &sourcePath, const std::string &targetPath, const std::string &alignmentPath)
  :basic_flag(false)
  ,lsed_flag(true)
  ,refined_flag(true)
  ,length_filter_flag(true)
  ,parse_flag(true)
  ,min_match(70)
  ,multiple_flag(true)
  ,multiple_slack(0)
  ,multiple_max(100)
{
  cerr << "creating suffix array" << endl;
  suffixArray = new tmmt::SuffixArray( sourcePath );

  //cerr << "loading source data" << endl;
  //load_corpus(sourcePath, source);

  cerr << "loading target data" << endl;
  load_target(targetPath, targetAndAlignment);

  cerr << "loading alignment" << endl;
  load_alignment(alignmentPath, targetAndAlignment);

  // create suffix array
  //load_corpus(m_config[0], input);

  cerr << "loading completed" << endl;
}

string FuzzyMatchWrapper::Extract(long translationId, const string &dirNameStr)
{
  const Moses::StaticData &staticData = Moses::StaticData::Instance();

  WordIndex wordIndex;

  string fuzzyMatchFile = ExtractTM(wordIndex, translationId, dirNameStr);

  // create extrac files
  create_xml(fuzzyMatchFile);

  // create phrase table with usual Moses scoring and consolidate programs
  string cmd;
  cmd = "LC_ALL=C sort " + fuzzyMatchFile + ".extract | gzip -c > "
        + fuzzyMatchFile + ".extract.sorted.gz";
  system(cmd.c_str());
  cmd = "LC_ALL=C sort " + fuzzyMatchFile + ".extract.inv | gzip -c > "
        + fuzzyMatchFile + ".extract.inv.sorted.gz";
  system(cmd.c_str());

#ifdef IS_XCODE
  cmd = "/Users/hieuhoang/unison/workspace/github/moses-smt/bin";
#elif IS_ECLIPSE
  cmd = "/home/hieu/workspace/github/moses-smt/bin";
#else
  cmd = staticData.GetBinDirectory();
#endif

  cmd += string("/../scripts/training/train-model.perl -dont-zip -first-step 6 -last-step 6 -f en -e fr -hierarchical ")
         + " -extract-file " + fuzzyMatchFile + ".extract -lexical-file - -score-options \"--NoLex\" "
         + " -phrase-translation-table " + fuzzyMatchFile + ".pt";
  system(cmd.c_str());


  return fuzzyMatchFile + ".pt.gz";
}

string FuzzyMatchWrapper::ExtractTM(WordIndex &wordIndex, long translationId, const string &dirNameStr)
{
  const std::vector< std::vector< WORD_ID > > &source = suffixArray->GetCorpus();

  string inputPath = dirNameStr + "/in";
  string fuzzyMatchFile = dirNameStr + "/fuzzyMatchFile";
  ofstream fuzzyMatchStream(fuzzyMatchFile.c_str());

  vector< vector< WORD_ID > > input;
  load_corpus(inputPath, input);

  assert(input.size() == 1);
  size_t sentenceInd = 0;

  clock_t start_clock = clock();
  // if (i % 10 == 0) cerr << ".";

  // establish some basic statistics

  // int input_length = compute_length( input[i] );
  int input_length = input[sentenceInd].size();
  int best_cost = input_length * (100-min_match) / 100 + 1;

  int match_count = 0; // how many substring matches to be considered
  //cerr << endl << "sentence " << i << ", length " << input_length << ", best_cost " << best_cost << endl;

  // find match ranges in suffix array
  vector< vector< pair< SuffixArray::INDEX, SuffixArray::INDEX > > > match_range;
  for(int start=0; start<input[sentenceInd].size(); start++) {
    SuffixArray::INDEX prior_first_match = 0;
    SuffixArray::INDEX prior_last_match = suffixArray->GetSize()-1;
    vector< string > substring;
    bool stillMatched = true;
    vector< pair< SuffixArray::INDEX, SuffixArray::INDEX > > matchedAtThisStart;
    //cerr << "start: " << start;
    for(size_t word=start; stillMatched && word<input[sentenceInd].size(); word++) {
      substring.push_back( GetVocabulary().GetWord( input[sentenceInd][word] ) );

      // only look up, if needed (i.e. no unnecessary short gram lookups)
      //				if (! word-start+1 <= short_match_max_length( input_length ) )
      //			{
      SuffixArray::INDEX first_match, last_match;
      stillMatched = false;
      if (suffixArray->FindMatches( substring, first_match, last_match, prior_first_match, prior_last_match ) ) {
        stillMatched = true;
        matchedAtThisStart.push_back( make_pair( first_match, last_match ) );
        //cerr << " (" << first_match << "," << last_match << ")";
        //cerr << " " << ( last_match - first_match + 1 );
        prior_first_match = first_match;
        prior_last_match = last_match;
      }
      //}
    }
    //cerr << endl;
    match_range.push_back( matchedAtThisStart );
  }

  clock_t clock_range = clock();

  map< int, vector< Match > > sentence_match;
  map< int, int > sentence_match_word_count;

  // go through all matches, longest first
  for(int length = input[sentenceInd].size(); length >= 1; length--) {
    // do not create matches, if these are handled by the short match function
    if (length <= short_match_max_length( input_length ) ) {
      continue;
    }

    unsigned int count = 0;
    for(int start = 0; start <= input[sentenceInd].size() - length; start++) {
      if (match_range[start].size() >= length) {
        pair< SuffixArray::INDEX, SuffixArray::INDEX > &range = match_range[start][length-1];
        // cerr << " (" << range.first << "," << range.second << ")";
        count += range.second - range.first + 1;

        for(SuffixArray::INDEX i=range.first; i<=range.second; i++) {
          size_t position = suffixArray->GetPosition( i );

          // sentence length mismatch
          size_t sentence_id = suffixArray->GetSentence( position );
          int sentence_length = suffixArray->GetSentenceLength( sentence_id );
          int diff = abs( (int)sentence_length - (int)input_length );
          // cerr << endl << i << "\tsentence " << sentence_id << ", length " << sentence_length;
          //if (length <= 2 && input_length>=5 &&
          //		sentence_match.find( sentence_id ) == sentence_match.end())
          //	continue;

          if (diff > best_cost)
            continue;

          // compute minimal cost
          int start_pos = suffixArray->GetWordInSentence( position );
          int end_pos = start_pos + length-1;
          // cerr << endl << "\t" << start_pos << "-" << end_pos << " (" << sentence_length << ") vs. "
          // << start << "-" << (start+length-1) << " (" << input_length << ")";
          // different number of prior words -> cost is at least diff
          int min_cost = abs( start - start_pos );

          // same number of words, but not sent. start -> cost is at least 1
          if (start == start_pos && start>0)
            min_cost++;

          // different number of remaining words -> cost is at least diff
          min_cost += abs( ( sentence_length-1 - end_pos ) -
                           ( input_length-1 - (start+length-1) ) );

          // same number of words, but not sent. end -> cost is at least 1
          if ( sentence_length-1 - end_pos ==
               input_length-1 - (start+length-1)
               && end_pos != sentence_length-1 )
            min_cost++;

          // cerr << " -> min_cost " << min_cost;
          if (min_cost > best_cost)
            continue;

          // valid match
          match_count++;

          // compute maximal cost
          int max_cost = max( start, start_pos )
                         + max( sentence_length-1 - end_pos,
                                input_length-1 - (start+length-1) );
          // cerr << ", max_cost " << max_cost;

          Match m = Match( start, start+length-1,
                           start_pos, start_pos+length-1,
                           min_cost, max_cost, 0);
          sentence_match[ sentence_id ].push_back( m );
          sentence_match_word_count[ sentence_id ] += length;

          if (max_cost < best_cost) {
            best_cost = max_cost;
            if (best_cost == 0) break;
          }
          //if (match_count >= MAX_MATCH_COUNT) break;
        }
      }
      // cerr << endl;
      if (best_cost == 0) break;
      //if (match_count >= MAX_MATCH_COUNT) break;
    }
    // cerr << count << " matches at length " << length << " in " << sentence_match.size() << " tm." << endl;

    if (best_cost == 0) break;
    //if (match_count >= MAX_MATCH_COUNT) break;
  }
  cerr << match_count << " matches in " << sentence_match.size() << " sentences." << endl;

  clock_t clock_matches = clock();

  // consider each sentence for which we have matches
  int old_best_cost = best_cost;
  int tm_count_word_match = 0;
  int tm_count_word_match2 = 0;
  int pruned_match_count = 0;
  if (short_match_max_length( input_length )) {
    init_short_matches(wordIndex, translationId, input[sentenceInd] );
  }
  vector< int > best_tm;
  typedef map< int, vector< Match > >::iterator I;

  clock_t clock_validation_sum = 0;

  for(I tm=sentence_match.begin(); tm!=sentence_match.end(); tm++) {
    int tmID = tm->first;
    int tm_length = suffixArray->GetSentenceLength(tmID);
    vector< Match > &match = tm->second;
    add_short_matches(wordIndex, translationId, match, source[tmID], input_length, best_cost );

    //cerr << "match in sentence " << tmID << ": " << match.size() << " [" << tm_length << "]" << endl;

    // quick look: how many words are matched
    int words_matched = 0;
    for(size_t m=0; m<match.size(); m++) {

      if (match[m].min_cost <= best_cost) // makes no difference
        words_matched += match[m].input_end - match[m].input_start + 1;
    }
    if (max(input_length,tm_length) - words_matched > best_cost) {
      if (length_filter_flag) continue;
    }
    tm_count_word_match++;

    // prune, check again how many words are matched
    vector< Match > pruned = prune_matches( match, best_cost );
    words_matched = 0;
    for(size_t p=0; p<pruned.size(); p++) {
      words_matched += pruned[p].input_end - pruned[p].input_start + 1;
    }
    if (max(input_length,tm_length) - words_matched > best_cost) {
      if (length_filter_flag) continue;
    }
    tm_count_word_match2++;

    pruned_match_count += pruned.size();
    int prior_best_cost = best_cost;
    int cost;

    clock_t clock_validation_start = clock();
    if (! parse_flag ||
        pruned.size()>=10) { // to prevent worst cases
      string path;
      cost = sed( input[sentenceInd], source[tmID], path, false );
      if (cost <  best_cost) {
        best_cost = cost;
      }
    }

    else {
      cost = parse_matches( pruned, input_length, tm_length, best_cost );
      if (prior_best_cost != best_cost) {
        best_tm.clear();
      }
    }
    clock_validation_sum += clock() - clock_validation_start;
    if (cost == best_cost) {
      best_tm.push_back( tmID );
    }
  }
  cerr << "reduced best cost from " << old_best_cost << " to " << best_cost << endl;
  cerr << "tm considered: " << sentence_match.size()
       << " word-matched: " << tm_count_word_match
       << " word-matched2: " << tm_count_word_match2
       << " best: " << best_tm.size() << endl;

  cerr << "pruned matches: " << ((float)pruned_match_count/(float)tm_count_word_match2) << endl;

  // create xml and extract files
  string inputStr, sourceStr;
  for (size_t pos = 0; pos < input_length; ++pos) {
    inputStr += GetVocabulary().GetWord(input[sentenceInd][pos]) + " ";
  }

  // do not try to find the best ... report multiple matches
  if (multiple_flag) {
    for(size_t si=0; si<best_tm.size(); si++) {
      int s = best_tm[si];
      string path;
      sed( input[sentenceInd], source[s], path, true );
      const vector<WORD_ID> &sourceSentence = source[s];
      vector<SentenceAlignment> &targets = targetAndAlignment[s];
      create_extract(sentenceInd, best_cost, sourceSentence, targets, inputStr, path, fuzzyMatchStream);

    }
  } // if (multiple_flag)
  else {

    // find the best matches according to letter sed
    string best_path = "";
    int best_match = -1;
    unsigned int best_letter_cost;
    if (lsed_flag) {
      best_letter_cost = compute_length( input[sentenceInd] ) * min_match / 100 + 1;
      for(size_t si=0; si<best_tm.size(); si++) {
        int s = best_tm[si];
        string path;
        unsigned int letter_cost = sed( input[sentenceInd], source[s], path, true );
        if (letter_cost < best_letter_cost) {
          best_letter_cost = letter_cost;
          best_path = path;
          best_match = s;
        }
      }
    }
    // if letter sed turned off, just compute path for first match
    else {
      if (best_tm.size() > 0) {
        string path;
        sed( input[sentenceInd], source[best_tm[0]], path, false );
        best_path = path;
        best_match = best_tm[0];
      }
    }
    cerr << "elapsed: " << (1000 * (clock()-start_clock) / CLOCKS_PER_SEC)
         << " ( range: " << (1000 * (clock_range-start_clock) / CLOCKS_PER_SEC)
         << " match: " << (1000 * (clock_matches-clock_range) / CLOCKS_PER_SEC)
         << " tm: " << (1000 * (clock()-clock_matches) / CLOCKS_PER_SEC)
         << " (validation: " << (1000 * (clock_validation_sum) / CLOCKS_PER_SEC) << ")"
         << " )" << endl;
    if (lsed_flag) {
      //cout << best_letter_cost << "/" << compute_length( input[sentenceInd] ) << " (";
    }
    //cout << best_cost <<"/" << input_length;
    if (lsed_flag) {
      //cout << ")";
    }
    //cout << " ||| " << best_match << " ||| " << best_path << endl;

    if (best_match == -1) {
      UTIL_THROW_IF2(source.size() == 0, "Empty source phrase");
      best_match = 0;
    }

    // creat xml & extracts
    const vector<WORD_ID> &sourceSentence = source[best_match];
    vector<SentenceAlignment> &targets = targetAndAlignment[best_match];
    create_extract(sentenceInd, best_cost, sourceSentence, targets, inputStr, best_path, fuzzyMatchStream);

  } // else if (multiple_flag)

  fuzzyMatchStream.close();

  return fuzzyMatchFile;
}

void FuzzyMatchWrapper::load_corpus( const std::string &fileName, vector< vector< WORD_ID > > &corpus )
{
  // source
  ifstream fileStream;
  fileStream.open(fileName.c_str());
  if (!fileStream) {
    cerr << "file not found: " << fileName << endl;
    exit(1);
  }
  cerr << "loading " << fileName << endl;

  istream *fileStreamP = &fileStream;

  string line;
  while(getline(*fileStreamP, line)) {
    corpus.push_back( GetVocabulary().Tokenize( line.c_str() ) );
  }
}

void FuzzyMatchWrapper::load_target(const std::string &fileName, vector< vector< SentenceAlignment > > &corpus)
{
  ifstream fileStream;
  fileStream.open(fileName.c_str());
  if (!fileStream) {
    cerr << "file not found: " << fileName << endl;
    exit(1);
  }
  cerr << "loading " << fileName << endl;

  istream *fileStreamP = &fileStream;

  WORD_ID delimiter = GetVocabulary().StoreIfNew("|||");

  int lineNum = 0;
  string line;
  while(getline(*fileStreamP, line)) {
    vector<WORD_ID> toks = GetVocabulary().Tokenize( line.c_str() );

    corpus.push_back(vector< SentenceAlignment >());
    vector< SentenceAlignment > &vec = corpus.back();

    vec.push_back(SentenceAlignment());
    SentenceAlignment *sentence = &vec.back();

    const WORD &countStr = GetVocabulary().GetWord(toks[0]);
    sentence->count = atoi(countStr.c_str());

    for (size_t i = 1; i < toks.size(); ++i) {
      WORD_ID wordId = toks[i];

      if (wordId == delimiter) {
        // target and alignments can have multiple sentences.
        vec.push_back(SentenceAlignment());
        sentence = &vec.back();

        // count
        ++i;

        const WORD &countStr = GetVocabulary().GetWord(toks[i]);
        sentence->count = atoi(countStr.c_str());
      } else {
        // just a normal word, add
        sentence->target.push_back(wordId);
      }
    }

    ++lineNum;

  }

}


void FuzzyMatchWrapper::load_alignment(const std::string &fileName, vector< vector< SentenceAlignment > > &corpus )
{
  ifstream fileStream;
  fileStream.open(fileName.c_str());
  if (!fileStream) {
    cerr << "file not found: " << fileName << endl;
    exit(1);
  }
  cerr << "loading " << fileName << endl;

  istream *fileStreamP = &fileStream;

  string delimiter = "|||";

  int lineNum = 0;
  string line;
  while(getline(*fileStreamP, line)) {
    vector< SentenceAlignment > &vec = corpus[lineNum];
    size_t targetInd = 0;
    SentenceAlignment *sentence = &vec[targetInd];

    vector<string> toks = Moses::Tokenize(line);

    for (size_t i = 0; i < toks.size(); ++i) {
      string &tok = toks[i];

      if (tok == delimiter) {
        // target and alignments can have multiple sentences.
        ++targetInd;
        sentence = &vec[targetInd];

        ++i;
      } else {
        // just a normal alignment, add
        vector<int> alignPoint = Moses::Tokenize<int>(tok, "-");
        assert(alignPoint.size() == 2);
        sentence->alignment.push_back(pair<int,int>(alignPoint[0], alignPoint[1]));
      }
    }

    ++lineNum;

  }
}

bool FuzzyMatchWrapper::GetLSEDCache(const std::pair< WORD_ID, WORD_ID > &key, unsigned int &value) const
{
#ifdef WITH_THREADS
  boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
  map< pair< WORD_ID, WORD_ID >, unsigned int >::const_iterator lookup = m_lsed.find( key );
  if (lookup != m_lsed.end()) {
    value = lookup->second;
    return true;
  }

  return false;
}

void FuzzyMatchWrapper::SetLSEDCache(const std::pair< WORD_ID, WORD_ID > &key, const unsigned int &value)
{
#ifdef WITH_THREADS
  boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
  m_lsed[ key ] = value;
}

/* Letter string edit distance, e.g. sub 'their' to 'there' costs 2 */

unsigned int FuzzyMatchWrapper::letter_sed( WORD_ID aIdx, WORD_ID bIdx )
{
  // check if already computed -> lookup in cache
  pair< WORD_ID, WORD_ID > pIdx = make_pair( aIdx, bIdx );
  unsigned int value;
  bool ret = GetLSEDCache(pIdx, value);
  if (ret) {
    return value;
  }

  // get surface strings for word indices
  const string &a = GetVocabulary().GetWord( aIdx );
  const string &b = GetVocabulary().GetWord( bIdx );

  // initialize cost matrix
  unsigned int **cost  = (unsigned int**) calloc( sizeof( unsigned int*  ), a.size()+1 );
  for( unsigned int i=0; i<=a.size(); i++ ) {
    cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 );
    cost[i][0] = i;
  }
  for( unsigned int j=0; j<=b.size(); j++ ) {
    cost[0][j] = j;
  }

  // core string edit distance loop
  for( unsigned int i=1; i<=a.size(); i++ ) {
    for( unsigned int j=1; j<=b.size(); j++ ) {

      unsigned int ins = cost[i-1][j] + 1;
      unsigned int del = cost[i][j-1] + 1;
      bool match = (a.substr(i-1,1).compare( b.substr(j-1,1) ) == 0);
      unsigned int diag = cost[i-1][j-1] + (match ? 0 : 1);

      unsigned int min = (ins < del) ? ins : del;
      min = (diag < min) ? diag : min;

      cost[i][j] = min;
    }
  }

  // clear out memory
  unsigned int final = cost[a.size()][b.size()];
  for( unsigned int i=0; i<=a.size(); i++ ) {
    free( cost[i] );
  }
  free( cost );

  // cache and return result
  SetLSEDCache(pIdx, final);
  return final;
}

/* string edit distance implementation */

unsigned int FuzzyMatchWrapper::sed( const vector< WORD_ID > &a, const vector< WORD_ID > &b, string &best_path, bool use_letter_sed )
{

  // initialize cost and path matrices
  unsigned int **cost  = (unsigned int**) calloc( sizeof( unsigned int* ), a.size()+1 );
  char **path = (char**) calloc( sizeof( char* ), a.size()+1 );

  for( unsigned int i=0; i<=a.size(); i++ ) {
    cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 );
    path[i] = (char*) calloc( sizeof(char), b.size()+1 );
    if (i>0) {
      cost[i][0] = cost[i-1][0];
      if (use_letter_sed) {
        cost[i][0] += GetVocabulary().GetWord( a[i-1] ).size();
      } else {
        cost[i][0]++;
      }
    } else {
      cost[i][0] = 0;
    }
    path[i][0] = 'I';
  }

  for( unsigned int j=0; j<=b.size(); j++ ) {
    if (j>0) {
      cost[0][j] = cost[0][j-1];
      if (use_letter_sed) {
        cost[0][j] +=	GetVocabulary().GetWord( b[j-1] ).size();
      } else {
        cost[0][j]++;
      }
    } else {
      cost[0][j] = 0;
    }
    path[0][j] = 'D';
  }

  // core string edit distance algorithm
  for( unsigned int i=1; i<=a.size(); i++ ) {
    for( unsigned int j=1; j<=b.size(); j++ ) {
      unsigned int ins = cost[i-1][j];
      unsigned int del = cost[i][j-1];
      unsigned int match;
      if (use_letter_sed) {
        ins += GetVocabulary().GetWord( a[i-1] ).size();
        del += GetVocabulary().GetWord( b[j-1] ).size();
        match = letter_sed( a[i-1], b[j-1] );
      } else {
        ins++;
        del++;
        match = ( a[i-1] == b[j-1] ) ? 0 : 1;
      }
      unsigned int diag = cost[i-1][j-1] + match;

      char action = (ins < del) ? 'I' : 'D';
      unsigned int min = (ins < del) ? ins : del;
      if (diag < min) {
        action = (match>0) ? 'S' : 'M';
        min = diag;
      }

      cost[i][j] = min;
      path[i][j] = action;
    }
  }

  // construct string for best path
  unsigned int i = a.size();
  unsigned int j = b.size();
  best_path = "";
  while( i>0 || j>0 ) {
    best_path = path[i][j] + best_path;
    if (path[i][j] == 'I') {
      i--;
    } else if (path[i][j] == 'D') {
      j--;
    } else {
      i--;
      j--;
    }
  }


  // clear out memory
  unsigned int final = cost[a.size()][b.size()];

  for( unsigned int i=0; i<=a.size(); i++ ) {
    free( cost[i] );
    free( path[i] );
  }
  free( cost );
  free( path );

  // return result
  return final;
}

/* utlility function: compute length of sentence in characters
 (spaces do not count) */

unsigned int FuzzyMatchWrapper::compute_length( const vector< WORD_ID > &sentence )
{
  unsigned int length = 0;
  for( unsigned int i=0; i<sentence.size(); i++ ) {
    length += GetVocabulary().GetWord( sentence[i] ).size();
  }
  return length;
}

/* brute force method: compare input to all corpus sentences */

void FuzzyMatchWrapper::basic_fuzzy_match( vector< vector< WORD_ID > > source,
    vector< vector< WORD_ID > > input )
{
  // go through input set...
  for(unsigned int i=0; i<input.size(); i++) {
    bool use_letter_sed = false;

    // compute sentence length and worst allowed cost
    unsigned int input_length;
    if (use_letter_sed) {
      input_length = compute_length( input[i] );
    } else {
      input_length = input[i].size();
    }
    unsigned int best_cost = input_length * (100-min_match) / 100 + 2;
    string best_path = "";
    //int best_match = -1;

    // go through all corpus sentences
    for(unsigned int s=0; s<source.size(); s++) {
      int source_length;
      if (use_letter_sed) {
        source_length = compute_length( source[s] );
      } else {
        source_length = source[s].size();
      }
      int diff = abs((int)source_length - (int)input_length);
      if (length_filter_flag && (diff >= best_cost)) {
        continue;
      }

      // compute string edit distance
      string path;
      unsigned int cost = sed( input[i], source[s], path, use_letter_sed );

      // update if new best
      if (cost < best_cost) {
        best_cost = cost;
        best_path = path;
        //best_match = s;
      }
    }
    //cout << best_cost << " ||| " << best_match << " ||| " << best_path << endl;
  }
}

/* definition of short matches
 very short n-gram matches (1-grams) will not be looked up in
 the suffix array, since there are too many matches
 and for longer sentences, at least one 2-gram match must occur */

int FuzzyMatchWrapper::short_match_max_length( int input_length )
{
  if ( ! refined_flag )
    return 0;
  if ( input_length >= 5 )
    return 1;
  return 0;
}


/* if we have non-short matches in a sentence, we need to
 take a closer look at it.
 this function creates a hash map for all input words and their positions
 (to be used by the next function)
 (done here, because this has be done only once for an input sentence) */

void FuzzyMatchWrapper::init_short_matches(WordIndex &wordIndex, long translationId, const vector< WORD_ID > &input )
{
  int max_length = short_match_max_length( input.size() );
  if (max_length == 0)
    return;

  wordIndex.clear();

  // store input words and their positions in hash map
  for(size_t i=0; i<input.size(); i++) {
    if (wordIndex.find( input[i] ) == wordIndex.end()) {
      vector< int > position_vector;
      wordIndex[ input[i] ] = position_vector;
    }
    wordIndex[ input[i] ].push_back( i );
  }
}

/* add all short matches to list of matches for a sentence */

void FuzzyMatchWrapper::add_short_matches(WordIndex &wordIndex, long translationId, vector< Match > &match, const vector< WORD_ID > &tm, int input_length, int best_cost )
{
  int max_length = short_match_max_length( input_length );
  if (max_length == 0)
    return;

  int tm_length = tm.size();
  map< WORD_ID,vector< int > >::iterator input_word_hit;
  for(int t_pos=0; t_pos<tm.size(); t_pos++) {
    input_word_hit = wordIndex.find( tm[t_pos] );
    if (input_word_hit != wordIndex.end()) {
      vector< int > &position_vector = input_word_hit->second;
      for(size_t j=0; j<position_vector.size(); j++) {
        int &i_pos = position_vector[j];

        // before match
        int max_cost = max( i_pos , t_pos );
        int min_cost = abs( i_pos - t_pos );
        if ( i_pos>0 && i_pos == t_pos )
          min_cost++;

        // after match
        max_cost += max( (input_length-i_pos) , (tm_length-t_pos));
        min_cost += abs( (input_length-i_pos) - (tm_length-t_pos));
        if ( i_pos != input_length-1 && (input_length-i_pos) == (tm_length-t_pos))
          min_cost++;

        if (min_cost <= best_cost) {
          Match new_match( i_pos,i_pos, t_pos,t_pos, min_cost,max_cost,0 );
          match.push_back( new_match );
        }
      }
    }
  }
}

/* remove matches that are subsumed by a larger match */

vector< Match > FuzzyMatchWrapper::prune_matches( const vector< Match > &match, int best_cost )
{
  //cerr << "\tpruning";
  vector< Match > pruned;
  for(int i=match.size()-1; i>=0; i--) {
    //cerr << " (" << match[i].input_start << "," << match[i].input_end
    //		 << " ; " << match[i].tm_start << "," << match[i].tm_end
    //		 << " * " << match[i].min_cost << ")";

    //if (match[i].min_cost > best_cost)
    //	continue;

    bool subsumed = false;
    for(int j=match.size()-1; j>=0; j--) {
      if (i!=j // do not compare match with itself
          && ( match[i].input_end - match[i].input_start <=
               match[j].input_end - match[j].input_start ) // i shorter than j
          && ((match[i].input_start == match[j].input_start &&
               match[i].tm_start    == match[j].tm_start	) ||
              (match[i].input_end   == match[j].input_end &&
               match[i].tm_end      == match[j].tm_end) ) ) {
        subsumed = true;
      }
    }
    if (! subsumed && match[i].min_cost <= best_cost) {
      //cerr << "*";
      pruned.push_back( match[i] );
    }
  }
  //cerr << endl;
  return pruned;
}

/* A* parsing method to compute string edit distance */

int FuzzyMatchWrapper::parse_matches( vector< Match > &match, int input_length, int tm_length, int &best_cost )
{
  // cerr << "sentence has " << match.size() << " matches, best cost: " << best_cost << ", lengths input: " << input_length << " tm: " << tm_length << endl;

  if (match.size() == 1)
    return match[0].max_cost;
  if (match.size() == 0)
    return input_length+tm_length;

  int this_best_cost = input_length + tm_length;
  for(size_t i=0; i<match.size(); i++) {
    this_best_cost = min( this_best_cost, match[i].max_cost );
  }
  // cerr << "\tthis best cost: " << this_best_cost << endl;

  // bottom up combination of spans
  vector< vector< Match > > multi_match;
  multi_match.push_back( match );

  int match_level = 1;
  while(multi_match[ match_level-1 ].size()>0) {
    // init vector
    vector< Match > empty;
    multi_match.push_back( empty );

    for(int first_level = 0; first_level <= (match_level-1)/2; first_level++) {
      int second_level = match_level - first_level -1;
      //cerr << "\tcombining level " << first_level << " and " << second_level << endl;

      vector< Match > &first_match  = multi_match[ first_level ];
      vector< Match > &second_match = multi_match[ second_level ];

      for(size_t i1 = 0; i1 < first_match.size(); i1++) {
        for(size_t i2 = 0; i2 < second_match.size(); i2++) {

          // do not combine the same pair twice
          if (first_level == second_level && i2 <= i1) {
            continue;
          }

          // get sorted matches (first is before second)
          Match *first, *second;
          if (first_match[i1].input_start < second_match[i2].input_start ) {
            first = &first_match[i1];
            second = &second_match[i2];
          } else {
            second = &first_match[i1];
            first = &second_match[i2];
          }

          //cerr << "\tcombining "
          //		 << "(" << first->input_start << "," << first->input_end << "), "
          //		 << first->tm_start << " [" << first->internal_cost << "]"
          //		 << " with "
          //		 << "(" << second->input_start << "," << second->input_end << "), "
          //		 << second->tm_start<< " [" << second->internal_cost << "]"
          //		 << endl;

          // do not process overlapping matches
          if (first->input_end >= second->input_start) {
            continue;
          }

          // no overlap / mismatch in tm
          if (first->tm_end >= second->tm_start) {
            continue;
          }

          // compute cost
          int min_cost = 0;
          int max_cost = 0;

          // initial
          min_cost += abs( first->input_start - first->tm_start );
          max_cost += max( first->input_start, first->tm_start );

          // same number of words, but not sent. start -> cost is at least 1
          if (first->input_start == first->tm_start && first->input_start > 0) {
            min_cost++;
          }

          // in-between
          int skipped_words = second->input_start - first->input_end -1;
          int skipped_words_tm = second->tm_start - first->tm_end -1;
          int internal_cost = max( skipped_words, skipped_words_tm );
          internal_cost += first->internal_cost + second->internal_cost;
          min_cost += internal_cost;
          max_cost += internal_cost;

          // final
          min_cost += abs( (tm_length-1 - second->tm_end) -
                           (input_length-1 - second->input_end) );
          max_cost += max( (tm_length-1 - second->tm_end),
                           (input_length-1 - second->input_end) );

          // same number of words, but not sent. end -> cost is at least 1
          if ( ( input_length-1 - second->input_end
                 == tm_length-1 - second->tm_end )
               && input_length-1 != second->input_end ) {
            min_cost++;
          }

          // cerr << "\tcost: " << min_cost << "-" << max_cost << endl;

          // if worst than best cost, forget it
          if (min_cost > best_cost) {
            continue;
          }

          // add match
          Match new_match( first->input_start,
                           second->input_end,
                           first->tm_start,
                           second->tm_end,
                           min_cost,
                           max_cost,
                           internal_cost);
          multi_match[ match_level ].push_back( new_match );
          // cerr << "\tstored\n";

          // possibly updating this_best_cost
          if (max_cost < this_best_cost) {
            // cerr << "\tupdating this best cost to " << max_cost << "\n";
            this_best_cost = max_cost;

            // possibly updating best_cost
            if (max_cost < best_cost) {
              // cerr << "\tupdating best cost to " << max_cost << "\n";
              best_cost = max_cost;
            }
          }
        }
      }
    }
    match_level++;
  }
  return this_best_cost;
}


void FuzzyMatchWrapper::create_extract(int sentenceInd, int cost, const vector< WORD_ID > &sourceSentence, const vector<SentenceAlignment> &targets, const string &inputStr, const string  &path, ofstream &outputFile)
{
  string sourceStr;
  for (size_t pos = 0; pos < sourceSentence.size(); ++pos) {
    WORD_ID wordId = sourceSentence[pos];
    sourceStr += GetVocabulary().GetWord(wordId) + " ";
  }

  for (size_t targetInd = 0; targetInd < targets.size(); ++targetInd) {
    const SentenceAlignment &sentenceAlignment = targets[targetInd];
    string targetStr = sentenceAlignment.getTargetString(GetVocabulary());
    string alignStr = sentenceAlignment.getAlignmentString();

    outputFile
        << sentenceInd << endl
        << cost << endl
        << sourceStr << endl
        << inputStr << endl
        << targetStr << endl
        << alignStr << endl
        << path << endl
        << sentenceAlignment.count << endl;

  }
}

} // namespace