File size: 1,831 Bytes
d3dc4c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | #include "search/rule.hh"
#include "lm/model.hh"
#include "search/context.hh"
#include <ostream>
#include <cmath>
namespace search {
template <class Model> ScoreRuleRet ScoreRule(const Model &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing) {
ScoreRuleRet ret;
ret.prob = 0.0;
ret.oov = 0;
const lm::WordIndex oov = model.GetVocabulary().NotFound(), bos = model.GetVocabulary().BeginSentence();
lm::ngram::RuleScore<Model> scorer(model, *(writing++));
std::vector<lm::WordIndex>::const_iterator word = words.begin();
if (word != words.end() && *word == bos) {
scorer.BeginSentence();
++word;
}
for (; word != words.end(); ++word) {
if (*word == kNonTerminal) {
ret.prob += scorer.Finish();
scorer.Reset(*(writing++));
} else {
if (*word == oov) ++ret.oov;
scorer.Terminal(*word);
}
}
ret.prob += scorer.Finish();
return ret;
}
template ScoreRuleRet ScoreRule(const lm::ngram::RestProbingModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
template ScoreRuleRet ScoreRule(const lm::ngram::ProbingModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
template ScoreRuleRet ScoreRule(const lm::ngram::TrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
template ScoreRuleRet ScoreRule(const lm::ngram::QuantTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
template ScoreRuleRet ScoreRule(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
template ScoreRuleRet ScoreRule(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words, lm::ngram::ChartState *writing);
} // namespace search
|