Upload 561 files

.gitattributes

@@ -38,3 +38,8 @@ HiSd/phrase-table.minphr filter=lfs diff=lfs merge=lfs -text
 HiSd/reordering-table.minlexr filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/cmph-2.0.2/lib/libcmph.a filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/cmph-2.0.2/src/.libs/libcmph.a filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/expected-bleu-training/bin/gcc-9/release/link-static/threading-multi/prepare-expected-bleu-training filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/expected-bleu-training/bin/gcc-9/release/link-static/threading-multi/train-expected-bleu filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1 filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/server/bin/gcc-9/release/link-static/threading-multi/mosesserver filter=lfs diff=lfs merge=lfs -text

mosesdecoder/contrib/DIMwid/DIMputs.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+import collections
+import re
+
+
+class DataInput():
+    def __init__(self, file_name):
+        self.file = open(file_name, "r")
+        self.sentences = None
+
+        
+    def read_phrase(self):
+        self.sentences = []
+        sentence = None
+        span_reg = re.compile("\|[0-9]+-[0-9]+\|")
+        previous = ""
+        for line in self.file:
+            sentence = Single()
+            for word in line.split():
+                if span_reg.match(word):
+                    sentence.spans[tuple([int(i) for i in word.strip("|").split("-")])] = previous.strip()
+                    previous = " "
+                else:
+                    previous += word + " "
+            sentence.set_length()
+            self.sentences.append(sentence)
+            sentence.number = len(self.sentences)
+
+    def read_syntax(self):
+        self.sentences = []
+        sentence = None
+        number = -1
+        for line in self.file:
+            if int(line.split()[2]) != number:
+                if sentence is not None:
+                    sentence.set_length()
+                    self.sentences.append(sentence)
+                sentence = Single()
+                sentence.number = int(line.split()[2])
+                number = sentence.number
+            sentence.spans[tuple([int(i) for i in line.split()[3].strip(":[]").split("..")])] \
+ = line.strip()
+
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+                # = tuple([line.split(":")[1], line.split(":")[2], line.split(":")[3]])
+                
+    
+    def read_syntax_cubes(self, cell_limit):
+        self.sentences = []
+        sentence = None
+        number = -1
+        new_item = False
+        for line in self.file:
+            if  line.startswith("Chart Cell"):
+                pass  # we dont care for those lines
+            elif line.startswith("---------"):
+                new_item = True
+            elif line.startswith("Trans Opt") and new_item is True:
+                new_item = False
+                if int(line.split()[2]) != number:
+                    if sentence is not None:
+                        sentence.set_length()
+                        self.sentences.append(sentence)
+                    sentence = Multiple()
+                    sentence.number = int(line.split()[2])
+                    number = sentence.number
+                span = tuple([int(i) for i in line.split()[3].strip(":[]").split("..")])                
+                if len(sentence.spans[span]) < cell_limit:
+                    sentence.spans[span].append(line.strip())
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+    
+    def read_phrase_stack_flag(self, cell_limit):
+        self.sentences = []
+        sentence = None
+        number = -1
+        for line in self.file:
+            if len(line.split()) < 6:
+                pass
+#            elif re.match("recombined=[0-9]+", line.split()[6]):
+#                pass
+            else:
+                if int(line.split()[0]) != number:
+                    if sentence is not None:
+                        sentence.set_length()
+                        self.sentences.append(sentence)
+                    sentence = Multiple()
+                    sentence.number = int(line.split()[0])
+                    number = sentence.number
+#                span = tuple([int(i) for i in line.split()[8].split("=")[1].split("-")])
+                span = re.search(r"covered=([0-9]+\-[0-9]+)", line).expand("\g<1>")
+                # print span.expand("\g<1>")
+                span = tuple([int(i) for i in span.split("-")])
+                if len(sentence.spans[span]) < cell_limit:
+                    sentence.spans[span].append(line.strip())
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+            
+    def read_phrase_stack_verbose(self, cell_limit):
+        self.sentences = []
+        sentence = None
+        number = -1
+        span_input = False
+        for line in self.file:
+            if line.startswith("Translating: "):
+                if sentence is not None:
+                    sentence.set_length()
+                    self.sentences.append(sentence)
+                    
+                number += 1
+                sentence = Multiple()
+                sentence.number = number
+            else:
+                if re.match("\[[A-Z,a-z,\ ]+;\ [0-9]+-[0-9]+\]", line):
+                    span = tuple([int(i) for i in line.split(";")[1].strip().strip("]").split("-")])
+                    sentence.spans[span].append(line.strip())
+                    span_input = True
+#                    print line,
+                elif span_input is True:
+                    if line.strip() == "":
+                        span_input = False
+#                        print "X"
+                    else:
+                        if len(sentence.spans[span]) < cell_limit:
+                            sentence.spans[span].append(line.strip())
+#                        print line,
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+            
+            
+
+    def read_syntax_cube_flag(self, cell_limit):
+        self.sentences = []
+        sentence = None
+        number = -1
+        for line in self.file:
+            if len(line.split()) < 6:
+                pass
+            else:
+                if int(line.split()[0]) != number:
+                    if sentence is not None:
+                        sentence.set_length()
+                        self.sentences.append(sentence)
+                    sentence = Multiple()  # 
+                    sentence.number = int(line.split()[0])
+                    number = sentence.number
+                span = re.search(r"\[([0-9]+)\.\.([0-9]+)\]", line).expand("\g<1> \g<2>")
+                span = tuple([int(i) for i in span.split()])
+                if len(sentence.spans[span]) < cell_limit:
+                    sentence.spans[span].append(line.strip())
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+        
+        
+    def read_mbot(self, cell_limit):
+        self.sentences = []
+        sentence = None
+        number = -1
+        hypo = False
+        rule = False
+        popping = False
+        target = ""
+        source = ""
+        source_parent = ""
+        target_parent = ""
+        alignment = ""
+        for line in self.file:
+            if line.startswith("Translating:"):
+                if sentence is not None:
+                    sentence.set_length()
+                    self.sentences.append(sentence)
+                sentence = Multiple()
+                sentence.number = number + 1
+                number = sentence.number
+            elif line.startswith("POPPING"):
+                popping = True 
+            elif popping is True:
+                popping = False
+                span = tuple([int(i) for i in line.split()[1].strip("[").split("]")[0].split("..")])
+                hypo = True
+            elif hypo is True:
+                if line.startswith("Target Phrases"):
+                    target = line.split(":", 1)[1].strip()
+                
+                elif line.startswith("Alignment Info"):
+                    alignment = line.split(":", 1)[1].strip()
+                    if alignment == "":
+                        alignment = "(1)"
+                    
+                elif line.startswith("Source Phrase"):
+                    source = line.split(":", 1)[1].strip()
+    
+                elif line.startswith("Source Left-hand-side"):
+                    source_parent = line.split(":", 1)[1].strip()
+    
+                elif line.startswith("Target Left-hand-side"):
+                    target_parent = line.split(":", 1)[1].strip()
+    
+                    # Input stored: now begin translation into rule-format
+                    alignment = re.sub(r"\([0-9]+\)", "||", alignment)
+                    align_blocks = alignment.split("||")[:-1]
+                    target = re.sub(r"\([0-9]+\)", "||", target)
+                    target = [x.split() for x in target.split("||")][:-1]
+                    source = source.split()
+    
+                    for i in range(len(source)):
+                        if source[i].isupper():
+                            source[i] = "[" + source[i] + "]"
+                            for k in range(len(align_blocks)):
+                                align_pairs = [tuple([int(y) for y in x.split("-")]) for x in align_blocks[k].split()]
+                                for j in filter(lambda x: x[0] == i, align_pairs):
+                                    source[i] = source[i] + "[" + target[k][j[1]] + "]"
+    
+                    for i in range(len(target)):
+                        for j in range(len(target[i])):
+                            align_pairs = [tuple([int(y) for y in x.split("-")]) for x in align_blocks[i].split()]
+                            for k in filter(lambda x: x[1] == j, align_pairs):
+                                target[i][j] = source[k[0]].split("]")[0] + "][" + target[i][j] + "]"
+                
+                
+                
+                    target = " || ".join([" ".join(x) for x in target]) + " ||"
+    
+                    source = " ".join(source)
+                    source = source + "  [" + source_parent + "]"
+    
+                    tp = re.sub(r"\([0-9]+\)", "", target_parent).split()
+                    for i in tp:
+                        target = target.replace("||", " [" + i + "] !!", 1)
+                    target = target.replace("!!", "||")
+                        
+                    rule = False
+                    search_pattern = "|||  " + source + " ||| " + target + "| ---  ||| " + alignment + "|" 
+    
+                    sentence.spans[span].append(search_pattern)
+#                    print search_pattern, span
+                    if len(sentence.spans[span]) < cell_limit:
+                        sentence.spans[span].append(search_pattern)
+            else:
+                pass
+        if sentence is not None:
+            sentence.set_length()
+            self.sentences.append(sentence)
+
+
+
+
+class Single():
+    def __init__(self):
+        self.number = None
+        self.spans = {}
+        self.length = None
+
+    def set_length(self):
+        self.length = max([x[1] for x in self.spans.keys()])
+    
+    def __str__(self):
+        number = str(self.number)
+        length = str(self.length)
+        spans = "\n"
+        for i in self.spans.keys():
+            spans += str(i) + " - " + str(self.spans[i]) + "\n"
+        return str((number, length, spans))
+
+class Multiple():
+    def __init__(self):
+        self.number = None
+        self.spans = collections.defaultdict(list)
+        self.length = None
+
+    def set_length(self):
+        self.length = max([x[1] for x in self.spans.keys()])
+    
+    def __str__(self):
+        number = str(self.number)
+        length = str(self.length)
+        spans = "\n"
+        for i in self.spans.keys():
+            spans += str(i) + " - " + str(self.spans[i]) + "\n"
+        return str((number, length, spans))
+
+
+

mosesdecoder/contrib/DIMwid/DIMterface.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from PyQt4 import QtCore, QtGui
+
+import DIMputs as my_DI
+
+
+        
+class MainWindow(QtGui.QWidget):
+    updateSignal = QtCore.pyqtSignal()
+    def __init__(self, parent=None):
+    
+    
+        self.path = ""
+        self.cur_rein_num = 0
+        self.data = None
+        self.format = ""
+        self.cell_limit = float("inf")
+
+        super(MainWindow, self).__init__(parent)
+
+        # upper buttons
+        pathLabel = QtGui.QLabel("Path:")
+        self.pathLabel = QtGui.QLabel(self.path)
+        self.pathLabel.setFrameStyle(QtGui.QFrame.StyledPanel | 
+                                     QtGui.QFrame.Sunken)
+        self.pathLabel.setToolTip("Current File")
+        self.pathButton = QtGui.QPushButton("P&ath...")
+        self.pathButton.setToolTip("Set the item you want to inspect")
+        self.connect(self.pathButton, QtCore.SIGNAL("clicked()"), self.setPath)
+        
+
+        # cell limit label and text field
+        cell_limit_label = QtGui.QLabel("Cell Limit:")
+        self.cell_limit_chooser = QtGui.QSpinBox()
+        self.cell_limit_chooser.setMaximum(99999)
+        cell_limit_label.setToolTip("Limits the number of elements per cell")
+        self.cell_limit_chooser.setToolTip("Set to zero to show all elements")
+
+        # format drop down menu
+        self.format_drop = QtGui.QToolButton(self)
+        self.format_drop.setPopupMode(QtGui.QToolButton.MenuButtonPopup)
+        self.format_drop.setMenu(QtGui.QMenu(self.format_drop))
+        self.format_drop.setText("Format")
+
+        self.format_syntax = QtGui.QPushButton("Syntax")
+        self.format_phrase = QtGui.QPushButton("Phrase")
+        self.format_syntaxCube = QtGui.QPushButton("Syntax Cube (-Tall flag)")
+        self.format_phraseStackFlag = QtGui.QPushButton("Phrase Stack (search-graph)")
+        self.format_phraseStackVerbose = QtGui.QPushButton("Phrase Stack (verbose)")
+        self.format_syntaxCubeFlag = QtGui.QPushButton("Syntax Cube (search-graph)")
+        self.format_mbot = QtGui.QPushButton("MBOT")
+        
+
+        format_action_syntax = QtGui.QWidgetAction(self.format_drop)
+        format_action_syntax.setDefaultWidget(self.format_syntax)
+
+        format_action_phrase = QtGui.QWidgetAction(self.format_drop)
+        format_action_phrase.setDefaultWidget(self.format_phrase)
+        
+        format_action_syntaxCube = QtGui.QWidgetAction(self.format_drop)
+        format_action_syntaxCube.setDefaultWidget(self.format_syntaxCube)
+        
+        format_action_phraseStackFlag = QtGui.QWidgetAction(self.format_drop)
+        format_action_phraseStackFlag.setDefaultWidget(self.format_phraseStackFlag)
+        
+        format_action_phraseStackVerbose = QtGui.QWidgetAction(self.format_drop)
+        format_action_phraseStackVerbose.setDefaultWidget(self.format_phraseStackVerbose)
+
+        format_action_syntaxCubeFlag = QtGui.QWidgetAction(self.format_drop)
+        format_action_syntaxCubeFlag.setDefaultWidget(self.format_syntaxCubeFlag)
+        
+        format_action_mbot = QtGui.QWidgetAction(self.format_drop)
+        format_action_mbot.setDefaultWidget(self.format_mbot)
+
+        self.format_drop.menu().addAction(format_action_syntax)
+        self.format_drop.menu().addAction(format_action_phrase)
+        self.format_drop.menu().addAction(format_action_syntaxCube)
+        self.format_drop.menu().addAction(format_action_phraseStackFlag)
+        self.format_drop.menu().addAction(format_action_phraseStackVerbose)
+        self.format_drop.menu().addAction(format_action_syntaxCubeFlag)
+        self.format_drop.menu().addAction(format_action_mbot)
+
+        
+        self.format_syntax.clicked.connect(self.set_format_syntax)
+        self.format_phrase.clicked.connect(self.set_format_phrase)
+        self.format_syntaxCube.clicked.connect(self.set_format_syntaxCube)
+        self.format_phraseStackFlag.clicked.connect(self.set_format_phraseStackFlag)
+        self.format_phraseStackVerbose.clicked.connect(self.set_format_phraseStackVerbose)
+        self.format_syntaxCubeFlag.clicked.connect(self.set_format_syntaxCubeFlag)
+        self.format_mbot.clicked.connect(self.set_format_mbot)
+
+
+
+        # table
+        self.table_widget = HoverTable(self)
+        self.w = []  # future popup window
+        # self.table_widget = QtGui.QTableWidget(self)
+
+        # lower buttons        
+        self.buttonBox = QtGui.QDialogButtonBox()
+        self.sentence_spinbox = QtGui.QSpinBox(parent=self.buttonBox)
+        self.sentence_spinbox.setMaximum(999999)
+
+        self.goto_button = self.buttonBox.addButton(
+            "&GoTo", QtGui.QDialogButtonBox.ActionRole)
+        self.next_button = self.buttonBox.addButton(
+            "&Next", QtGui.QDialogButtonBox.ActionRole)
+        self.prev_button = self.buttonBox.addButton(
+            "&Prev", QtGui.QDialogButtonBox.ActionRole)
+        self.next_button.clicked.connect(self.next_parse)
+        self.prev_button.clicked.connect(self.prev_parse)
+        self.goto_button.clicked.connect(self.cur_parse)
+        self.quit_button = self.buttonBox.addButton(
+            "&Quit", QtGui.QDialogButtonBox.ActionRole)
+        self.quit_button.clicked.connect(
+            QtCore.QCoreApplication.instance().quit)
+
+
+        
+        # Disable navigation buttons until data is loaded: see setPath for reactivation
+        self.goto_button.setDisabled(True)
+        self.next_button.setDisabled(True)
+        self.prev_button.setDisabled(True)
+        
+
+        
+
+
+        # Layouting
+
+        layout = QtGui.QVBoxLayout()
+
+        topLayout = QtGui.QHBoxLayout()
+        topLayout.addWidget(self.format_drop)
+        topLayout.addWidget(cell_limit_label)
+        topLayout.addWidget(self.cell_limit_chooser)
+        self.cell_limit_chooser.valueChanged.connect(self.setCellLimit)
+        topLayout.addWidget(pathLabel)
+        topLayout.addWidget(self.pathLabel, 1)
+        topLayout.addWidget(self.pathButton)
+
+        bottomLayout = QtGui.QHBoxLayout()
+        bottomLayout.addWidget(self.buttonBox)
+
+        layout.addLayout(topLayout)
+        layout.addWidget(self.table_widget)
+        layout.addLayout(bottomLayout)
+
+        self.sentence_spinbox.valueChanged.connect(self.set_cur_rein_num)
+
+        self.setLayout(layout)
+        self.updateSignal.connect(self.update_table)
+        
+        QtCore.QObject.connect(
+        self.table_widget,
+        QtCore.SIGNAL("cellDoubleClicked(int, int)"),
+        self.popup)
+        
+
+    def closeEvent(self, *args, **kwargs):
+        # reimplementation of the close-event for closing down everything 
+        # when the main window is closed
+        QtCore.QCoreApplication.quit()
+        return QtGui.QWidget.closeEvent(self, *args, **kwargs)
+    
+    
+    def setCellLimit(self, value):
+        if value == 0:
+            value = float("inf")
+        self.cell_limit = value
+            
+            
+    def setPath(self):
+        path = QtGui.QFileDialog.getOpenFileName(self,
+                "Select File", self.pathLabel.text())
+        if path:
+            self.goto_button.setDisabled(False)
+            self.prev_button.setDisabled(False)
+            self.next_button.setDisabled(False)
+            self.pathLabel.setText(QtCore.QDir.toNativeSeparators(path))
+            self.path = unicode(path)
+            self.data = my_DI.DataInput(self.path)
+            try:
+                if self.format == "syntax":
+                    self.data.read_syntax()
+                elif self.format == "phrase":
+                    self.data.read_phrase()
+                elif self.format == "syntaxCube":
+                    self.data.read_syntax_cubes(self.cell_limit)
+                elif self.format == "phraseStackFlag":
+                    self.data.read_phrase_stack_flag(self.cell_limit)
+                elif self.format == "phraseStackVerbose":
+                    self.data.read_phrase_stack_verbose(self.cell_limit)
+                elif self.format == "syntaxCubeFlag":
+                    self.data.read_syntax_cube_flag(self.cell_limit)
+                elif self.format == "mbot":
+                    self.data.read_mbot(self.cell_limit)
+                self.populate(0)
+                self.sentence_spinbox.setValue(0)
+            except (ValueError, IndexError) as exc:
+                self.error_dialog = QtGui.QDialog()
+                self.error_dialog.setModal(True)
+                layout = QtGui.QVBoxLayout()
+                text = QtGui.QLabel(
+                    """Something went wrong when choosing your input format/file
+                    \n""")
+                button = QtGui.QPushButton("Ok")
+                button.clicked.connect(self.error_dialog.close)
+                layout.addWidget(text)
+                layout.addWidget(button)
+                self.error_dialog.setLayout(layout)
+                self.error_dialog.show()
+            
+
+
+    def next_parse(self):
+        self.cur_rein_num += 1
+        if self.cur_rein_num < 0:
+            self.cur_rein_num = len(self.data.sentences) + self.cur_rein_num
+        if self.cur_rein_num >= len(self.data.sentences):
+            self.cur_rein_num = 0 
+        self.sentence_spinbox.setValue(self.cur_rein_num)
+        self.populate(self.cur_rein_num)
+            
+    def prev_parse(self):
+        self.cur_rein_num -= 1
+        if self.cur_rein_num < 0:
+            self.cur_rein_num = len(self.data.sentences) + self.cur_rein_num
+        if self.cur_rein_num >= len(self.data.sentences):
+            self.cur_rein_num = 0
+        self.sentence_spinbox.setValue(self.cur_rein_num)
+        self.populate(self.cur_rein_num)
+    
+    def cur_parse(self):
+        if self.cur_rein_num >= len(self.data.sentences):
+            self.cur_rein_num = 0
+        self.sentence_spinbox.setValue(self.cur_rein_num)
+        self.populate(self.cur_rein_num)
+        
+        
+    def set_cur_rein_num(self, value):
+        self.cur_rein_num = value  # self.sentence_spinbox.value()
+
+    def populate(self, cur_rein_num):
+        cur_sent = self.data.sentences[cur_rein_num]
+        nrows, ncols = cur_sent.length + 1, cur_sent.length + 1
+        nrows, ncols = ncols, nrows  # switcher
+        self.table_widget.setSortingEnabled(False)
+        self.table_widget.setRowCount(nrows)
+        self.table_widget.setColumnCount(ncols)
+        # for starting the numbering of the table at zero as the spans
+        self.table_widget.setHorizontalHeaderLabels([str(x) for x in range(ncols)])
+        self.table_widget.setVerticalHeaderLabels([str(x) for x in range(nrows)])
+        for i in range(nrows):
+            for j in range(ncols):
+                try:
+                    # item = TableItem("%s:%s \n %s"
+                    #                  % (i+1, j+1, cur_sent.spans[(i,j)]))
+                    item = str(i) + ".." + str(j) + "  \n"
+                    if isinstance(cur_sent.spans[(i, j)], basestring):
+                        item += cur_sent.spans[(i, j)] + "\n"
+                    else:
+                        for rule in cur_sent.spans[(i, j)]:
+                            item += str(rule) + "\n"
+                        if cur_sent.spans[(i, j)] == []:
+                            if j - i < 0:
+                                item = ""
+                            else:
+                                item = "-"
+                    item = TableItem(item.decode("utf-8"))
+                    
+
+                except KeyError:
+                    if j - i < 0:
+                        item = QtGui.QTableWidgetItem("")
+                    else:
+                        item = QtGui.QTableWidgetItem("-")
+                self.table_widget.setItem(i, j, item)
+                self.table_widget.setColumnWidth(j, 40)
+#                self.connect(
+#                    self.table_widget, QtCore.SIGNAL("itemDoubleClicked(QTableWidgetItem)"),
+#                    self.popup)
+
+        self.updateSignal.emit()
+        self.table_widget.setSortingEnabled(True)
+        
+    def update_table(self):
+        self.table_widget.sortItems(0, QtCore.Qt.DescendingOrder)
+
+
+
+
+    def set_format_syntax(self):
+        self.format = "syntax"
+        self.format_drop.setText("Syntax")
+        self.format_drop.menu().hide()
+
+    def set_format_phrase(self):
+        self.format = "phrase"
+        self.format_drop.setText("Phrase")
+        self.format_drop.menu().hide()
+        
+    def set_format_syntaxCube(self):
+        self.format = "syntaxCube"
+        self.format_drop.setText("Syntax Cube (-Tall flag)")
+        self.format_drop.menu().hide()
+        
+    def set_format_phraseStackFlag(self):
+        self.format = "phraseStackFlag"
+        self.format_drop.setText("Phrase Stack (search-graph)")
+        self.format_drop.menu().hide()
+        
+    def set_format_phraseStackVerbose(self):
+        self.format = "phraseStackVerbose"
+        self.format_drop.setText("Phrase Stack (verbose)")
+        self.format_drop.menu().hide()
+        
+    def set_format_syntaxCubeFlag(self):
+        self.format = "syntaxCubeFlag"
+        self.format_drop.setText("Syntax Cube (search-graph)")
+        self.format_drop.menu().hide()
+        
+    def set_format_mbot(self):
+        self.format = "mbot"
+        self.format_drop.setText("MBOT")
+        self.format_drop.menu().hide()
+        
+        
+#    @QtCore.pyqtSlot(QtGui.QTableWidgetItem, result=QtCore.QObject)
+#    def popup(self, item):
+#    @pyqtSlot(int, int, result=QtCore.QObject)
+#    @pyqtSignature("popup(int int)")
+    def popup(self, r, c):
+#        """ C++: QObject popup(int, int) """
+#        self.w = PopUpCell(item.text)
+        self.w.append(PopUpCell(self.table_widget.item(r, c).text()))
+        # self.w.setGeometry(QRect(100, 100, 400, 200))
+        self.w[-1].show()
+        
+
+class HoverTable(QtGui.QTableWidget):
+    
+    def __init__(self, parent=None):
+        super(HoverTable, self).__init__(parent)
+        self.setMouseTracking(True)
+        self.horizontalHeader().setClickable(False)
+#        self.verticalHeader().setDefaultSectionSize(self.verticalHeader.fontMetrics().height()+2);
+        
+
+
+class PopUpCell(QtGui.QWidget):
+    def __init__(self, cell_text):
+        QtGui.QWidget.__init__(self)
+        layout = QtGui.QHBoxLayout()
+        text_list = map(lambda x: x, cell_text.split("\n"))
+        wind_cont = QtGui.QTextEdit()  # "<br/>".join(text_list[1:]))
+        wind_cont.setReadOnly(True)
+        wind_cont.setWindowTitle(text_list[0])
+        wind_cont.setPlainText(cell_text)  # "\n".join(text_list))
+        layout.addWidget(wind_cont)
+        self.setWindowTitle(text_list[0])
+        self.setLayout(layout)
+        self.resize(960, 320)
+
+
+    
+
+        
+class TableItem(QtGui.QTableWidgetItem):
+    
+    def __init__(self, cell_text, type=1000):
+        super(TableItem, self).__init__(cell_text)
+        if len(cell_text.split("\n")) > 20:
+            self.setToolTip("\n".join(cell_text.split("\n")[:19]))
+        else:
+            self.setToolTip(cell_text)
+        self.cell_text = cell_text
+
+

mosesdecoder/contrib/DIMwid/DIMwid.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import sys
+from PyQt4 import QtCore, QtGui
+
+import DIMterface as my_gui
+
+
+
+if __name__ == "__main__":
+    app = QtGui.QApplication(sys.argv)
+    wnd = my_gui.MainWindow()
+    wnd.resize(640, 480)
+    wnd.setWindowTitle("DIMwid")
+    wnd.show()
+    sys.exit(app.exec_())

mosesdecoder/contrib/DIMwid/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 RobinQrtz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

mosesdecoder/contrib/DIMwid/README.md

@@ -0,0 +1,67 @@
+DIMwid
+ ======
+
+DIMwid (Decoder Inspection for Moses using widgets) is a tool
+presenting Moses' different chart/stack outputs in a readable tabular
+view.
+
+
+Installation
+============
+
+In order to run DIMwid you need to install PyQt, Qt 4.8 and Python
+2.7. Other versions have not yet been tested.  Linux/Unix users simply
+install these packages using their package-manager or built them from
+source.  Windows can skip the installation of Qt since PyQt itself
+does cover everything, except Python.
+
+Usage
+=====
+
+Users are recommended to read the accompanying paper "DIMwid --
+Decoder Inspection for Moses (using Widgets)" appearing in PBML XY.
+
+DIMwid is able to read multiple decoder outputs of the Moses
+translation system. These include the standard trace outputs for both
+phrase- and syntax-based decoding, the search-graphs for both, the
+"level 3 verbose" output for phrase-based and a special trace output
+(available as a Moses fork at :
+https://github.com/RobinQrtz/mosesdecoder) for all possible
+translations for syntax-based decoding.
+
+After producing the outputs from Moses, start DIMwid by running
+DIMwid.py and first select your format and after that your file. If
+you have chosen the wrong file or format an error message will
+appear. Otherwise you will see the first sentence. Cells can be
+inspected by either double-clicking, opening a new window with the
+full content, or hovering over the cell, showing a tooltip with the
+first 20 lines of the cell's content.
+
+If needed, the user can restrict the number of rules per cell, using
+the "Cell Limit" spinbox.
+
+Navigating through the sentences of the input file can be done by
+either using the "Next" and "Prev" buttons, or choosing a certain
+sentence number using the lower left spinbox and clicking the "GoTo"
+button.
+
+Moses
+=====
+
+Information about Moses can be found here: http://statmt.org/moses/
+
+The used flags for the output are:
+    * -t for phrase-based trace
+    * -T for syntax-based trace
+    * -v 3 for phrase-based verbose level 3
+    * -output-search-graph for both search graphs
+    * -Tall for the Moses fork's new feature
+
+
+Trouble
+=======
+
+If you are running into trouble using DIMwid or have suggestions for
+improvements or new features email me at 
+
+robin DOT qrtz AT gmail DOT com

mosesdecoder/contrib/Makefiles/install-dependencies.gmake

@@ -0,0 +1,103 @@
+# -*- mode: makefile; tab-width: 4; -*-
+# Makefile for installing 3rd-party software required to build Moses.
+# author: Ulrich Germann
+#
+# run as
+#    make -f /path/to/this/file
+#
+# By default, everything will be installed in ./opt.
+# If you want an alternative destination specify PREFIX=... with the make call
+#
+#    make -f /path/to/this/file PREFIX=/where/to/install/things
+#
+# The name of the current directory must not contain spaces! The build scripts for
+# at least some of the external software can't handle them.
+
+space :=
+space +=
+# $(CWD) may contain space, safepath escapes them
+# Update: doesn't work, because the build scripts for some of the external packages
+# can't handle spaces in path names.
+safepath=$(subst $(space),\$(space),$1)
+
+# current working directory: bit of a hack to get the nfs-accessible
+# path instead of the local real path
+CWD := $(shell cd . && pwd)
+
+# by default, we install in ./opt and build in ./build
+PREFIX ?= $(CWD)/opt
+BUILD_DIR = $(CWD)/opt/build/${URL}
+
+# you can also specify specific prefixes for different packages:
+XMLRPC_PREFIX ?= ${PREFIX}
+CMPH_PREFIX   ?= ${PREFIX}
+IRSTLM_PREFIX ?= ${PREFIX}/irstlm-5.80.08
+BOOST_PREFIX  ?= ${PREFIX}
+
+# currently, the full enchilada means xmlrpc-c, cmph, irstlm, boost
+all: xmlrpc cmph boost
+
+# we use bash and fail when pipelines fail
+SHELL = /bin/bash -e -o pipefail
+
+
+
+# evaluate prefixes now to avoid recursive evaluation problems later ...
+XMLRPC_PREFIX := ${XMLRPC_PREFIX}
+CMPH_PREFIX   := ${CMPH_PREFIX}
+IRSTLM_PREFIX := ${IRSTLM_PREFIX}
+BOOST_PREFIX  := ${BOOST_PREFIX}
+
+# Code repositories:
+github      = https://github.com/
+sourceforge = http://downloads.sourceforge.net/project
+
+# functions for building software from sourceforge
+nproc := $(shell getconf _NPROCESSORS_ONLN)
+sfget  = mkdir -p '${TMP}' && cd '${TMP}' && wget -qO- ${URL} | tar xz
+configure-make-install  = cd '$1' && ./configure --prefix='${PREFIX}'
+configure-make-install += && make -j${nproc} && make install
+
+# XMLRPC-C for moses server
+xmlrpc: URL=$(sourceforge)/xmlrpc-c/Xmlrpc-c%20Super%20Stable/1.33.17/xmlrpc-c-1.33.17.tgz
+xmlrpc: TMP=$(CWD)/build/xmlrpc
+xmlrpc: override PREFIX=${XMLRPC_PREFIX}
+xmlrpc: | $(call safepath,${XMLRPC_PREFIX}/bin/xmlrpc-c-config)
+$(call safepath,${XMLRPC_PREFIX}/bin/xmlrpc-c-config):
+	$(sfget)
+	$(call configure-make-install,${TMP}/xmlrpc-c-1.33.17)
+	rm -rf ${TMP}
+
+# CMPH for CompactPT
+cmph: URL=$(sourceforge)/cmph/cmph/cmph-2.0.tar.gz
+cmph: TMP=$(CWD)/build/cmph
+cmph: override PREFIX=${CMPH_PREFIX}
+cmph: | $(call safepath,${CMPH_PREFIX}/bin/cmph)
+$(call safepath,${CMPH_PREFIX}/bin/cmph):
+	$(sfget)
+	$(call configure-make-install,${TMP}/cmph-2.0)
+	rm -rf ${TMP}
+
+# irstlm for irstlm
+irstlm: URL=$(sourceforge)/irstlm/irstlm/irstlm-5.80/irstlm-5.80.08.tgz
+irstlm: TMP=$(CWD)/build/irstlm
+irstlm: VERSION=$(basename $(notdir $(irstlm_url)))
+irstlm: override PREFIX=${IRSTLM_PREFIX}
+irstlm: | $(call safepath,$(IRSTLM_PREFIX)/bin/build-lm.sh)
+$(call safepath,$(IRSTLM_PREFIX)/bin/build-lm.sh):
+	$(sfget)
+	cd $$(find '${TMP}' -name trunk) && ./regenerate-makefiles.sh \
+	&& ./configure --prefix='${PREFIX}' && make -j${nproc} && make install -j${nproc}
+	rm -rf ${TMP}
+
+# boost
+boost: VERSION=1.68.0
+boost: UNDERSCORED=$(subst .,_,$(VERSION))
+boost: URL=http://sourceforge.net/projects/boost/files/boost/${VERSION}/boost_${UNDERSCORED}.tar.gz/download
+boost: TMP=$(CWD)/build/boost
+boost: override PREFIX=${BOOST_PREFIX}
+boost: | $(call safepath,${BOOST_PREFIX}/include/boost)
+$(call safepath,${BOOST_PREFIX}/include/boost):
+	$(sfget)
+	cd '${TMP}/boost_${UNDERSCORED}' && ./bootstrap.sh && ./b2 --prefix=${PREFIX} -j${nproc} --layout=system link=static install
+	rm -rf ${TMP}

mosesdecoder/contrib/arrow-pipelines/README

@@ -0,0 +1,58 @@
+Arrow Based Moses Training Pipeline
+===================================
+
+This demonstration implements a training pipeline that is shown in the Dia diagram in documentation/training-pipeline/moses-pypeline.dia.
+
+The demo has been tested with:
+
+ - Moses v1.0
+ - Giza++ v1.0.7
+ - IRSTLM v5.70.04
+
+
+Setup
+-----
+
+To use the demonstration you must first initialise the git submodules for this clone. Return to the top level directory and issue the following command:
+
+$ git submodule update --init --recursive
+
+This will clone PCL, available at Github (git://github.com/ianj-als/pcl.git), and Pypeline submodules, available at GitHub (git://github.com/ianj-als/pypeline.git).
+
+Return to the arrow-pipelines contrib directory:
+
+$ cd contrib/arrow-pipelines
+
+To use the PCL compiler and run-time set the following environment variables (assuming Bash shell):
+
+$ export PATH=$PATH:`pwd`/python/pcl/src/pclc:`pwd`/python/pcl/src/pcl-run
+$ export PYTHONPATH=$PYTHONPATH:`pwd`/python/pcl/libs/pypeline/src
+$ export PCL_IMPORT_PATH=`pwd`/python/pcl/src/runtime:`pwd`/pcl
+
+Three environment variables need to be set before the pipeline can be run, they are:
+
+ - MOSES_HOME : The directory where Moses has been cloned, or installed,
+ - IRSTLM : The installation directory of your IRSTLM, and
+ - GIZA_HOME : The installation directory of GIZA++.
+
+
+Building the example training pipeline
+--------------------------------------
+
+$ cd pcl
+$ make
+
+
+Running the example training pipeline
+-------------------------------------
+
+To execute the training pipeline run the following command:
+
+$ pcl-run.py training_pipeline
+
+Once complete the output of the pipeline can be found in the directories:
+
+ - training/tokenisation
+ - training/model
+ - training/lm
+ - training/mert

mosesdecoder/contrib/arrow-pipelines/bash/training_pipeline.sh

@@ -0,0 +1,226 @@
+#!/bin/bash
+
+MOSES_HOME=/opt/moses
+GIZA_HOME=${MOSES_HOME}/giza++-v1.0.7
+IRSTLM=${MOSES_HOME}/irstlm-5.70.04
+
+function tokenise() {
+    local LANG="$1"
+    local FILENAME="$2"
+    local WORKING_DIR="$3"
+    local BASENAME="`basename ${FILENAME}`"
+
+    if [ ! -f ${WORKING_DIR} ]; then
+	mkdir -p ${WORKING_DIR}
+    fi
+
+    NEW_BASENAME=`echo ${BASENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "tok."; } } }'`
+
+    TOKENISED_FILENAME="${WORKING_DIR}/${NEW_BASENAME}"
+    ${MOSES_HOME}/scripts/tokenizer/tokenizer.perl -q -l ${LANG} < ${FILENAME} > ${TOKENISED_FILENAME}
+}
+
+function cleanup() {
+    local SRC_FILENAME="$1"
+    local TGT_FILENAME="$2"
+    local SEGMENT_LENGTH="$3"
+    SRC_CLEANUP_FILENAME=`echo ${SRC_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "clean."; } } }'`
+    TGT_CLEANUP_FILENAME=`echo ${TGT_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "clean."; } } }'`
+
+    truncate -s 0 ${SRC_CLEANUP_FILENAME}
+    truncate -s 0 ${TGT_CLEANUP_FILENAME}
+
+    paste -d'\n' ${SRC_FILENAME} ${TGT_FILENAME} | while read SRC_LINE && read TGT_LINE;
+    do
+      declare -i SRC_NO_WORDS=`echo "${SRC_LINE}" | wc -w`
+      declare -i TGT_NO_WORDS=`echo "${TGT_LINE}" | wc -w`
+      if [ ${SRC_NO_WORDS} -lt 20 -a ${TGT_NO_WORDS} -lt 20 ]; then
+	  echo "${SRC_LINE}" >> ${SRC_CLEANUP_FILENAME}
+	  echo "${TGT_LINE}" >> ${TGT_CLEANUP_FILENAME}
+      fi
+    done
+}
+
+function data_split() {
+    local SRC_FILENAME="$1"
+    local TGT_FILENAME="$2"
+    declare -i DEV_SIZE="$3"
+    declare -i EVAL_SIZE="$4"
+
+    SRC_TRAIN_FILENAME=`echo ${SRC_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "train."; } } }'`
+    TGT_TRAIN_FILENAME=`echo ${TGT_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "train."; } } }'`
+    SRC_DEVEL_FILENAME=`echo ${SRC_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "devel."; } } }'`
+    TGT_DEVEL_FILENAME=`echo ${TGT_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "devel."; } } }'`
+    SRC_EVAL_FILENAME=`echo ${SRC_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "eval."; } } }'`
+    TGT_EVAL_FILENAME=`echo ${TGT_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) { printf a[i]; if (i<length(a)) { printf "."; } if (i==length(a)-1) { printf "eval."; } } }'`
+
+    local ALL_FILES=(${SRC_TRAIN_FILENAME} ${TGT_TRAIN_FILENAME} ${SRC_DEVEL_FILENAME} ${TGT_DEVEL_FILENAME} ${SRC_EVAL_FILENAME} ${TGT_EVAL_FILENAME})
+    for FN in ${ALL_FILES}
+    do
+      truncate -s 0 ${FN}
+    done
+
+    declare -i DEV_EVAL_SIZE=$(($DEV_SIZE + $EVAL_SIZE))
+    declare -i LINE_CNT=1
+    paste -d'\n' ${SRC_FILENAME} ${TGT_FILENAME} | while read SRC_LINE && read TGT_LINE;
+    do
+      if [ ${LINE_CNT} -le ${DEV_EVAL_SIZE} ]; then
+	  if [ ${LINE_CNT} -le ${DEV_SIZE} ]; then
+	      echo "${SRC_LINE}" >> ${SRC_DEVEL_FILENAME}
+	      echo "${TGT_LINE}" >> ${TGT_DEVEL_FILENAME}
+	  else
+	      echo "${SRC_LINE}" >> ${SRC_EVAL_FILENAME}
+	      echo "${TGT_LINE}" >> ${TGT_EVAL_FILENAME}
+	  fi
+      else
+	  echo "${SRC_LINE}" >> ${SRC_TRAIN_FILENAME}
+	  echo "${TGT_LINE}" >> ${TGT_TRAIN_FILENAME}
+      fi
+      LINE_CNT=$(($LINE_CNT + 1))
+    done
+}
+
+function translation_model_train() {
+    declare -l TT_SRC_LANG="$1"
+    declare -l TT_TGT_LANG="$2"
+    local SRC_FILENAME="`realpath $3`"
+    local TGT_FILENAME="`realpath $4`"
+    local ALIGNMENT_METHOD="$5"
+    local REORDERING_METHOD="$6"
+    local WORKING_DIR="$7"
+
+    declare -r SRC_CORPORA_NAME=`echo ${SRC_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i < length(a); i++) { printf a[i]; if (i < length(a) - 1) { printf "."; } } }'`
+    declare -r TGT_CORPORA_NAME=`echo ${TGT_FILENAME} | gawk '{split($0, a, "."); for(i = 1; i < length(a); i++) { printf a[i]; if (i < length(a) - 1) { printf "."; } } }'`
+
+    if [ "${SRC_CORPORA_NAME}" != "${TGT_CORPORA_NAME}" ]; then
+	echo "Arrrgh"
+	exit 1
+    fi
+
+    if [ -f ${WORKING_DIR} ]; then
+	rm -Rf ${WORKING_DIR} >& /dev/null
+    fi
+    mkdir -p ${WORKING_DIR}
+    WORKING_DIR=`realpath ${WORKING_DIR}`
+
+    declare -r DUMMY_FILE="${WORKING_DIR}/dummy.lm"
+    echo "dummy lm file" > ${DUMMY_FILE}
+
+    declare -r LOG_FILE="${WORKING_DIR}/log"
+
+    ${MOSES_HOME}/scripts/training/train-model.perl -root-dir ${WORKING_DIR} -corpus ${SRC_CORPORA_NAME} -f ${TT_SRC_LANG} -e ${TT_TGT_LANG} -alignment ${ALIGNMENT_METHOD} -reordering ${REORDERING_METHOD} -lm 0:5:${DUMMY_FILE}:0 -external-bin-dir ${GIZA_HOME} 2> ${LOG_FILE}
+
+    MOSES_INI_FILE="${WORKING_DIR}/model/moses.ini"
+}
+
+function language_model_train() {
+    local FILENAME="$1"
+    local SMOOTHING_METHOD="$2"
+    local WORKING_DIR="$3"
+
+    if [ ! -f ${WORKING_DIR} ]; then
+	mkdir -p ${WORKING_DIR}
+    fi
+
+    declare -r BASENAME=`basename ${FILENAME}`
+    declare -r START_END_OUTPUT_FILENAME=${WORKING_DIR}/`echo ${BASENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) {if(i == 3) { printf "sb."; } else { printf a[i]; if (i < length(a) - 1) { printf "."; } } } }'`
+    declare -r LM_FILENAME=${WORKING_DIR}/`echo ${BASENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) {if(i == 3) { printf "lm."; } else { printf a[i]; if (i < length(a) - 1) { printf "."; } } } }'`
+    COMPILED_LM_FILENAME=${WORKING_DIR}/`echo ${BASENAME} | gawk '{split($0, a, "."); for(i = 1; i <= length(a); i++) {if(i == 3) { printf "arpa."; } else { printf a[i]; if (i < length(a) - 1) { printf "."; } } } }'`
+
+    export IRSTLM
+
+    ${IRSTLM}/bin/add-start-end.sh < ${FILENAME} > ${START_END_OUTPUT_FILENAME}
+    
+    declare -r TMP_DIR=`mktemp -dp /tmp`
+    ${IRSTLM}/bin/build-lm.sh -i ${START_END_OUTPUT_FILENAME} -t ${TMP_DIR} -p -s ${SMOOTHING_METHOD} -o ${LM_FILENAME}
+    if [ -f ${TMP_DIR} ]; then
+	rm -Rf ${TMP_DIR} >& /dev/null
+    fi
+
+    ${IRSTLM}/bin/compile-lm --text yes ${LM_FILENAME}.gz ${COMPILED_LM_FILENAME}
+}
+
+function mert() {
+    local MOSES_INI_FILENAME="`realpath $1`"
+    local COMPILED_LM_FILENAME="`realpath $2`"
+    local EVAL_FILENAME="$3"
+    declare -lr _SRC_LANG="$4"
+    declare -lr _TGT_LANG="$5"
+    declare -ri MODEL_ORDER="$6"
+    declare -ri MODEL_TYPE="$7"
+    local WORKING_DIR="$8"
+    declare -ri MAX_NO_ITERS="$9"
+
+    local INFILENAME=`realpath ${EVAL_FILENAME}`
+    INFILENAME=`echo ${INFILENAME} | gawk '{split($0, a, "."); for(i = 1; i < length(a); i++) { printf a[i]; if (i < length(a) - 1) { printf "."; } } }'`
+
+    if [ ! -f ${MOSES_INI_FILENAME} ]; then
+	echo "${MOSES_INI_FILENAME} does not exist."
+	exit 1
+    fi
+
+    if [ -f ${WORKING_DIR} ]; then
+	rm -Rf ${WORKING_DIR} >& /dev/null
+    fi
+    mkdir -p ${WORKING_DIR}
+
+    WORKING_DIR=`realpath ${WORKING_DIR}`
+    MERT_INI_FILENAME="${WORKING_DIR}/trained-moses.ini"
+    local SED_PROG="/\[lmodel-file\]/,/^[[:space:]]*\$/c\[lmodel-file\]\n${MODEL_TYPE} 0 ${MODEL_ORDER} ${COMPILED_LM_FILENAME}\n"
+    eval cat ${MOSES_INI_FILENAME} | sed "${SED_PROG}" > ${MERT_INI_FILENAME}
+
+    ${MOSES_HOME}/scripts/training/mert-moses.pl --maximum-iterations ${MAX_NO_ITERS} --mertdir ${MOSES_HOME}/bin --working-dir ${WORKING_DIR} ${INFILENAME}.${_SRC_LANG} ${INFILENAME}.${_TGT_LANG} ${MOSES_HOME}/bin/moses ${MERT_INI_FILENAME} 2> ${WORKING_DIR}/log
+}
+
+
+if [ $# -lt 4 ]; then
+   echo "`basename $0` usage:"
+   echo "  `basename $0` src_file tgt_file src_lang tgt_lang"
+   echo
+   exit 1
+fi
+
+declare -r SRC_LANG="$3"
+declare -r TGT_LANG="$4"
+
+# Tokenise
+tokenise "${SRC_LANG}" "$1" "training/tokeniser"
+declare -r SRC_TOKENISED_FILENAME="${TOKENISED_FILENAME}"
+
+tokenise "${TGT_LANG}" "$2" "training/tokeniser"
+declare -r TGT_TOKENISED_FILENAME="${TOKENISED_FILENAME}"
+
+echo ${SRC_TOKENISED_FILENAME}
+echo ${TGT_TOKENISED_FILENAME}
+
+# Cleanup
+cleanup "${SRC_TOKENISED_FILENAME}" "${TGT_TOKENISED_FILENAME}" 20
+
+echo ${SRC_CLEANUP_FILENAME}
+echo ${TGT_CLEANUP_FILENAME}
+
+# Data split: src, tgt, dev size, eval size
+data_split "${SRC_CLEANUP_FILENAME}" "${TGT_CLEANUP_FILENAME}" 1000 500
+
+echo ${SRC_TRAIN_FILENAME}
+echo ${TGT_TRAIN_FILENAME}
+echo ${SRC_DEVEL_FILENAME}
+echo ${TGT_DEVEL_FILENAME}
+echo ${SRC_EVAL_FILENAME}
+echo ${TGT_EVAL_FILENAME}
+
+# Train the translation model
+translation_model_train "${SRC_LANG}" "${TGT_LANG}" "${SRC_DEVEL_FILENAME}" "${TGT_DEVEL_FILENAME}" "grow-diag-final-and" "msd-bidirectional-fe" "training/model"
+
+declare -r MOSES_TT_INI_FILENAME="${MOSES_INI_FILE}"
+echo ${MOSES_TT_INI_FILENAME}
+
+# Language model training
+language_model_train "${TGT_TOKENISED_FILENAME}" "improved-kneser-ney" "training/lm"
+
+echo ${COMPILED_LM_FILENAME}
+
+# MERT
+mert "${MOSES_TT_INI_FILENAME}" "${COMPILED_LM_FILENAME}" "${SRC_EVAL_FILENAME}" "${SRC_LANG}" "${TGT_LANG}" 3 9 "training/mert" 1
+
+echo ${MERT_INI_FILENAME}

mosesdecoder/contrib/arrow-pipelines/pcl/Makefile

@@ -0,0 +1,23 @@
+CC = pclc.py
+CFLAGS=-i
+SOURCES = training_pipeline.pcl
+OBJS = $(SOURCES:.pcl=.py)
+SUBDIRS = components
+
+all: subdirs build
+
+build: $(OBJS)
+
+%.py: %.pcl
+	$(CC) $(CFLAGS) $<
+
+clean:
+	for dir in $(SUBDIRS); do \
+		$(MAKE) -C $$dir clean; \
+	done
+	rm -f *.py *.pyc *.log *~
+
+subdirs:
+	for dir in $(SUBDIRS); do \
+		$(MAKE) -C $$dir ; \
+	done

mosesdecoder/contrib/arrow-pipelines/pcl/components/Makefile

@@ -0,0 +1,24 @@
+CC = pclc.py
+CFLAGS = -i
+SOURCES = src_trg_tokeniser.pcl translation_model_training.pcl
+OBJS = $(SOURCES:.pcl=.py)
+SUBDIRS = wrappers
+
+all: subdirs build
+
+build: $(OBJS)
+
+%.py: %.pcl
+	$(CC) $(CFLAGS) $<
+
+clean:
+	for dir in $(SUBDIRS); do \
+    $(MAKE) -C $$dir clean; \
+  done
+	rm -f *.py *.pyc *.log *~
+
+subdirs:
+	for dir in $(SUBDIRS); do \
+    $(MAKE) -C $$dir ; \
+  done
+

mosesdecoder/contrib/arrow-pipelines/pcl/components/src_trg_tokeniser.cfg

@@ -0,0 +1,10 @@
+[Configuration]
+tokeniser.src.language = en
+tokeniser.src.tokenisation_dir = test_data/src_trg_tokenizer/tokenised
+tokeniser.trg.language = lt
+tokeniser.trg.tokenisation_dir = test_data/src_trg_tokenizer/tokenised
+tokeniser.moses.installation = /opt/moses
+
+[Inputs]
+src_filename = test_data/src_trg_tokenizer/cleantrain.en
+trg_filename = test_data/src_trg_tokenizer/cleantrain.lt

mosesdecoder/contrib/arrow-pipelines/pcl/components/src_trg_tokeniser.pcl

@@ -0,0 +1,40 @@
+#
+# Import all of the components to be composed
+#
+import wrappers.tokenizer.tokenizer as tokeniser
+
+#
+# Component definition
+#
+#                 +---------+                +---------+                          +---------+                          +---------+
+# src_filename -->+         +--> filename -->+-- src --+--> tokenised_filename -->+---------+--> tokenised_filename -->+         +--> tokenised_src_filename
+#                 |         |                |         |                          |         |                          |         |
+# trg_filename -->+         +--> filename -->+---------+-------> filename ------->+-- trg --+--> tokenised_filename -->+         +--> tokenised_trg_filename
+#                 +---------+                +---------+                          +---------+                          +---------+
+#                                  Config: {language::String,              Config: {language::String,
+#                                           tokenisation_dir::String,               tokenisation_dir::String,
+#                                           moses_installation_dir::String}         moses_installation_dir::String}
+#
+component src_trg_tokeniser
+  inputs (src_filename), (trg_filename)
+  outputs (tokenised_src_filename), (tokenised_trg_filename)
+  configuration tokeniser.src.language,
+                tokeniser.src.tokenisation_dir,
+                tokeniser.trg.language,
+                tokeniser.trg.tokenisation_dir,
+                tokeniser.moses.installation
+  declare
+    src_tokeniser := new tokeniser with
+      tokeniser.src.language -> corpus.language,
+      tokeniser.src.tokenisation_dir -> working.directory.root,
+      tokeniser.moses.installation -> moses.installation
+    trg_tokeniser := new tokeniser with
+      tokeniser.trg.language -> corpus.language,
+      tokeniser.trg.tokenisation_dir -> working.directory.root,
+      tokeniser.moses.installation -> moses.installation
+  as
+    wire (src_filename -> corpus.filename),
+         (trg_filename -> corpus.filename) >>>
+    (src_tokeniser *** trg_tokeniser) >>>
+    wire (corpus.tokenised.filename -> tokenised_src_filename),
+         (corpus.tokenised.filename -> tokenised_trg_filename)

mosesdecoder/contrib/arrow-pipelines/pcl/components/translation_model_training.cfg

@@ -0,0 +1,15 @@
+[Configuration]
+model_training.max_segment_length = 20
+model_training.corpus.development_size = 4500
+model_training.corpus.evaluation_size = 5000
+model_training.src.language = en
+model_training.trg.language = lt
+model_training.method.alignment = grow-diag-final-and
+model_training.method.reordering = msd-bidirectional-fe
+model_training.moses.installation = /opt/moses
+model_training.giza.installation = /opt/moses/giza++-v1.0.7
+model_training.translation_model.dir = test_data/translation_model_training/translation_model
+
+[Inputs]
+src_filename = test_data/translation_model_training/cleantrain.en
+trg_filename = test_data/translation_model_training/cleantrain.lt

mosesdecoder/contrib/arrow-pipelines/pcl/components/translation_model_training.pcl

@@ -0,0 +1,70 @@
+#
+# Import all of the components to be composed
+#
+import wrappers.cleanup.cleanup as cleanup
+import wrappers.data_split.data_split as data_split
+import wrappers.model_training.model_training as model_training
+
+#
+# Component definition
+#
+#        {cleaned_src_filename, {src_filename, {[devel|eval|train]_src_filename, {src_filename,    {moses_ini_file,
+#         cleaned_trg_filename}  trg_filename}  [devel|eval|train]_trg_filename}  trg_filename}     evaluation_data_filename}
+#                             |  |              |                                 | +-------+                              |
+#                   +-------+ |  | +-------+    |         +-------+               V | Model | {moses_ini_file} +-------+   V
+#                   | Clean | V  V | Data  |    V         |       +---------------->+ Train +----------------->+ Merge +----->
+# {src_filename, -->+       +----->+       +------------->+ Split |                 +-------+                  +---+---+
+#  trg_filename}    |  Up   |      | Split |              |       +---\  Config: {[src|trg]_language::String,      ^
+#                   +-------+      +-------+              +-------+   |           alignment_method::String,        |
+# Config: {segment_length::Int} Config: {development_size::Int,       |           reordering_method::String,       |
+#                                        evaluation_size::Int}        |           giza_installation_dir::String,   |
+#                                                                     |           model_directory::String}         |
+#                                                                     \--------------------------------------------/
+#
+component translation_model_training
+  inputs src_filename, trg_filename
+  outputs evaluation_data_filename, moses_ini_filename
+  configuration model_training.max_segment_length,
+                model_training.corpus.development_size,
+                model_training.corpus.evaluation_size,
+                model_training.src.language,
+                model_training.trg.language,
+                model_training.method.alignment,
+                model_training.method.reordering,
+                model_training.moses.installation,
+                model_training.giza.installation,
+                model_training.translation_model.dir
+  declare
+    cleanup := new cleanup with
+      model_training.max_segment_length -> segment_length_limit
+    data_split := new data_split with
+      model_training.corpus.development_size -> development_data_size,
+      model_training.corpus.evaluation_size -> evaluation_data_size
+    model_training := new model_training with
+      model_training.src.language -> source_language,
+      model_training.trg.language -> target_language,
+      model_training.method.alignment -> alignment_method,
+      model_training.method.reordering -> reordering_method,
+      model_training.moses.installation -> moses_installation_dir,
+      model_training.giza.installation -> giza_installation_dir,
+      model_training.translation_model.dir -> translation_model_directory
+  as
+    cleanup >>>
+    wire cleaned_src_filename -> src_filename,
+         cleaned_trg_filename -> trg_filename >>>
+    data_split >>>
+    wire devel_src_filename -> devel_src_filename,
+         eval_src_filename -> evaluation_data_filename,
+         train_trg_filename -> _,
+         train_src_filename -> _,
+         eval_trg_filename -> _,
+         devel_trg_filename -> devel_trg_filename >>>
+    ((wire devel_src_filename -> src_filename,
+           devel_trg_filename -> trg_filename,
+           evaluation_data_filename -> _ >>>
+      model_training) &&&
+     wire evaluation_data_filename -> evaluation_data_filename,
+          devel_src_filename -> _,
+          devel_trg_filename -> _) >>>
+    merge top[moses_ini_filename] -> moses_ini_filename,
+          bottom[evaluation_data_filename] -> evaluation_data_filename

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/Makefile

@@ -0,0 +1,14 @@
+SUBDIRS = tokenizer
+
+all: subdirs
+
+clean:
+	for dir in $(SUBDIRS); do \
+    $(MAKE) -C $$dir clean; \
+  done
+
+subdirs:
+	for dir in $(SUBDIRS); do \
+    $(MAKE) -C $$dir ; \
+  done
+

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/cleanup/cleanup.py

@@ -0,0 +1,129 @@
+def get_name():
+    return 'cleanup'
+
+def get_inputs():
+    return ['src_filename', 'trg_filename']
+
+def get_outputs():
+    return ['cleaned_src_filename', 'cleaned_trg_filename']
+
+def get_configuration():
+    return ['segment_length_limit']
+
+def configure(args):
+    return {'segment_length' : args['segment_length_limit']}
+
+def initialise(config):
+    def _filter(limit, ifh1, ofh1, ifh2, ofh2):
+        def _short(line):
+            n = 0
+            for c in line:
+                if c == " ":
+                    n += 1
+            return n < limit
+
+        for (l1, l2) in zip(ifh1, ifh2):
+            if _short(l1) and _short(l2):
+                print >>ofh1, l1,
+                print >>ofh2, l2,
+
+    def _make_cleaned_filename(filename):
+        bits = filename.split(".")
+        bits.insert(-1, "clean")
+        return ".".join(bits)
+
+    def _filter_main(a, s):
+        limit = config['segment_length']
+        (ifh1, ifh2, ofh1, ofh2) = (None, None, None, None)
+        try:
+            input_src_filename = a['src_filename']
+            input_trg_filename = a['trg_filename']
+
+            print "Cleanup: Cleaning [%s] and [%s]..." % (input_src_filename, input_trg_filename)
+
+            ifh1 = open(input_src_filename, "r")
+            ifh2 = open(input_trg_filename, "r")
+
+            cleaned_src_filename = _make_cleaned_filename(input_src_filename)
+            cleaned_trg_filename = _make_cleaned_filename(input_trg_filename)
+            ofh1 = open(cleaned_src_filename, "w")
+            ofh2 = open(cleaned_trg_filename, "w")
+
+            _filter(limit, ifh1, ofh1, ifh2, ofh2)
+
+            return {'cleaned_src_filename': cleaned_src_filename,
+                    'cleaned_trg_filename': cleaned_trg_filename}
+        finally:
+            def _safe_close(fh):
+                if fh is not None:
+                    fh.close()
+                _safe_close(ifh1)
+                _safe_close(ifh2)
+                _safe_close(ofh1)
+                _safe_close(ofh2)
+    
+    return _filter_main
+
+
+if __name__ == '__main__':
+    import os
+    import tempfile
+    import test.test as thelp
+
+    from pypeline.helpers.helpers import eval_pipeline
+
+
+    def _test_main():
+        configuration = {'segment_length_limit': 20}
+
+        src_filename = tempfile.mkstemp(suffix = ".src", dir = "/tmp")
+        trg_filename = tempfile.mkstemp(suffix = ".trg", dir = "/tmp")
+
+        box_eval = {
+            'src_filename': src_filename[1],
+            'trg_filename': trg_filename[1],
+            'cleaned_src_file_expected': src_filename[1] + ".expected",
+            'cleaned_trg_file_expected': trg_filename[1] + ".expected"}
+
+        try:
+            _prep_files(box_eval)
+            _run_test(configuration, box_eval)
+        finally:
+            _cleanup_files(box_eval)
+
+
+    def _run_test(configuration, box_eval):
+        box_config = configure(configuration)
+        box = initialise(box_config)
+    
+        output = eval_pipeline(box, box_eval, box_config)
+        try:
+            thelp.diff(box_eval['cleaned_src_file_expected'], output['cleaned_src_filename'])
+            thelp.diff(box_eval['cleaned_trg_file_expected'], output['cleaned_trg_filename'])
+        finally:
+            os.unlink(output['cleaned_src_filename'])
+            os.unlink(output['cleaned_trg_filename'])
+
+
+    def _line(line_lengths):
+        def _gen_line(tokens):
+            return " ".join(map(lambda n: "tok" + str(n), range(tokens)))
+        return map(_gen_line, line_lengths)
+
+
+    def _prep_files(box_eval):
+        thelp.cat(box_eval['src_filename'], _line([10, 20, 30, 40, 17, 21]))
+        thelp.cat(box_eval['trg_filename'], _line([40, 30, 20, 10, 20, 21]))
+        thelp.cat(box_eval['cleaned_src_file_expected'], _line([17]))
+        thelp.cat(box_eval['cleaned_trg_file_expected'], _line([20]))
+
+
+        def _cleanup_files(box_eval):
+            try:
+                for key, filename in box_eval.items():
+                    os.unlink(filename)
+            except:
+                pass
+
+
+    _test_main()

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/data_split/data_split.cfg

@@ -0,0 +1,7 @@
+[Configuration]
+evaluation_data_size = 7
+development_data_size = 13
+
+[Inputs]
+src_filename = test_data/data.en
+trg_filename = test_data/data.de

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/data_split/data_split.py

@@ -0,0 +1,144 @@
+def get_name():
+    return 'data_split'
+
+def get_inputs():
+    return ['src_filename', 'trg_filename']
+
+def get_outputs():
+    return ['devel_src_filename', 'devel_trg_filename',
+            'eval_src_filename', 'eval_trg_filename',
+            'train_src_filename', 'train_trg_filename']
+
+def get_configuration():
+    return ['evaluation_data_size', 'development_data_size']
+
+def configure(args):
+    result = {}
+    result['evaluate_size'] = args['evaluation_data_size']
+    result['development_size'] = args['development_data_size']
+    return result
+
+def initialise(config):
+    def _copy(size, inp, ofh1, ofh2):
+        try:
+            while size != 0:
+                (l1, l2) = inp.next()
+                print >>ofh1, l1,
+                print >>ofh2, l2,
+                size -= 1
+        except StopIteration:
+            pass
+
+    def _make_split_filename(filename, data_set):
+        bits = filename.split(".")
+        bits.insert(-1, data_set)
+
+        new_filename = ".".join(bits)
+        return new_filename
+
+    def _splitter_main(a, s):
+        (ifh1, ifh2, ofh1, ofh2) = (None, None, None, None)
+        try:
+            input_src_filename = a['src_filename']
+            input_trg_filename = a['trg_filename']
+
+            ifh1 = open(input_src_filename, "r")
+            ifh2 = open(input_trg_filename, "r")
+            inp = iter(zip(ifh1, ifh2))
+
+            result = {}
+            for (data_set, size) in [('devel', config['development_size']),
+                                     ('eval', config['evaluate_size']),
+                                     ('train', -1)]:
+                output_src_filename = _make_split_filename(input_src_filename, data_set)
+                output_trg_filename = _make_split_filename(input_trg_filename, data_set)
+                ofh1 = open(output_src_filename, "w")
+                ofh2 = open(output_trg_filename, "w")
+
+                _copy(size, inp, ofh1, ofh2)
+                result[data_set + '_src_filename'] = output_src_filename
+                result[data_set + '_trg_filename'] = output_trg_filename
+
+            return result
+        finally:
+            def _safe_close(fh):
+                if fh is not None:
+                    fh.close()
+                _safe_close(ifh1)
+                _safe_close(ifh2)
+                _safe_close(ofh1)
+                _safe_close(ofh2)
+    
+    return _splitter_main
+
+
+if __name__ == '__main__':
+    import os
+    import tempfile
+    import test.test as thelp
+
+    from pypeline.helpers.helpers import eval_pipeline
+
+
+    def _test_main():
+        configuration = {'evaluation_data_size': 7,
+                         'development_data_size': 13}
+
+        src_filename = tempfile.mkstemp(suffix = ".src", dir = "/tmp")
+        trg_filename = tempfile.mkstemp(suffix = ".trg", dir = "/tmp")
+
+        box_eval = {'src_filename': src_filename[1],
+                    'trg_filename': trg_filename[1],
+                    'devel_src_expected': src_filename[1] + ".devel.expected",
+                    'devel_trg_expected': trg_filename[1] + ".devel.expected",
+                    'eval_src_expected': src_filename[1] + ".eval.expected",
+                    'eval_trg_expected': trg_filename[1] + ".eval.expected",
+                    'train_src_expected': src_filename[1] + ".train.expected",
+                    'train_trg_expected': trg_filename[1] + ".train.expected"}
+
+        try:
+            _prep_files(box_eval)
+            _run_test(configuration, box_eval)
+        finally:
+            _cleanup_files(box_eval)
+
+
+    def _run_test(configuration, box_eval):
+        box_config = configure(configuration)
+        box = initialise(box_config)
+    
+        output = eval_pipeline(box, box_eval, box_config)
+        for data_set in ['devel', 'eval', 'train']:
+            for lang in ['src', 'trg']:
+                filename = output[data_set + '_' + lang + '_filename']
+                filename_expected = box_eval[data_set + '_' + lang + '_expected']
+            thelp.diff(filename_expected, filename)
+
+
+    def _line(line_lengths):
+        def _gen_line(tokens):
+            return " ".join(map(lambda n: "tok" + str(n), range(tokens)))
+        return map(_gen_line, line_lengths)
+
+
+    def _prep_files(box_eval):
+        thelp.cat(box_eval['src_filename'], _line(range(50)))
+        thelp.cat(box_eval['trg_filename'], _line(range(50)))
+        #expected output:
+        thelp.cat(box_eval['devel_src_expected'], _line(range(0,13)))
+        thelp.cat(box_eval['devel_trg_expected'], _line(range(0,13)))
+        thelp.cat(box_eval['eval_src_expected'], _line(range(13,20)))
+        thelp.cat(box_eval['eval_trg_expected'], _line(range(13,20)))
+        thelp.cat(box_eval['train_src_expected'], _line(range(20,50)))
+        thelp.cat(box_eval['train_trg_expected'], _line(range(20,50)))
+
+
+    def _cleanup_files(box_eval):
+        try:
+            for key, filename in box_eval.items():
+                os.unlink(filename)
+        except:
+            pass
+
+
+    _test_main()

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/data_split/test_data/data.de

@@ -0,0 +1,50 @@
+
+tok0
+tok0 tok1
+tok0 tok1 tok2
+tok0 tok1 tok2 tok3
+tok0 tok1 tok2 tok3 tok4
+tok0 tok1 tok2 tok3 tok4 tok5
+tok0 tok1 tok2 tok3 tok4 tok5 tok6
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46 tok47
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46 tok47 tok48

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/data_split/test_data/data.en

@@ -0,0 +1,50 @@
+
+tok0
+tok0 tok1
+tok0 tok1 tok2
+tok0 tok1 tok2 tok3
+tok0 tok1 tok2 tok3 tok4
+tok0 tok1 tok2 tok3 tok4 tok5
+tok0 tok1 tok2 tok3 tok4 tok5 tok6
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46 tok47
+tok0 tok1 tok2 tok3 tok4 tok5 tok6 tok7 tok8 tok9 tok10 tok11 tok12 tok13 tok14 tok15 tok16 tok17 tok18 tok19 tok20 tok21 tok22 tok23 tok24 tok25 tok26 tok27 tok28 tok29 tok30 tok31 tok32 tok33 tok34 tok35 tok36 tok37 tok38 tok39 tok40 tok41 tok42 tok43 tok44 tok45 tok46 tok47 tok48

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/irstlm_build/irstlm_build.py

@@ -0,0 +1,117 @@
+import os
+import shutil
+import subprocess
+import tempfile
+
+
+def get_name():
+    return 'irstlm_build'
+
+def get_inputs():
+    return ['input_filename']
+
+def get_outputs():
+    return ['add_start_end_filename', 'lm_filename', 'compiled_lm_filename']
+
+def get_configuration():
+    return ['irstlm_installation_dir', 'irstlm_smoothing_method', 'language_model_directory']
+
+def configure(args):
+    config = dict()
+    config['irstlm_install_directory'] = args['irstlm_installation_dir']
+    config['smoothing_method'] = args['irstlm_smoothing_method']
+    config['lm_directory'] = args['language_model_directory']
+    return config    
+
+def initialise(config):
+    def process(a, s):
+        # Create the LM directory if we need to
+        if os.path.exists(config['lm_directory']) is False:
+            os.makedirs(config['lm_directory'])
+
+        # The filename of the file to chew through
+        start_end_input_filename = a['input_filename']
+        if os.path.exists(start_end_input_filename) is False:
+            raise Exception("IRSTLM Build: Input file could not be found at [%s]" % start_end_input_filename)
+
+        # Derive the output file name for the add start-end marker processor
+        filename_bits = os.path.basename(start_end_input_filename).split(".")
+        filename_bits[2] = "sb";
+        start_end_output_filename = os.path.join(config['lm_directory'], ".".join(filename_bits))
+
+        # Derive the output file name of the LM build
+        filename_bits[2] = "lm"
+        lm_filename = os.path.join(config['lm_directory'], ".".join(filename_bits))
+
+        # Derive the compiled LM file name
+        filename_bits[2] = "arpa"
+        compiled_lm_filename = os.path.join(config['lm_directory'], ".".join(filename_bits))
+
+        # First thing to do is add start and end markers
+        start_end_cmdline = [os.path.join(config['irstlm_install_directory'], "bin", "add-start-end.sh")]
+        infile = open(start_end_input_filename, 'r')
+        outfile = open(start_end_output_filename, 'w')
+        print "IRSTLM Build: Invoking [%s]..." % " ".join(start_end_cmdline)
+        return_code = subprocess.check_call(start_end_cmdline, stdin = infile, stdout = outfile)
+        if return_code:
+            raise Exception("IRSTLM add start and end markers failed: input file = [%s], output file = [%s], return code = [%d]" % \
+                            start_end_input_filename, start_end_output_filename, return_code)
+
+        # Next build the language model
+        tmp_dir = tempfile.mkdtemp(dir = "/tmp")
+        try:
+            build_lm_cmdline = [os.path.join(config['irstlm_install_directory'], "bin", "build-lm.sh"),
+                                "-i", start_end_output_filename,
+                                "-t", tmp_dir,
+                                "-p",
+                                "-s", config['smoothing_method'],
+                                "-o", lm_filename]
+            print "IRSTLM Build: Invoking [%s]..." % " ".join(build_lm_cmdline)
+            return_code = subprocess.check_call(build_lm_cmdline)
+            if return_code: 
+                raise Exception("IRST language model failed to build: return code = [%d]" % return_code)
+        finally:
+            if os.path.exists(tmp_dir):
+                shutil.rmtree(tmp_dir)
+
+        # Compile the LM
+        lm_filename = lm_filename + ".gz"
+        compile_lm_cmdline = [os.path.join(config['irstlm_install_directory'], "bin", "compile-lm"),
+                              "--text", "yes",
+                              lm_filename,
+                              compiled_lm_filename]
+        print "IRSTLM Build: Invoking [%s]..." % " ".join(compile_lm_cmdline)
+        return_code = subprocess.check_call(compile_lm_cmdline)
+        if return_code:
+            raise Exception("IRST language model compilation failed: return code = [%d]" % return_code)
+
+        output = {'add_start_end_filename': start_end_output_filename,
+                  'lm_filename': lm_filename,
+                  'compiled_lm_filename': compiled_lm_filename}
+
+        print "IRSTLM Build: Output = %s" % output
+
+        return output
+
+    return process
+
+
+if __name__ == '__main__':
+    from pypeline.helpers.helpers import eval_pipeline, cons_function_component
+
+    lm_dir = os.environ["PWD"]
+    configuration = {'irstlm_root': os.environ["IRSTLM"],
+                     'irstlm_smoothing_method': 'improved-kneser-ney',
+                     'language_model_directory': lm_dir}
+    component_config = configure(configuration)
+    component = initialise(component_config)
+
+    value = eval_pipeline(cons_function_component(component),
+                          {'input_filename': '/Users/ianjohnson/Dropbox/Documents/MTM2012/tokenised_files/news-commentary-v7.fr-en.tok.en'},
+                          component_config)
+    target = {'add_start_end_filename': os.path.join(lm_dir, 'news-commentary-v7.fr-en.sb.en'),
+              'lm_filename': os.path.join(lm_dir, 'news-commentary-v7.fr-en.lm.en.gz'),
+              'compiled_lm_filename': os.path.join(lm_dir, 'news-commentary-v7.fr-en.arpa.en')}
+    print "Target: %s" % target
+    if value != target:
+        raise Exception("Massive fail!")

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/mert/mert.py

@@ -0,0 +1,98 @@
+import os
+import shutil
+import subprocess
+
+def get_name():
+    return 'mert'
+
+def get_inputs():
+    return ['evaluation_data_filename', 'trg_language_model_filename',
+            'trg_language_model_order', 'trg_language_model_type',
+            'moses_ini_filename']
+
+def get_outputs():
+    return ['moses_ini_filename']
+
+def get_configuration():
+    return ['source_language', 'target_language',
+            'moses_installation_dir', 'mert_working_directory',
+            'mert_max_no_iterations']
+
+def configure(args):
+    result = {}
+    result['src_lang'] = args['source_language']
+    result['trg_lang'] = args['target_language']
+    result['moses_installation_dir'] = args['moses_installation_dir']
+    result['mert_working_dir'] = args['mert_working_directory']
+    result['max_no_iterations'] = args['mert_max_no_iterations']
+    return result
+
+def initialise(config):
+    def process(a, s):
+        infilename = os.path.abspath(a['evaluation_data_filename'])
+        infilename = ".".join(infilename.split(".")[:-1])
+        lm_file = os.path.abspath(a['trg_language_model_filename'])
+        lm_order = int(a['trg_language_model_order'])
+        lm_type = int(a['trg_language_model_type'])
+        max_no_iters = int(config['max_no_iterations'])
+        orig_moses_ini = os.path.abspath(a['moses_ini_filename'])
+        
+        if not os.path.exists(orig_moses_ini):
+            raise Exception, "Error: Input moses.ini does not exist"
+
+        workdir = os.path.abspath(config['mert_working_dir'])
+        #simply call the training perl script
+        #remove the workdir if it is already there
+        if os.path.exists(workdir):
+            shutil.rmtree(workdir)
+        os.makedirs(workdir)
+
+        #local vars
+        moses_install_dir = os.path.abspath(config['moses_installation_dir'])
+        mert_perl = os.path.join(moses_install_dir, 'scripts', 'training', 'mert-moses.pl')
+        bin_dir = os.path.join(moses_install_dir, 'bin')
+        moses_bin = os.path.join(moses_install_dir, 'bin', 'moses')
+        src_file = infilename + '.' + config['src_lang']
+        ref_file = infilename + '.' + config['trg_lang']
+        logfile = os.path.join(workdir, 'log')
+        #change lm configuration in moses ini
+        moses_ini = os.path.join(workdir, 'trained-moses.ini')
+        cmd = r"cat %(orig_moses_ini)s | sed '/\[lmodel-file\]/,/^[[:space:]]*$/c\[lmodel-file\]\n%(lm_type)s 0 %(lm_order)s %(lm_file)s\n' > %(moses_ini)s"
+        cmd = cmd % locals()
+        os.system(cmd)
+        
+        #the command
+        cmd = '%(mert_perl)s --maximum-iterations %(max_no_iters)d --mertdir %(bin_dir)s --working-dir %(workdir)s %(src_file)s %(ref_file)s %(moses_bin)s %(moses_ini)s 2> %(logfile)s'
+        cmd = cmd % locals()
+
+        pipe = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, shell=True)
+        pipe.wait()
+
+        #check the moses ini
+        new_mosesini = os.path.join(workdir, 'moses.ini')
+        if not os.path.exists(new_mosesini):
+            raise Exception, 'Failed MERT'
+        
+        return {'moses_ini_filename' : new_mosesini}
+
+    return process
+
+
+if __name__ == '__main__':
+    def __test():
+        configuration = {'src_lang':'en',
+                         'trg_lang':'lt',
+                         'moses_installation_dir':os.path.abspath('../../../../'),
+                         'mert_working_dir':'../../../../../tuning'}
+        values = {'development_data_filename':'../../../../../corpus/tune',
+                  'moses_ini_file':'../../../../../model/model/moses.ini',
+                  'trg_language_model_filename':'../../../../../corpus/train.lt.lm',
+                  'trg_language_model_type':9,
+                  'trg_language_model_order':4}
+        from pypeline.helpers.helpers import run_pipeline
+        box_config = configure(configuration)
+        box = initialise(configuration)
+        print run_pipeline(box, values, None)
+
+    #do some test
+    __test()

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/model_training/model_training.py

@@ -0,0 +1,103 @@
+import os
+import shutil
+import subprocess
+
+
+def get_name():
+    return 'model_training'
+
+def get_inputs():
+    return ['src_filename', 'trg_filename']
+
+def get_outputs():
+    return ['moses_ini_filename']
+
+def get_configuration():
+    return ['source_language', 'target_language',
+            'moses_installation_dir', 'giza_installation_dir',
+            'translation_model_directory', 'alignment_method',
+            'reordering_method']
+
+# Alignment = grow-diag-final-and
+# Reordering = msd-bidirectional-fe
+def configure(args):
+    result = {}
+    result['src_lang'] = args['source_language']
+    result['trg_lang'] = args['target_language']
+    result['moses_installation_dir'] = args['moses_installation_dir']
+    result['external_bin_dir'] = args['giza_installation_dir']
+    result['model_directory'] = args['translation_model_directory']
+    result['alignment'] = args['alignment_method']
+    result['reordering'] = args['reordering_method']
+    return result
+
+def initialise(config):
+    def process(a, s):
+        get_corpora_name_fn = lambda fn: ".".join(os.path.basename(fn).split('.')[:-1])
+        src_filename = os.path.abspath(a['src_filename'])
+        trg_filename = os.path.abspath(a['trg_filename'])
+        src_corpora_name = get_corpora_name_fn(src_filename)
+        trg_corpora_name = get_corpora_name_fn(trg_filename)
+        if src_corpora_name != trg_corpora_name:
+            raise Exception, "Mismatch of source [%s] and target [%s] filename" % (src_filename, trg_filename)
+
+        infilename = os.path.abspath(os.path.join(os.path.dirname(src_filename), src_corpora_name))
+        workdir = os.path.abspath(config['model_directory'])
+        #simply call the training perl script
+        #remove the workdir if it is already there
+        if os.path.exists(workdir):
+            shutil.rmtree(workdir)
+        os.makedirs(workdir)
+        
+        #local vars
+        train_model_perl = os.path.abspath(os.path.join(config['moses_installation_dir'],
+                                                        'scripts',
+                                                        'training',
+                                                        'train-model.perl'))
+        src_lang = config['src_lang'].lower()
+        trg_lang = config['trg_lang'].lower()
+        external_bin = os.path.abspath(config['external_bin_dir'])
+        #create a dummy lm file
+        dummy_lmfile = os.path.join(workdir, 'dummy.lm')
+        f = open(dummy_lmfile, 'w')
+        print >> f, "dummy lm file"
+        f.close()
+        logfile = os.path.join(workdir, 'log')
+        
+        #the command
+        alignment_method = config['alignment']
+        reordering_method = config['reordering']
+        cmd = '%(train_model_perl)s -root-dir %(workdir)s -corpus %(infilename)s ' \
+              '-f %(src_lang)s -e %(trg_lang)s -alignment %(alignment_method)s ' \
+              '-reordering %(reordering_method)s -lm 0:5:%(dummy_lmfile)s:0 ' \
+              '-external-bin-dir %(external_bin)s 2> %(logfile)s'
+        cmd = cmd % locals()
+
+        pipe = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, shell=True)
+        pipe.wait()
+
+        # check the moses ini
+        mosesini = os.path.join(workdir, 'model', 'moses.ini')
+        if not os.path.exists(mosesini):
+            raise Exception, 'Failed training model'
+
+        return {'moses_ini_filename' : mosesini}
+
+    return process
+
+
+if __name__ == '__main__':
+    def __test():
+        configuration = {'src_lang' : 'en',
+                         'trg_lang' : 'lt',
+                         'moses_installation_dir' : os.environ['MOSES_HOME'],
+                         'giza_installation_dir' : os.environ['GIZA_HOME'],
+                         'translation_model_directory' : 'model-dir'}
+        values = {'training_data_filename' : '/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain'}
+        from pypeline.helpers.helpers import run_pipeline
+        box_config = configure(configuration)
+        box = initialise(box_config)
+        print run_pipeline(box, values, None)
+
+    #do some test
+    __test()

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/tokenizer/Makefile

@@ -0,0 +1,15 @@
+CC = pclc.py
+CFLAGS = -i
+SOURCES = tokenizer.pcl
+OBJS = $(SOURCES:.pcl=.py)
+
+all: build
+
+build: $(OBJS)
+
+%.py: %.pcl
+	$(CC) $(CFLAGS) $<
+
+clean:
+	rm -f *.py *.pyc *.log *~
+

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/tokenizer/tokenizer.cfg

@@ -0,0 +1,7 @@
+[Configuration]
+corpus.language = en
+working.directory.root = tokenised
+moses.installation = /opt/moses
+
+[Inputs]
+corpus.filename = test_data/test.en

mosesdecoder/contrib/arrow-pipelines/pcl/components/wrappers/tokenizer/tokenizer.pcl

@@ -0,0 +1,38 @@
+import pcl.io.file as file
+import pcl.os.path as path
+import pcl.system.process as process
+import pcl.util.list as list
+import pcl.util.string as string
+
+component tokenizer
+  input corpus.filename
+  output corpus.tokenised.filename
+  configuration corpus.language, working.directory.root, moses.installation
+  do
+    language <- string.lower(@corpus.language)
+
+    corpus.file.basename <- path.basename(corpus.filename)
+    corpus.file.basename.bits <- string.split(corpus.file.basename, ".")
+    list.insert(corpus.file.basename.bits, -1, "tok")
+    result.basename <- string.join(corpus.file.basename.bits, ".")
+    result.pathname <- path.join(@working.directory.root, result.basename)
+
+    working.exists <- path.exists(@working.directory.root)
+    if working.exists == False then
+      path.makedirs(@working.directory.root)
+      return ()
+    else
+      return ()
+    endif
+
+    tokeniser.cmd <- path.join(@moses.installation, "scripts",
+                               "tokenizer", "tokenizer.perl")
+    tokeniser.cmd.line <- list.cons(tokeniser.cmd, "-l", language, "-q")
+
+    corpus.file <- file.openFile(corpus.filename, "r")
+    result.file <- file.openFile(result.pathname, "w")
+    process.callAndCheck(tokeniser.cmd.line, corpus.file, result.file)
+    file.closeFile(result.file)
+    file.closeFile(corpus.file)
+
+    return corpus.tokenised.filename <- result.pathname

mosesdecoder/contrib/arrow-pipelines/pcl/training_pipeline.cfg

@@ -0,0 +1,21 @@
+[Configuration]
+source_language = en
+target_language = lt
+max_segment_length = 20
+corpus_development_size = 1000
+corpus_evaluation_size = 500
+alignment_method = grow-diag-final-and
+reordering_method = msd-bidirectional-fe
+smoothing_method = improved-kneser-ney
+tokenisation_directory = training/tokenisation
+translation_model_directory = training/model
+language_model_directory = training/lm
+mert_directory = training/mert
+mert_max_no_iterations = 10
+moses_installation_directory = $(MOSES_HOME)
+giza_installation_directory = $(GIZA_HOME)
+irstlm_installation_directory = $(IRSTLM)
+
+[Inputs]
+src_filename = ../test_data/cleantrain.en
+trg_filename = ../test_data/cleantrain.lt

mosesdecoder/contrib/arrow-pipelines/pcl/training_pipeline.pcl

@@ -0,0 +1,117 @@
+#
+# Import all of the components to be composed
+#
+import components.src_trg_tokeniser as tokeniser
+import components.translation_model_training as model_training
+import components.wrappers.irstlm_build.irstlm_build as lang_model
+import components.wrappers.mert.mert as mert
+
+#
+# Component definition
+#
+#                                                        Config: {model_training.max_segment_length,
+#                                                                 model_training.corpus.[development_size|evaluation_size],
+#                                                                 model_training.[src|trg].language,
+#                                                                 model_training.method.[alignment|reordering],  {moses_ini_filename,
+#                                                                 model_training.giza.installation,               evaluation_data_filename}
+# {src_filename,    {tokenised_src_filename,                      model_training.translation_model.dir}           |
+#  trg_filename}     tokenised_trg_filename}             +-----------------------------------------+   +-------+  |          {moses_ini_filename}
+#  |  +-------+    +-------+    +-------+ |  +-------+   | tokenised_src_filename -> src_filename, |   | Model |  V    +-------+               |
+#  V  |       +--->+ Src/  +--->+       | V  |       +-->+ tokenised_trg_filename -> trg_filename  +-->+ Train +------>+       |      +------+ V
+# --->+ Split |    | Trg   |    | Merge +--->+ Split |   +-----------------------------------------+   +-------+       | Merge +----->+ MERT +--->
+#     |       +--->+ Token +--->+       |    |       +--\  +------------------------------------------+   +--------+   |       |  ^   +------+
+#     +-------+    +-------+    +-------+    +-------+  \->+ tokenised_trg_filename -> input_filename +-->+ IRSTLM +-->+       |  |
+# Config: {tokeniser.[src|trg].language,                   +------------------------------------------+   +--------+ ^ +-------+  |
+#          tokeniser.[src|trg].tokeniser_dir                             Config: {irstlm_installation_dir::String,   |            |
+#          tokeniser.moses.installation}                                          irstlm_smoothing_method::String,   |            |
+#                                                                                 language_model_directory}          |            |
+#                                                                                                                    |            |
+#                                                           {lm_filename, compiled_lm_filename, add_start_end_filename}           |
+#                                                                                                                                 |
+#                                                            {moses_ini_file, evaluation_data_filename, trg_language_model_filename,
+#                                                             trg_language_model_order, trg_language_model_type}
+#
+component training_pipeline
+  inputs src_filename, trg_filename
+  output moses_ini_filename
+  configuration source_language,
+                target_language,
+                max_segment_length,
+                corpus_development_size,
+                corpus_evaluation_size,
+                alignment_method,
+                reordering_method,
+                smoothing_method,
+                tokenisation_directory,
+                translation_model_directory,
+                language_model_directory,
+                mert_directory,
+                mert_max_no_iterations,
+                moses_installation_directory,
+                giza_installation_directory,
+                irstlm_installation_directory
+  declare
+    tokeniser := new tokeniser with
+      source_language -> tokeniser.src.language,
+      target_language -> tokeniser.trg.language,
+      tokenisation_directory -> tokeniser.src.tokenisation_dir,
+      tokenisation_directory -> tokeniser.trg.tokenisation_dir,
+      moses_installation_directory -> tokeniser.moses.installation
+    model_training := new model_training with
+      max_segment_length -> model_training.max_segment_length,
+      corpus_development_size -> model_training.corpus.development_size,
+      corpus_evaluation_size -> model_training.corpus.evaluation_size,
+      translation_model_directory -> model_training.translation_model.dir,
+      alignment_method -> model_training.method.alignment,
+      reordering_method -> model_training.method.reordering,
+      source_language -> model_training.src.language,
+      moses_installation_directory -> model_training.moses.installation,
+      giza_installation_directory -> model_training.giza.installation,
+      target_language -> model_training.trg.language
+    irstlm := new lang_model with
+      irstlm_installation_directory -> irstlm_installation_dir,
+      smoothing_method -> irstlm_smoothing_method,
+      language_model_directory -> language_model_directory
+    mert := new mert with
+      source_language -> source_language,
+      target_language -> target_language,
+      moses_installation_directory -> moses_installation_dir,
+      mert_directory -> mert_working_directory,
+      mert_max_no_iterations -> mert_max_no_iterations
+  as
+    # Split and transform the input to the tokeniser component
+    #  Inputs: src_filename, trg_filename
+    # Outputs: (tokenised_src_filename), (tokenised_trg_filename)
+    (wire src_filename -> src_filename,
+          trg_filename -> _ &&&
+     wire trg_filename -> trg_filename,
+          src_filename -> _) >>>
+    tokeniser >>>
+
+    # Merge output from tokeniser
+    #  Inputs: (tokenised_src_filename), (tokenised_trg_filename)
+    # Outputs: tokenised_src_filename, tokenised_trg_filename
+    merge top[tokenised_src_filename] -> tokenised_src_filename,
+          bottom[tokenised_trg_filename] -> tokenised_trg_filename >>>
+
+    # Train the translation table and target language model
+    #  Inputs: tokenised_src_filename, tokenised_trg_filename
+    # Outputs: (moses_ini_filename), ('add_start_end_filename', 'lm_filename', 'compiled_lm_filename')
+    ((wire tokenised_src_filename -> src_filename,
+           tokenised_trg_filename -> trg_filename >>> model_training) &&&
+     (wire tokenised_trg_filename -> input_filename,
+           tokenised_src_filename -> _ >>> irstlm)) >>>
+
+    # Merge the output from the TT and LM training component
+    #  Inputs: (moses_ini_filename, evaluation_data_filename),
+    #          (compiled_lm_filename, add_start_end_filename, lm_filename)
+    # Outputs: moses_ini_filename, evaluation_data_filename, evaluation_data_filename,
+    #          trg_language_model_filename, trg_language_model_order, trg_language_model_type
+    merge top[moses_ini_filename] -> moses_ini_filename,
+          top[evaluation_data_filename] -> evaluation_data_filename,
+          bottom[compiled_lm_filename] -> trg_language_model_filename,
+          bottom[add_start_end_filename] -> _,
+          bottom[lm_filename] -> _,
+          3 -> trg_language_model_order,
+          9 -> trg_language_model_type >>>
+    mert

mosesdecoder/contrib/c++tokenizer/Jamfile

@@ -0,0 +1,13 @@
+
+with-re2 = [ option.get "with-re2" ] ;
+if $(with-re2) {
+  lib re2 : : <search>$(with-re2)/lib ;
+	external-lib glib-2.0 ;
+	glib-cflags = [ _shell "pkg-config --cflags glib-2.0" ] ;
+  includes += <include>$(with-re2)/include ;
+	exe tokenizer : tokenizer.cpp tokenizer_main.cpp Parameters.cpp re2 glib-2.0 : <cflags>-std=c++0x <cflags>$(glib-cflags) $(includes) ;
+}
+else {
+  alias tokenizer ;
+}
+

mosesdecoder/contrib/c++tokenizer/Parameters.cpp

@@ -0,0 +1,39 @@
+#include "Parameters.h"
+
+#ifdef TOKENIZER_NAMESPACE
+namespace TOKENIZER_NAMESPACE {
+#endif
+
+Parameters::Parameters()
+: nthreads(0)
+, chunksize(2000)
+, cfg_path(0)
+, verbose_p(false)
+, detag_p(false)
+, alltag_p(false)
+, entities_p(false)
+, escape_p(false)
+, aggro_p(false)
+, supersub_p(false)
+, url_p(true)
+, downcase_p(false)
+, normalize_p(false)
+, penn_p(false)
+, words_p(false)
+, denumber_p(false)
+, narrow_latin_p(false)
+, narrow_kana_p(false)
+, refined_p(false)
+, unescape_p(false)
+, drop_bad_p(false)
+, split_p(false)
+, notokenization_p(false)
+, para_marks_p(false)
+, split_breaks_p(false)
+{
+}
+
+#ifdef TOKENIZER_NAMESPACE
+}
+#endif
+

mosesdecoder/contrib/c++tokenizer/Parameters.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#ifdef TOKENIZER_NAMESPACE
+namespace TOKENIZER_NAMESPACE {
+#endif
+
+struct Parameters
+{
+    std::string lang_iso;
+    std::vector<std::string> args;
+    std::string out_path;
+    int nthreads;
+    int chunksize;
+    const char *cfg_path;
+    bool verbose_p;
+    bool detag_p;
+    bool alltag_p;
+    bool entities_p;
+    bool escape_p;
+    bool aggro_p;
+    bool supersub_p;
+    bool url_p;
+    bool downcase_p;
+    bool normalize_p;
+    bool penn_p;
+    bool words_p;
+    bool denumber_p;
+    bool narrow_latin_p;
+    bool narrow_kana_p;
+    bool refined_p;
+    bool unescape_p;
+    bool drop_bad_p;
+    bool split_p;
+    bool notokenization_p;
+    bool para_marks_p;
+    bool split_breaks_p;
+
+	Parameters();
+
+    Parameters(const Parameters& _);
+};
+
+
+#ifdef TOKENIZER_NAMESPACE
+}
+#endif
+
+

mosesdecoder/contrib/c++tokenizer/tokenizer.cpp

@@ -0,0 +1,2246 @@
+#include "tokenizer.h"
+#include <re2/stringpiece.h>
+#include <sstream>
+#include <iterator>
+#include <memory>
+#include <vector>
+#include <algorithm>
+#include <cstring>
+#include <set>
+#include <glib.h>
+#include <stdexcept>
+#include <boost/thread.hpp>
+
+namespace { // anonymous namespace
+
+// frequently used regexp's are pre-compiled thus:
+
+RE2 genl_tags_x("<[/!\\p{L}]+[^>]*>");
+RE2 mult_spc_x(" +"); // multiple spaces
+RE2 tag_line_x("^<.+>$"); // lines beginning and ending with open/close angle-bracket pairs
+RE2 white_line_x("^\\s*$"); // lines entirely composed of whitespace
+RE2 slash_x("([\\p{L}\\p{N}])(/)([\\p{L}\\p{N}])"); // and slash-conjoined " "
+RE2 final_x("([^.])([.])([\\]\\)}>\"']*) ?$"); // sentence-final punctuation sequence (non qm em)
+RE2 qx_x("([?!])"); // one qm/em mark
+RE2 braces_x("([\\]\\[\\(\\){}<>])"); // any open or close of a pair
+RE2 endq_x("([^'])' "); // post-token single-quote or doubled single-quote
+RE2 letter_x("\\p{L}"); // a letter
+RE2 lower_x("^\\p{Ll}"); // a lower-case letter
+RE2 sinteger_x("^\\p{N}"); // not a digit mark
+RE2 numprefixed_x("[-+/.@\\\\#\\%&\\p{Sc}\\p{N}]*[\\p{N}]+-[-'`\"\\p{L}]*\\p{L}");
+RE2 quasinumeric_x("[-.;:@\\\\#\%&\\p{Sc}\\p{So}\\p{N}]*[\\p{N}]+");
+RE2 numscript_x("([\\p{N}\\p{L}])([\\p{No}]+)(\\p{Ll})");
+
+RE2 x1_v_d("([ ([{<])\""); // a valid non-letter preceeding a double-quote
+RE2 x1_v_gg("([ ([{<])``"); // a valid non-letter preceeding directional doubled open single-quote
+RE2 x1_v_g("([ ([{<])`([^`])"); //  a valid non-letter preceeding directional unitary single-quote
+RE2 x1_v_q("([ ([{<])'"); //  a valid non-letter preceeding undirected embedded quotes
+RE2 ndndcomma_x("([^\\p{N}]),([^\\p{N}])"); // non-digit,non-digit
+RE2 pdndcomma_x("([\\p{N}]),([^\\p{N}])"); // digit,non-digit
+RE2 ndpdcomma_x("([^\\p{N}]),([\\p{N}])"); // non-digit,digit
+RE2 symbol_x("([;:@\\#\\$%&\\p{Sc}\\p{So}])"); // usable punctuation mark not a quote or a brace
+RE2 contract_x("'([sSmMdD]) "); // english single letter contraction forms, as embedded
+RE2 right_x("[({¿¡]+"); // symbols which conjoin to the right
+RE2 left_x("[,.?!:;\\%\\p{Sc}})]+"); // symbols conjoin to the left
+RE2 curr_en_x("^[Nn]?[\'][\\p{L}]"); // english contraction suffixes conjoin to the left
+RE2 pre_en_x(".*[\\p{L}\\p{N}]+$"); // valid english contraction prefixes
+RE2 curr_fr_x(".*[\\p{L}\\p{N}]+[\']"); // french/italian contraction prefixes conjoin to the right
+RE2 post_fr_x("^[\\p{L}\\p{N}]*"); // valid french/italian contraction suffixes
+// anything rarely used will just be given as a string and compiled on demand by RE2
+
+const char *
+SPC_BYTE = " ";
+//const char *
+//URL_VALID_SYM_CHARS = "-._~:/?#[]@!$&'()*+,;=";
+
+inline bool
+class_follows_p(gunichar *s, gunichar *e, GUnicodeType gclass) {
+    while (s < e) {
+        GUnicodeType tclass = g_unichar_type(*s);
+        if (tclass == gclass)
+            return true;
+        switch (tclass) {
+        case G_UNICODE_SPACING_MARK:
+        case G_UNICODE_LINE_SEPARATOR:
+        case G_UNICODE_PARAGRAPH_SEPARATOR:
+        case G_UNICODE_SPACE_SEPARATOR:
+            ++s;
+            continue;
+            break;
+        default:
+            return false;
+        }
+    }
+    return false;
+}
+
+
+const char *ESCAPE_MOSES[] = {
+        "&#124;", // | 0
+        "&#91;", // [ 1
+        "&#93;",  // ] 2
+        "&amp;", // & 3 (26)
+        "&lt;", // < 4 (3c)
+        "&gt;", // > 5 (3e)
+        "&apos;", // ' 6 (27)
+        "&quot;", // " 7 (22)
+};
+
+const std::set<std::string>
+ESCAPE_SET = {
+    std::string(ESCAPE_MOSES[0]),
+    std::string(ESCAPE_MOSES[1]),
+    std::string(ESCAPE_MOSES[2]),
+    std::string(ESCAPE_MOSES[3]),
+    std::string(ESCAPE_MOSES[4]),
+    std::string(ESCAPE_MOSES[5]),
+    std::string(ESCAPE_MOSES[6]),
+    std::string(ESCAPE_MOSES[7]),
+};
+
+const std::map<std::wstring,gunichar>
+ENTITY_MAP = {
+    { std::wstring(L"&quot;"), L'"' },
+    { std::wstring(L"&amp;"), L'&' },
+    { std::wstring(L"&apos;"), L'\'' },
+    { std::wstring(L"&lt;"), L'<' },
+    { std::wstring(L"&gt;"), L'>' },
+    { std::wstring(L"&nbsp;"), L'\u00A0' },
+    { std::wstring(L"&iexcl;"), L'\u00A1' },
+    { std::wstring(L"&cent;"), L'\u00A2' },
+    { std::wstring(L"&pound;"), L'\u00A3' },
+    { std::wstring(L"&curren;"), L'\u00A4' },
+    { std::wstring(L"&yen;"), L'\u00A5' },
+    { std::wstring(L"&brvbar;"), L'\u00A6' },
+    { std::wstring(L"&sect;"), L'\u00A7' },
+    { std::wstring(L"&uml;"), L'\u00A8' },
+    { std::wstring(L"&copy;"), L'\u00A9' },
+    { std::wstring(L"&ordf;"), L'\u00AA' },
+    { std::wstring(L"&laquo;"), L'\u00AB' },
+    { std::wstring(L"&not;"), L'\u00AC' },
+    { std::wstring(L"&shy;"), L'\u00AD' },
+    { std::wstring(L"&reg;"), L'\u00AE' },
+    { std::wstring(L"&macr;"), L'\u00AF' },
+    { std::wstring(L"&deg;"), L'\u00B0' },
+    { std::wstring(L"&plusmn;"), L'\u00B1' },
+    { std::wstring(L"&sup2;"), L'\u00B2' },
+    { std::wstring(L"&sup3;"), L'\u00B3' },
+    { std::wstring(L"&acute;"), L'\u00B4' },
+    { std::wstring(L"&micro;"), L'\u00B5' },
+    { std::wstring(L"&para;"), L'\u00B6' },
+    { std::wstring(L"&middot;"), L'\u00B7' },
+    { std::wstring(L"&cedil;"), L'\u00B8' },
+    { std::wstring(L"&sup1;"), L'\u00B9' },
+    { std::wstring(L"&ordm;"), L'\u00BA' },
+    { std::wstring(L"&raquo;"), L'\u00BB' },
+    { std::wstring(L"&frac14;"), L'\u00BC' },
+    { std::wstring(L"&frac12;"), L'\u00BD' },
+    { std::wstring(L"&frac34;"), L'\u00BE' },
+    { std::wstring(L"&iquest;"), L'\u00BF' },
+    { std::wstring(L"&Agrave;"), L'\u00C0' },
+    { std::wstring(L"&Aacute;"), L'\u00C1' },
+    { std::wstring(L"&Acirc;"), L'\u00C2' },
+    { std::wstring(L"&Atilde;"), L'\u00C3' },
+    { std::wstring(L"&Auml;"), L'\u00C4' },
+    { std::wstring(L"&Aring;"), L'\u00C5' },
+    { std::wstring(L"&AElig;"), L'\u00C6' },
+    { std::wstring(L"&Ccedil;"), L'\u00C7' },
+    { std::wstring(L"&Egrave;"), L'\u00C8' },
+    { std::wstring(L"&Eacute;"), L'\u00C9' },
+    { std::wstring(L"&Ecirc;"), L'\u00CA' },
+    { std::wstring(L"&Euml;"), L'\u00CB' },
+    { std::wstring(L"&Igrave;"), L'\u00CC' },
+    { std::wstring(L"&Iacute;"), L'\u00CD' },
+    { std::wstring(L"&Icirc;"), L'\u00CE' },
+    { std::wstring(L"&Iuml;"), L'\u00CF' },
+    { std::wstring(L"&ETH;"), L'\u00D0' },
+    { std::wstring(L"&Ntilde;"), L'\u00D1' },
+    { std::wstring(L"&Ograve;"), L'\u00D2' },
+    { std::wstring(L"&Oacute;"), L'\u00D3' },
+    { std::wstring(L"&Ocirc;"), L'\u00D4' },
+    { std::wstring(L"&Otilde;"), L'\u00D5' },
+    { std::wstring(L"&Ouml;"), L'\u00D6' },
+    { std::wstring(L"&times;"), L'\u00D7' },
+    { std::wstring(L"&Oslash;"), L'\u00D8' },
+    { std::wstring(L"&Ugrave;"), L'\u00D9' },
+    { std::wstring(L"&Uacute;"), L'\u00DA' },
+    { std::wstring(L"&Ucirc;"), L'\u00DB' },
+    { std::wstring(L"&Uuml;"), L'\u00DC' },
+    { std::wstring(L"&Yacute;"), L'\u00DD' },
+    { std::wstring(L"&THORN;"), L'\u00DE' },
+    { std::wstring(L"&szlig;"), L'\u00DF' },
+    { std::wstring(L"&agrave;"), L'\u00E0' },
+    { std::wstring(L"&aacute;"), L'\u00E1' },
+    { std::wstring(L"&acirc;"), L'\u00E2' },
+    { std::wstring(L"&atilde;"), L'\u00E3' },
+    { std::wstring(L"&auml;"), L'\u00E4' },
+    { std::wstring(L"&aring;"), L'\u00E5' },
+    { std::wstring(L"&aelig;"), L'\u00E6' },
+    { std::wstring(L"&ccedil;"), L'\u00E7' },
+    { std::wstring(L"&egrave;"), L'\u00E8' },
+    { std::wstring(L"&eacute;"), L'\u00E9' },
+    { std::wstring(L"&ecirc;"), L'\u00EA' },
+    { std::wstring(L"&euml;"), L'\u00EB' },
+    { std::wstring(L"&igrave;"), L'\u00EC' },
+    { std::wstring(L"&iacute;"), L'\u00ED' },
+    { std::wstring(L"&icirc;"), L'\u00EE' },
+    { std::wstring(L"&iuml;"), L'\u00EF' },
+    { std::wstring(L"&eth;"), L'\u00F0' },
+    { std::wstring(L"&ntilde;"), L'\u00F1' },
+    { std::wstring(L"&ograve;"), L'\u00F2' },
+    { std::wstring(L"&oacute;"), L'\u00F3' },
+    { std::wstring(L"&ocirc;"), L'\u00F4' },
+    { std::wstring(L"&otilde;"), L'\u00F5' },
+    { std::wstring(L"&ouml;"), L'\u00F6' },
+    { std::wstring(L"&divide;"), L'\u00F7' },
+    { std::wstring(L"&oslash;"), L'\u00F8' },
+    { std::wstring(L"&ugrave;"), L'\u00F9' },
+    { std::wstring(L"&uacute;"), L'\u00FA' },
+    { std::wstring(L"&ucirc;"), L'\u00FB' },
+    { std::wstring(L"&uuml;"), L'\u00FC' },
+    { std::wstring(L"&yacute;"), L'\u00FD' },
+    { std::wstring(L"&thorn;"), L'\u00FE' },
+    { std::wstring(L"&yuml;"), L'\u00FF' },
+    { std::wstring(L"&OElig;"), L'\u0152' },
+    { std::wstring(L"&oelig;"), L'\u0153' },
+    { std::wstring(L"&Scaron;"), L'\u0160' },
+    { std::wstring(L"&scaron;"), L'\u0161' },
+    { std::wstring(L"&Yuml;"), L'\u0178' },
+    { std::wstring(L"&fnof;"), L'\u0192' },
+    { std::wstring(L"&circ;"), L'\u02C6' },
+    { std::wstring(L"&tilde;"), L'\u02DC' },
+    { std::wstring(L"&Alpha;"), L'\u0391' },
+    { std::wstring(L"&Beta;"), L'\u0392' },
+    { std::wstring(L"&Gamma;"), L'\u0393' },
+    { std::wstring(L"&Delta;"), L'\u0394' },
+    { std::wstring(L"&Epsilon;"), L'\u0395' },
+    { std::wstring(L"&Zeta;"), L'\u0396' },
+    { std::wstring(L"&Eta;"), L'\u0397' },
+    { std::wstring(L"&Theta;"), L'\u0398' },
+    { std::wstring(L"&Iota;"), L'\u0399' },
+    { std::wstring(L"&Kappa;"), L'\u039A' },
+    { std::wstring(L"&Lambda;"), L'\u039B' },
+    { std::wstring(L"&Mu;"), L'\u039C' },
+    { std::wstring(L"&Nu;"), L'\u039D' },
+    { std::wstring(L"&Xi;"), L'\u039E' },
+    { std::wstring(L"&Omicron;"), L'\u039F' },
+    { std::wstring(L"&Pi;"), L'\u03A0' },
+    { std::wstring(L"&Rho;"), L'\u03A1' },
+    { std::wstring(L"&Sigma;"), L'\u03A3' },
+    { std::wstring(L"&Tau;"), L'\u03A4' },
+    { std::wstring(L"&Upsilon;"), L'\u03A5' },
+    { std::wstring(L"&Phi;"), L'\u03A6' },
+    { std::wstring(L"&Chi;"), L'\u03A7' },
+    { std::wstring(L"&Psi;"), L'\u03A8' },
+    { std::wstring(L"&Omega;"), L'\u03A9' },
+    { std::wstring(L"&alpha;"), L'\u03B1' },
+    { std::wstring(L"&beta;"), L'\u03B2' },
+    { std::wstring(L"&gamma;"), L'\u03B3' },
+    { std::wstring(L"&delta;"), L'\u03B4' },
+    { std::wstring(L"&epsilon;"), L'\u03B5' },
+    { std::wstring(L"&zeta;"), L'\u03B6' },
+    { std::wstring(L"&eta;"), L'\u03B7' },
+    { std::wstring(L"&theta;"), L'\u03B8' },
+    { std::wstring(L"&iota;"), L'\u03B9' },
+    { std::wstring(L"&kappa;"), L'\u03BA' },
+    { std::wstring(L"&lambda;"), L'\u03BB' },
+    { std::wstring(L"&mu;"), L'\u03BC' },
+    { std::wstring(L"&nu;"), L'\u03BD' },
+    { std::wstring(L"&xi;"), L'\u03BE' },
+    { std::wstring(L"&omicron;"), L'\u03BF' },
+    { std::wstring(L"&pi;"), L'\u03C0' },
+    { std::wstring(L"&rho;"), L'\u03C1' },
+    { std::wstring(L"&sigmaf;"), L'\u03C2' },
+    { std::wstring(L"&sigma;"), L'\u03C3' },
+    { std::wstring(L"&tau;"), L'\u03C4' },
+    { std::wstring(L"&upsilon;"), L'\u03C5' },
+    { std::wstring(L"&phi;"), L'\u03C6' },
+    { std::wstring(L"&chi;"), L'\u03C7' },
+    { std::wstring(L"&psi;"), L'\u03C8' },
+    { std::wstring(L"&omega;"), L'\u03C9' },
+    { std::wstring(L"&thetasym;"), L'\u03D1' },
+    { std::wstring(L"&upsih;"), L'\u03D2' },
+    { std::wstring(L"&piv;"), L'\u03D6' },
+    { std::wstring(L"&ensp;"), L'\u2002' },
+    { std::wstring(L"&emsp;"), L'\u2003' },
+    { std::wstring(L"&thinsp;"), L'\u2009' },
+    { std::wstring(L"&zwnj;"), L'\u200C' },
+    { std::wstring(L"&zwj;"), L'\u200D' },
+    { std::wstring(L"&lrm;"), L'\u200E' },
+    { std::wstring(L"&rlm;"), L'\u200F' },
+    { std::wstring(L"&ndash;"), L'\u2013' },
+    { std::wstring(L"&mdash;"), L'\u2014' },
+    { std::wstring(L"&lsquo;"), L'\u2018' },
+    { std::wstring(L"&rsquo;"), L'\u2019' },
+    { std::wstring(L"&sbquo;"), L'\u201A' },
+    { std::wstring(L"&ldquo;"), L'\u201C' },
+    { std::wstring(L"&rdquo;"), L'\u201D' },
+    { std::wstring(L"&bdquo;"), L'\u201E' },
+    { std::wstring(L"&dagger;"), L'\u2020' },
+    { std::wstring(L"&Dagger;"), L'\u2021' },
+    { std::wstring(L"&bull;"), L'\u2022' },
+    { std::wstring(L"&hellip;"), L'\u2026' },
+    { std::wstring(L"&permil;"), L'\u2030' },
+    { std::wstring(L"&prime;"), L'\u2032' },
+    { std::wstring(L"&Prime;"), L'\u2033' },
+    { std::wstring(L"&lsaquo;"), L'\u2039' },
+    { std::wstring(L"&rsaquo;"), L'\u203A' },
+    { std::wstring(L"&oline;"), L'\u203E' },
+    { std::wstring(L"&frasl;"), L'\u2044' },
+    { std::wstring(L"&euro;"), L'\u20AC' },
+    { std::wstring(L"&image;"), L'\u2111' },
+    { std::wstring(L"&weierp;"), L'\u2118' },
+    { std::wstring(L"&real;"), L'\u211C' },
+    { std::wstring(L"&trade;"), L'\u2122' },
+    { std::wstring(L"&alefsym;"), L'\u2135' },
+    { std::wstring(L"&larr;"), L'\u2190' },
+    { std::wstring(L"&uarr;"), L'\u2191' },
+    { std::wstring(L"&rarr;"), L'\u2192' },
+    { std::wstring(L"&darr;"), L'\u2193' },
+    { std::wstring(L"&harr;"), L'\u2194' },
+    { std::wstring(L"&crarr;"), L'\u21B5' },
+    { std::wstring(L"&lArr;"), L'\u21D0' },
+    { std::wstring(L"&uArr;"), L'\u21D1' },
+    { std::wstring(L"&rArr;"), L'\u21D2' },
+    { std::wstring(L"&dArr;"), L'\u21D3' },
+    { std::wstring(L"&hArr;"), L'\u21D4' },
+    { std::wstring(L"&forall;"), L'\u2200' },
+    { std::wstring(L"&part;"), L'\u2202' },
+    { std::wstring(L"&exist;"), L'\u2203' },
+    { std::wstring(L"&empty;"), L'\u2205' },
+    { std::wstring(L"&nabla;"), L'\u2207' },
+    { std::wstring(L"&isin;"), L'\u2208' },
+    { std::wstring(L"&notin;"), L'\u2209' },
+    { std::wstring(L"&ni;"), L'\u220B' },
+    { std::wstring(L"&prod;"), L'\u220F' },
+    { std::wstring(L"&sum;"), L'\u2211' },
+    { std::wstring(L"&minus;"), L'\u2212' },
+    { std::wstring(L"&lowast;"), L'\u2217' },
+    { std::wstring(L"&radic;"), L'\u221A' },
+    { std::wstring(L"&prop;"), L'\u221D' },
+    { std::wstring(L"&infin;"), L'\u221E' },
+    { std::wstring(L"&ang;"), L'\u2220' },
+    { std::wstring(L"&and;"), L'\u2227' },
+    { std::wstring(L"&or;"), L'\u2228' },
+    { std::wstring(L"&cap;"), L'\u2229' },
+    { std::wstring(L"&cup;"), L'\u222A' },
+    { std::wstring(L"&int;"), L'\u222B' },
+    { std::wstring(L"&there4;"), L'\u2234' },
+    { std::wstring(L"&sim;"), L'\u223C' },
+    { std::wstring(L"&cong;"), L'\u2245' },
+    { std::wstring(L"&asymp;"), L'\u2248' },
+    { std::wstring(L"&ne;"), L'\u2260' },
+    { std::wstring(L"&equiv;"), L'\u2261' },
+    { std::wstring(L"&le;"), L'\u2264' },
+    { std::wstring(L"&ge;"), L'\u2265' },
+    { std::wstring(L"&sub;"), L'\u2282' },
+    { std::wstring(L"&sup;"), L'\u2283' },
+    { std::wstring(L"&nsub;"), L'\u2284' },
+    { std::wstring(L"&sube;"), L'\u2286' },
+    { std::wstring(L"&supe;"), L'\u2287' },
+    { std::wstring(L"&oplus;"), L'\u2295' },
+    { std::wstring(L"&otimes;"), L'\u2297' },
+    { std::wstring(L"&perp;"), L'\u22A5' },
+    { std::wstring(L"&sdot;"), L'\u22C5' },
+    { std::wstring(L"&lceil;"), L'\u2308' },
+    { std::wstring(L"&rceil;"), L'\u2309' },
+    { std::wstring(L"&lfloor;"), L'\u230A' },
+    { std::wstring(L"&rfloor;"), L'\u230B' },
+    { std::wstring(L"&lang;"), L'\u2329' },
+    { std::wstring(L"&rang;"), L'\u232A' },
+    { std::wstring(L"&loz;"), L'\u25CA' },
+    { std::wstring(L"&spades;"), L'\u2660' },
+    { std::wstring(L"&clubs;"), L'\u2663' },
+    { std::wstring(L"&hearts;"), L'\u2665' },
+    { std::wstring(L"&diams;"), L'\u2666' }
+};
+
+inline gunichar
+get_entity(gunichar *ptr, size_t len) {
+    // try hex, decimal entity first
+    gunichar ech(0);
+    if (ptr[1] == gunichar(L'#') && len > 3) {
+        std::wstringstream wss;
+        int wch = 0;
+        try {
+            wss << std::hex << std::wstring((wchar_t *)(ptr+2),len-3);
+            wss >> wch;
+            ech = gunichar(wch);
+        } catch (...) {
+            ech = 0;
+        }
+    } else if (g_unichar_type(ptr[1]) == G_UNICODE_DECIMAL_NUMBER) {
+        std::wstringstream wss;
+        int wch = 0;
+        try {
+            wss << std::dec << std::wstring((wchar_t *)(ptr+1),len-2);
+            wss >> wch;
+            ech = gunichar(wch);
+        } catch (...) {
+            ech = 0;
+        }
+    }
+    if (ech)
+        return ech;
+
+    std::map<std::wstring,gunichar>::const_iterator it =
+        ENTITY_MAP.find(std::wstring((wchar_t *)(ptr),len));
+    return it != ENTITY_MAP.end() ? it->second : gunichar(0);
+}
+
+
+inline gunichar
+get_entity(char *ptr, size_t len) {
+    glong ulen = 0;
+    gunichar *gtmp = g_utf8_to_ucs4_fast((const gchar *)ptr, len, &ulen);
+    gunichar gch = get_entity(gtmp,ulen);
+    g_free(gtmp);
+    return gch;
+}
+
+
+inline std::string
+trim(const std::string& in)
+{
+    std::size_t start = 0;
+    std::size_t limit = in.size();
+    while (start < limit && in.at(start) < '!') ++start;
+    while (start < limit && in.at(limit-1) < '!') --limit;
+    if (start == limit) return std::string("");
+    if (start > 0 || limit < in.size())
+        return in.substr(start,limit-start);
+    return std::string(in);
+}
+
+
+inline std::vector<std::string>
+split(const std::string& in)
+{
+    std::vector<std::string> outv;
+    std::istringstream iss(in);
+    std::copy(std::istream_iterator<std::string>(iss),
+              std::istream_iterator<std::string>(),
+              std::back_inserter(outv));
+    return outv;
+}
+
+}; // end anonymous namespace
+
+
+#ifdef TOKENIZER_NAMESPACE
+namespace TOKENIZER_NAMESPACE {
+#endif
+
+
+void
+Tokenizer::set_config_dir(const std::string& dir) {
+    if (dir.empty()) {
+        cfg_dir = ".";
+    } else {
+        cfg_dir.assign(dir);
+    }
+}
+
+
+Tokenizer::Tokenizer(const Parameters& _)
+    : nthreads(_.nthreads ? _.nthreads : 1)
+    , chunksize(_.chunksize)
+    , lang_iso(_.lang_iso)
+    , english_p(_.lang_iso.compare("en")==0)
+    , latin_p((!english_p) && (_.lang_iso.compare("fr")==0 || _.lang_iso.compare("it")==0))
+    , skip_xml_p(_.detag_p)
+    , skip_alltags_p(_.alltag_p)
+    , entities_p(_.entities_p)
+    , escape_p(_.escape_p)
+    , unescape_p(_.unescape_p)
+    , aggressive_hyphen_p(_.aggro_p)
+    , supersub_p(_.supersub_p)
+    , url_p(_.url_p)
+    , downcase_p(_.downcase_p)
+    , normalize_p(_.normalize_p)
+    , penn_p(_.penn_p)
+    , narrow_latin_p(_.narrow_latin_p)
+    , narrow_kana_p(_.narrow_kana_p)
+    , refined_p(_.refined_p)
+    , drop_bad_p(_.drop_bad_p)
+    , splits_p(_.split_p)
+    , verbose_p(_.verbose_p)
+    , para_marks_p(_.para_marks_p)
+    , split_breaks_p(_.split_breaks_p)
+{
+    if (_.cfg_path)
+        set_config_dir(_.cfg_path);
+}
+
+
+//
+// dtor deletes dynamically allocated per-language RE2 compiled expressions
+//
+Tokenizer::~Tokenizer()
+{
+    for (auto& ptr : prot_pat_vec) {
+        if (ptr == &numprefixed_x || ptr == &quasinumeric_x)
+            continue;
+        delete ptr;
+    }
+}
+
+
+//
+// stuffs numeric-only prefixes into nbpre_num_set,
+// others into nbpre_gen_set
+//
+std::pair<int,int>
+Tokenizer::load_prefixes(std::ifstream& ifs)
+{
+    RE2 numonly("(.*)[\\s]+(\\#NUMERIC_ONLY\\#)");
+    std::string line;
+    int nnon = 0;
+    int nnum = 0;
+
+    while (std::getline(ifs,line)) {
+        if (!line.empty() && line[0] != '#') {
+            std::string prefix;
+            if (RE2::PartialMatch(line,numonly,&prefix)) {
+                nbpre_num_set.insert(prefix);
+                gunichar * x=g_utf8_to_ucs4_fast((const gchar *)prefix.c_str(),prefix.size(),0);
+                nbpre_num_ucs4.insert(std::wstring((wchar_t *)x));
+                g_free(x);
+                nnum++;
+            } else {
+                nbpre_gen_set.insert(line);
+                gunichar * x=g_utf8_to_ucs4_fast((const gchar *)line.c_str(),line.size(),0);
+                nbpre_gen_ucs4.insert(std::wstring((wchar_t *)x));
+                g_free(x);
+                nnon++;
+            }
+        }
+    }
+    return std::make_pair(nnon,nnum);
+}
+
+
+//
+// load files (make sure to call set_config_dir before, if ever
+// for nonbreaking prefixes and protected patterns
+//
+void
+Tokenizer::init(const char *cfg_dir_optional) {
+    if (cfg_dir_optional)
+        set_config_dir(std::string(cfg_dir_optional));
+
+    std::string dir_path(cfg_dir);
+    dir_path.append("/nonbreaking_prefixes");
+    if (::access(dir_path.c_str(),X_OK)) {
+        dir_path = cfg_dir;
+    }
+
+    std::string nbpre_path(dir_path);
+    nbpre_path.append("/nonbreaking_prefix.").append(lang_iso);
+
+    // default to generic version
+    if (::access(nbpre_path.c_str(),R_OK))
+        nbpre_path = nbpre_path.substr(0,nbpre_path.size()-lang_iso.size()-1);
+
+    if (::access(nbpre_path.c_str(),R_OK) == 0) {
+        std::ifstream cfg(nbpre_path.c_str());
+        try {
+            std::pair<int,int> counts = load_prefixes(cfg);
+            if (verbose_p) {
+                std::cerr << "loaded " << counts.first << " non-numeric, "
+                          << counts.second << " numeric prefixes from "
+                          << nbpre_path << std::endl;
+            }
+        } catch (...) {
+            std::ostringstream ess;
+            ess << "I/O error reading " << nbpre_path << " in " << __FILE__ << " at " << __LINE__;
+            throw std::runtime_error(ess.str());
+        }
+    } else if (verbose_p) {
+        std::cerr << "no prefix file found: " << nbpre_path << std::endl;
+    }
+
+    if (nbpre_gen_set.empty() && nbpre_num_set.empty()) {
+        std::ostringstream ess;
+        ess << "Error at " << __FILE__ << ":" << __LINE__ << " : "
+            << "No known abbreviations for language " << lang_iso;
+        throw std::runtime_error(ess.str());
+    }
+
+    std::string protpat_path(cfg_dir);
+    protpat_path.append("/protected_pattern.").append(lang_iso);
+    // default to generic version
+    if (::access(protpat_path.c_str(),R_OK))
+        protpat_path = protpat_path.substr(0,protpat_path.size()-lang_iso.size()-1);
+
+    prot_pat_vec.push_back(&numprefixed_x);
+    prot_pat_vec.push_back(&quasinumeric_x);
+
+    if (::access(protpat_path.c_str(),R_OK) == 0) {
+        std::ifstream cfg(protpat_path.c_str());
+        char linebuf[1028];
+        int npat = 0;
+        try {
+            linebuf[0]='(';
+            while (cfg.good()) {
+                cfg.getline(linebuf+1,1024);
+                if (linebuf[1] && linebuf[1] != '#') {
+                    strcat(linebuf,")");
+                    prot_pat_vec.push_back(new RE2(linebuf));
+                    npat++;
+                }
+            }
+        } catch (...) {
+            std::ostringstream ess;
+            ess << "I/O error reading " << protpat_path << " in " << __FILE__ << " at " << __LINE__;
+            throw std::runtime_error(ess.str());
+        }
+        if (verbose_p) {
+            std::cerr << "loaded " << npat << " protected patterns from "
+                      << protpat_path << std::endl;
+        }
+    } else if (verbose_p) {
+        std::cerr << "no protected file found: " << protpat_path << std::endl;
+    }
+}
+
+
+void
+Tokenizer::reset() {
+}
+
+
+//
+// apply ctor-selected tokenization to a string, in-place, no newlines allowed,
+// assumes protections are applied already, some invariants are in place,
+// e.g. that successive chars <= ' ' have been normalized to a single ' '
+//
+void
+Tokenizer::protected_tokenize(std::string& text) {
+    std::vector<re2::StringPiece> words;
+    re2::StringPiece textpc(text);
+    int pos = 0;
+    if (textpc[pos] == ' ')
+        ++pos;
+    size_t next = text.find(' ',pos);
+    while (next != std::string::npos) {
+        if (next - pos)
+            words.push_back(textpc.substr(pos,next-pos));
+        pos = next + 1;
+        while (pos < textpc.size() && textpc[pos] == ' ')
+            ++pos;
+        next = textpc.find(' ',pos);
+    }
+    if (pos < textpc.size() && textpc[pos] != ' ')
+        words.push_back(textpc.substr(pos,textpc.size()-pos));
+
+    // regurgitate words with look-ahead handling for tokens with final mumble
+    std::string outs;
+    std::size_t nwords(words.size());
+    for (size_t ii = 0; ii < nwords; ++ii) {
+        bool more_p = ii < nwords - 1;
+        size_t len = words[ii].size();
+        bool sentence_break_p = len > 1 && words[ii][len-1] == '.';
+
+        // suppress break if it is an non-breaking prefix
+        if (sentence_break_p) {
+            re2::StringPiece pfx(words[ii].substr(0,len-1));
+            std::string pfxs(pfx.as_string());
+            if (nbpre_gen_set.find(pfxs) != nbpre_gen_set.end()) {
+                // general non-breaking prefix
+                sentence_break_p = false;
+            } else if (more_p && nbpre_num_set.find(pfxs) != nbpre_num_set.end() && RE2::PartialMatch(words[ii+1],sinteger_x)) {
+                // non-breaking before numeric
+                sentence_break_p = false;
+            } else if (pfxs.find('.') != std::string::npos && RE2::PartialMatch(pfx,letter_x)) {
+                // terminal isolated letter does not break
+                sentence_break_p = false;
+            } else if (more_p && RE2::PartialMatch(words[ii+1],lower_x)) {
+                // lower-case look-ahead does not break
+                sentence_break_p = false;
+            }
+        }
+
+        outs.append(words[ii].data(),len);
+        if (sentence_break_p)
+            outs.append(" .");
+        if (more_p)
+            outs.append(SPC_BYTE,1);
+    }
+    text.assign(outs.begin(),outs.end());
+}
+
+
+bool
+Tokenizer::unescape(std::string& word) {
+    std::ostringstream oss;
+    std::size_t was = 0; // last processed
+    std::size_t pos = 0; // last unprocessed
+    std::size_t len = 0; // processed length
+    bool hit = false;
+    for (std::size_t endp=0;
+         (pos = word.find('&',was)) != std::string::npos && (endp = word.find(';',pos)) != std::string::npos;
+         was = endp == std::string::npos ? pos : 1+endp) {
+        len = endp - pos + 1;
+        glong ulen(0);
+        gunichar *gtmp = g_utf8_to_ucs4_fast((const gchar *)word.c_str()+pos, len, &ulen);
+        gunichar gbuf[2] = { 0 };
+        if ((gbuf[0] = get_entity(gtmp,ulen)) != gunichar(0)) {
+            gchar *gstr = g_ucs4_to_utf8(gbuf,ulen,0,0,0);
+            if (escape_p && ESCAPE_SET.find(std::string(gstr)) != ESCAPE_SET.end()) {
+                // do not unescape moses escapes when escape flag is turned on
+                oss << word.substr(was,1+endp-was);
+            } else {
+                if (was < pos)
+                    oss << word.substr(was,pos-was);
+                oss << gstr;
+                was += ulen;
+                hit = true;
+            }
+            g_free(gstr);
+        } else {
+            oss << word.substr(was,1+endp-was);
+        }
+        g_free(gtmp);
+    }
+    if (was < word.size())
+        oss << word.substr(was);
+    if (hit)
+        word = oss.str();
+    return hit;
+}
+
+
+bool
+Tokenizer::escape(std::string& text) {
+    bool mod_p = false;
+    std::string outs;
+
+    const char *pp = text.c_str(); // from pp to pt is uncopied
+    const char *ep = pp + text.size();
+    const  char *pt = pp;
+
+    while (pt < ep) {
+        if (*pt & 0x80) {
+            const char *mk = (const char *)g_utf8_find_next_char((const gchar *)pt,(const gchar *)ep);
+            if (!mk) {
+                if (mod_p)
+                    outs.append(pp,pt-pp+1);
+            } else {
+                if (mod_p)
+                    outs.append(pp,mk-pp);
+                pt = --mk;
+            }
+            pp = ++pt;
+            continue;
+        }
+
+        const char *sequence_p = 0;
+        if (*pt < '?') {
+            if (*pt == '&') {
+                // check for a pre-existing escape
+                const char *sc = strchr(pt,';');
+                if (!sc || sc-pt < 2 || sc-pt > 9) {
+                    sequence_p = ESCAPE_MOSES[3];
+                }
+            } else if (*pt == '\'') {
+                sequence_p = ESCAPE_MOSES[6];
+            } else if (*pt == '"') {
+                sequence_p = ESCAPE_MOSES[7];
+            }
+        } else if (*pt > ']') {
+            if (*pt =='|') { // 7c
+                sequence_p = ESCAPE_MOSES[0];
+            }
+        } else if (*pt > 'Z') {
+            if (*pt == '<') { // 3e
+                sequence_p = ESCAPE_MOSES[4];
+            } else if (*pt == '>') { // 3c
+                sequence_p = ESCAPE_MOSES[5];
+            } else if (*pt == '[') { // 5b
+                sequence_p = ESCAPE_MOSES[1];
+            } else if (*pt == ']') { // 5d
+                sequence_p = ESCAPE_MOSES[2];
+            }
+        }
+
+        if (sequence_p) {
+            if (pt > pp)
+                outs.append(pp,pt-pp);
+            outs.append(sequence_p);
+            mod_p = true;
+            pp = ++pt;
+        } else {
+            ++pt;
+        }
+    }
+
+    if (mod_p) {
+        if (pp < pt) {
+            outs.append(pp,pt-pp);
+        }
+        text.assign(outs.begin(),outs.end());
+    }
+
+    return mod_p;
+}
+
+
+std::string
+Tokenizer::penn_tokenize(const std::string& buf)
+{
+    static const char *comma_refs = "\\1 , \\2";
+    static const char *isolate_ref = " \\1 ";
+    static const char *special_refs = "\\1 @\\2@ \\3";
+
+    std::string text(buf);
+    std::string outs;
+    if (skip_alltags_p)
+        RE2::GlobalReplace(&text,genl_tags_x,SPC_BYTE);
+
+    // directed quote patches
+    size_t len = text.size();
+    if (len > 2 && text.substr(0,2) == "``")
+        text.replace(0,2,"`` ",3);
+    else if (text[0] == '"')
+        text.replace(0,1,"`` ",3);
+    else if (text[0] == '`' || text[0] == '\'')
+        text.replace(0,1,"` ",2);
+    static char one_gg[] = "\\1 ``";
+    RE2::GlobalReplace(&text,x1_v_d,one_gg);
+    RE2::GlobalReplace(&text,x1_v_gg,one_gg);
+    RE2::GlobalReplace(&text,x1_v_g,"\\1 ` \\2");
+    RE2::GlobalReplace(&text,x1_v_q,"\\1 ` ");
+
+    // protect ellipsis
+    for (size_t pos = text.find("..."); pos != std::string::npos; pos = text.find("...",pos+11))
+        text.replace(pos,3,"MANYELIPSIS",11);
+
+    // numeric commas
+    RE2::GlobalReplace(&text,ndndcomma_x,comma_refs);
+    RE2::GlobalReplace(&text,pdndcomma_x,comma_refs);
+    RE2::GlobalReplace(&text,ndpdcomma_x,comma_refs);
+
+    // isolable symbols
+    RE2::GlobalReplace(&text,symbol_x,isolate_ref);
+
+    // isolable slash
+    RE2::GlobalReplace(&text,slash_x,special_refs);
+
+    // isolate final period
+    RE2::GlobalReplace(&text,final_x,"\\1 \\2\\3");
+
+    // isolate q.m., e.m.
+    RE2::GlobalReplace(&text,qx_x,isolate_ref);
+
+    // isolate braces
+    RE2::GlobalReplace(&text,braces_x,isolate_ref);
+
+    // convert open/close punctuation
+    RE2::GlobalReplace(&text,"\\(","-LRB-");
+    RE2::GlobalReplace(&text,"\\[","-LSB-");
+    RE2::GlobalReplace(&text,"\\{","-LCB-");
+    RE2::GlobalReplace(&text,"\\)","-RRB-");
+    RE2::GlobalReplace(&text,"\\]","-RSB-");
+    RE2::GlobalReplace(&text,"\\}","-RCB-");
+
+    // isolate double-dash hyphen
+    RE2::GlobalReplace(&text,"--"," -- ");
+
+    // insure leading and trailing space on line, to simplify exprs
+    // also make sure final . has one space on each side
+    len = text.size();
+    while (len > 1 && text[len-1] == ' ') --len;
+    if (len < text.size())
+        text.assign(text.substr(0,len));
+    if (len > 2 && text[len-1] == '.') {
+        if (text[len-2] != ' ') {
+            text.assign(text.substr(0,len-1));
+            text.append(" . ");
+        } else {
+            text.assign(text.substr(0,len-1));
+            text.append(". ");
+        }
+    } else {
+        text.append(SPC_BYTE,1);
+    }
+    std::string ntext(SPC_BYTE);
+    ntext.append(text);
+
+    // convert double quote to paired single-quotes
+    RE2::GlobalReplace(&ntext,"\""," '' ");
+
+    // deal with contractions in penn style
+    RE2::GlobalReplace(&ntext,endq_x,"\\1 ' ");
+    RE2::GlobalReplace(&ntext,contract_x," '\\1 ");
+    RE2::GlobalReplace(&ntext,"'ll "," 'll ");
+    RE2::GlobalReplace(&ntext,"'re "," 're ");
+    RE2::GlobalReplace(&ntext,"'ve "," 've ");
+    RE2::GlobalReplace(&ntext,"n't "," n't ");
+    RE2::GlobalReplace(&ntext,"'LL "," 'LL ");
+    RE2::GlobalReplace(&ntext,"'RE "," 'RE ");
+    RE2::GlobalReplace(&ntext,"'VE "," 'VE ");
+    RE2::GlobalReplace(&ntext,"N'T "," N'T ");
+    RE2::GlobalReplace(&ntext," ([Cc])annot "," \\1an not ");
+    RE2::GlobalReplace(&ntext," ([Dd])'ye "," \\1' ye ");
+    RE2::GlobalReplace(&ntext," ([Gg])imme "," \\1im me ");
+    RE2::GlobalReplace(&ntext," ([Gg])onna "," \\1on na ");
+    RE2::GlobalReplace(&ntext," ([Gg])otta "," \\1ot ta ");
+    RE2::GlobalReplace(&ntext," ([Ll])emme "," \\1em me ");
+    RE2::GlobalReplace(&ntext," ([Mm])ore'n "," \\1ore 'n ");
+    RE2::GlobalReplace(&ntext," '([Tt])is "," '\\1 is 'n ");
+    RE2::GlobalReplace(&ntext," '([Tt])was "," '\\1 was 'n ");
+    RE2::GlobalReplace(&ntext," '([Tt])were "," '\\1 were 'n ");
+    RE2::GlobalReplace(&ntext," ([Ww])anna "," \\1an na ");
+
+    protected_tokenize(ntext);
+
+    // restore ellipsis
+    RE2::GlobalReplace(&ntext,"MANYELIPSIS","...");
+
+    // collapse spaces
+    RE2::GlobalReplace(&ntext,mult_spc_x,SPC_BYTE);
+
+    // escape moses meta-characters
+    if (escape_p)
+        escape(ntext);
+
+    // strip out wrapping spaces from line in result string
+    outs.assign(ntext.substr(1,ntext.size()-2));
+    return outs;
+}
+
+
+std::string
+Tokenizer::quik_tokenize(const std::string& buf)
+{
+    std::string text(buf);
+    size_t pos;
+    int num = 0;
+
+    // this is the main moses-compatible tokenizer
+
+    // push all the prefixes matching protected patterns
+    std::vector<std::string> prot_stack;
+    std::string match;
+
+    for (auto& pat : prot_pat_vec) {
+        pos = 0;
+        while (RE2::PartialMatch(text.substr(pos),*pat,&match)) {
+            pos = text.find(match,pos);
+            if (pos == std::string::npos)
+                break;
+            size_t len = match.size();
+            if (text[pos-1] == ' ' || text[pos-1] == '\'' || text[pos-1] == '`'|| text[pos-1] == '"') {
+                char subst[32];
+                int nsubst = snprintf(subst,sizeof(subst)," THISISPROTECTED%.3d ",num++);
+                text.replace(pos,len,subst,nsubst);
+                prot_stack.push_back(match);
+                pos += nsubst;
+            } else {
+                pos += len;
+            }
+        }
+    }
+
+    const char *pt(text.c_str());
+    const char *ep(pt + text.size());
+    while (pt < ep && *pt >= 0 && *pt <= ' ')
+        ++pt;
+    glong ulen(0);
+    gunichar *usrc(g_utf8_to_ucs4_fast((const gchar *)pt,ep - pt, &ulen)); // g_free
+    gunichar *ucs4(usrc);
+    gunichar *lim4(ucs4 + ulen);
+
+    gunichar *nxt4 = ucs4;
+    gunichar *ubuf(g_new0(gunichar,ulen*6+1)); // g_free
+    gunichar *uptr(ubuf);
+
+    gunichar prev_uch(0);
+    gunichar next_uch(*ucs4);
+    gunichar curr_uch(0);
+
+    GUnicodeType curr_type(G_UNICODE_UNASSIGNED);
+    GUnicodeType next_type((ucs4 && *ucs4) ? g_unichar_type(*ucs4) : G_UNICODE_UNASSIGNED);
+    GUnicodeType prev_type(G_UNICODE_UNASSIGNED);
+
+    bool post_break_p = false;
+    bool in_num_p = next_uch <= gunichar(L'9') && next_uch >= gunichar(L'0');
+    bool in_url_p = false;
+    int since_start = 0;
+    int alpha_prefix = 0;
+    int bad_length = 0;
+
+    while (ucs4 < lim4) {
+        prev_uch = curr_uch;
+        prev_type = curr_type;
+        curr_uch = next_uch;
+        curr_type = next_type;
+
+        if (++nxt4 >= lim4) {
+            next_uch = 0;
+            next_type = G_UNICODE_UNASSIGNED;
+        } else {
+            next_uch = *nxt4;
+            next_type = g_unichar_type(next_uch);
+        }
+
+        if (url_p) {
+            if (!in_url_p && *ucs4 < 0x80L) { // url chars must be in the basic plane
+                if (!since_start) {
+                    if (std::isalpha(char(*ucs4)))
+                        alpha_prefix++;
+                } else if (alpha_prefix == since_start
+                           && char(*ucs4) == ':'
+                           && next_type != G_UNICODE_SPACE_SEPARATOR) {
+                    in_url_p = true;
+                }
+            }
+        }
+
+        bool pre_break_p = false;
+        const wchar_t *substitute_p = 0;
+
+        if (post_break_p) {
+            *uptr++ = gunichar(L' ');
+            since_start = bad_length = 0;
+            in_url_p = in_num_p = post_break_p = false;
+        }
+
+    retry:
+
+        switch (curr_type) {
+        case G_UNICODE_MODIFIER_LETTER:
+        case G_UNICODE_OTHER_LETTER:
+        case G_UNICODE_TITLECASE_LETTER:
+            if (in_url_p || in_num_p)
+                pre_break_p = true;
+            // fallthough
+        case G_UNICODE_UPPERCASE_LETTER:
+        case G_UNICODE_LOWERCASE_LETTER:
+            if (downcase_p && curr_type == G_UNICODE_UPPERCASE_LETTER)
+                curr_uch = g_unichar_tolower(*ucs4);
+            break;
+        case G_UNICODE_SPACING_MARK:
+            pre_break_p = true;
+            in_num_p = false;
+            curr_uch = 0;
+            break;
+        case G_UNICODE_DECIMAL_NUMBER:
+        case G_UNICODE_LETTER_NUMBER:
+        case G_UNICODE_OTHER_NUMBER:
+            if (!in_num_p && !in_url_p) {
+                switch (prev_type) {
+                case G_UNICODE_DASH_PUNCTUATION:
+                case G_UNICODE_FORMAT:
+                case G_UNICODE_OTHER_PUNCTUATION:
+                case G_UNICODE_UPPERCASE_LETTER:
+                case G_UNICODE_LOWERCASE_LETTER:
+                case G_UNICODE_DECIMAL_NUMBER:
+                    break;
+                default:
+                    pre_break_p = true;
+                }
+            }
+            in_num_p = true;
+            break;
+        case G_UNICODE_CONNECT_PUNCTUATION:
+            if (curr_uch != gunichar(L'_')) {
+                if (in_url_p) {
+                    in_url_p = false;
+                    post_break_p = pre_break_p = true;
+                }
+            }
+            if (in_num_p) {
+                post_break_p = pre_break_p = true;
+            } else {
+                switch (next_type) {
+                case G_UNICODE_LOWERCASE_LETTER:
+                case G_UNICODE_MODIFIER_LETTER:
+                case G_UNICODE_OTHER_LETTER:
+                case G_UNICODE_TITLECASE_LETTER:
+                    break;
+                default:
+                    post_break_p = pre_break_p = true;
+                }
+                switch (prev_type) {
+                case G_UNICODE_LOWERCASE_LETTER:
+                case G_UNICODE_MODIFIER_LETTER:
+                case G_UNICODE_OTHER_LETTER:
+                case G_UNICODE_TITLECASE_LETTER:
+                    break;
+                default:
+                    post_break_p = pre_break_p = true;
+                }
+            }
+            break;
+        case G_UNICODE_FORMAT:
+            in_url_p = in_num_p = false;
+            break;
+        case G_UNICODE_DASH_PUNCTUATION:
+            if (aggressive_hyphen_p && !in_url_p && curr_uch != next_uch && prev_uch != curr_uch && (!(prev_uch == L' ' || !prev_uch) && !(next_uch == L' ' || !next_uch))) {
+                substitute_p = L"@-@";
+                post_break_p = pre_break_p = true;
+            } else if ( ( curr_uch > gunichar(L'\u002D') && curr_uch < gunichar(L'\u2010') ) ||
+                        ( curr_uch > gunichar(L'\u2011')
+                          && curr_uch != gunichar(L'\u30A0')
+                          && curr_uch < gunichar(L'\uFE63') ) ) {
+                // dash, not a hyphen
+                post_break_p = pre_break_p = true;
+            } else if (next_type == G_UNICODE_SPACE_SEPARATOR) {
+            } else {
+                if (prev_type == curr_type) {
+                    if (next_type != curr_type) {
+                        post_break_p = !in_url_p;
+                    }
+                } else if (next_type == curr_type) {
+                    pre_break_p = !in_url_p;
+                } else if ((prev_type == G_UNICODE_UPPERCASE_LETTER ||
+                            prev_type == G_UNICODE_LOWERCASE_LETTER) &&
+                           next_type == G_UNICODE_DECIMAL_NUMBER) {
+                    in_num_p = false;
+                } else if (in_num_p || since_start == 0) {
+                    switch (next_type) {
+                    case G_UNICODE_UPPERCASE_LETTER:
+                    case G_UNICODE_LOWERCASE_LETTER:
+                    case G_UNICODE_MODIFIER_LETTER:
+                    case G_UNICODE_OTHER_LETTER:
+                    case G_UNICODE_TITLECASE_LETTER:
+                    case G_UNICODE_SPACE_SEPARATOR:
+                        in_num_p = false;
+                        break;
+                    case G_UNICODE_DECIMAL_NUMBER:
+                    case G_UNICODE_LETTER_NUMBER:
+                    case G_UNICODE_OTHER_NUMBER:
+                    case G_UNICODE_OTHER_PUNCTUATION:
+                        break;
+                    default:
+                        post_break_p = true;
+                        pre_break_p = prev_uch != curr_uch;
+                    }
+                } else if (in_url_p) {
+                    pre_break_p = curr_uch != gunichar(L'-');
+                } else {
+                    switch (prev_type) {
+                    case G_UNICODE_UPPERCASE_LETTER:
+                    case G_UNICODE_LOWERCASE_LETTER:
+                    case G_UNICODE_MODIFIER_LETTER:
+                    case G_UNICODE_OTHER_LETTER:
+                    case G_UNICODE_TITLECASE_LETTER:
+                    case G_UNICODE_DECIMAL_NUMBER:
+                    case G_UNICODE_LETTER_NUMBER:
+                    case G_UNICODE_OTHER_NUMBER:
+                    case G_UNICODE_OTHER_PUNCTUATION:
+                        switch (next_type) {
+                        case G_UNICODE_UPPERCASE_LETTER:
+                        case G_UNICODE_LOWERCASE_LETTER:
+                        case G_UNICODE_MODIFIER_LETTER:
+                        case G_UNICODE_OTHER_LETTER:
+                        case G_UNICODE_TITLECASE_LETTER:
+                        case G_UNICODE_DECIMAL_NUMBER:
+                        case G_UNICODE_LETTER_NUMBER:
+                        case G_UNICODE_OTHER_NUMBER:
+                            break;
+                        case G_UNICODE_OTHER_PUNCTUATION:
+                            if (prev_type != next_type)
+                                break;
+                        default:
+                            post_break_p = pre_break_p = prev_uch != curr_uch;
+                        }
+                        break;
+                    default:
+                        post_break_p = pre_break_p = prev_uch != curr_uch;
+                        break;
+                    }
+                }
+            }
+            break;
+        case G_UNICODE_OTHER_PUNCTUATION:
+            switch (curr_uch) {
+            case gunichar(L':'):
+            case gunichar(L'/'):
+                if (refined_p && !in_url_p
+                    && prev_type == G_UNICODE_DECIMAL_NUMBER
+                    && next_type == G_UNICODE_DECIMAL_NUMBER) {
+                    break;
+                }
+            // fall-through
+            case gunichar(L'!'):
+            case gunichar(L'#'):
+            case gunichar(L';'):
+            case gunichar(L'?'):
+            case gunichar(L'@'):
+                post_break_p = pre_break_p = !in_url_p || next_type != G_UNICODE_SPACE_SEPARATOR;
+            break;
+            case gunichar(L'+'):
+                post_break_p = pre_break_p = !in_num_p && since_start > 0;
+                in_num_p = in_num_p || since_start == 0;
+                break;
+            case gunichar(L'&'):
+                if (unescape_p) {
+                    if (next_type == G_UNICODE_LOWERCASE_LETTER || next_type == G_UNICODE_UPPERCASE_LETTER
+                        || next_type == G_UNICODE_DECIMAL_NUMBER || next_uch == gunichar(L'#')) {
+                        gunichar *eptr = nxt4;
+                        GUnicodeType eptr_type(G_UNICODE_UNASSIGNED);
+                        for (++eptr; eptr < lim4 && *eptr != gunichar(L';'); ++eptr) {
+                            eptr_type = g_unichar_type(*eptr);
+                            if (eptr_type != G_UNICODE_LOWERCASE_LETTER
+                                && eptr_type != G_UNICODE_UPPERCASE_LETTER
+                                && eptr_type != G_UNICODE_DECIMAL_NUMBER)
+                                break;
+                        }
+                        gunichar ech(0);
+                        if (*eptr == gunichar(L';') && (ech = get_entity(ucs4,eptr-ucs4+1))) {
+                            curr_uch = ech;
+                            curr_type = g_unichar_type(ech);
+                            ucs4 = eptr;
+                            nxt4 = ++eptr;
+                            next_uch = *nxt4;
+                            next_type = nxt4 < lim4 ? g_unichar_type(next_uch) : G_UNICODE_UNASSIGNED;
+                            goto retry;
+                        }
+                    }
+                }
+                if (entities_p && !in_url_p) {
+                    gunichar *cur4 = nxt4;
+                    if (*cur4 == gunichar('#')) ++cur4;
+                    while (g_unichar_isalnum(*cur4)) ++cur4;
+                    if (cur4 > nxt4 && *cur4 == gunichar(';')) {
+                        if (since_start) {
+                            *uptr++ = gunichar(L' ');
+                            since_start = 0;
+                        }
+                        ++cur4;
+                        memcpy(uptr,ucs4,cur4-ucs4);
+                        uptr += cur4-ucs4;
+                        ucs4 = cur4;
+                        *uptr++ = gunichar(L' ');
+                        pre_break_p = post_break_p = false;
+                        curr_uch = *ucs4;
+                        curr_type = ucs4 < lim4 ? g_unichar_type(curr_uch) : G_UNICODE_UNASSIGNED;
+                        nxt4 = ++cur4;
+                        next_uch = *nxt4;
+                        next_type = nxt4 < lim4 ? g_unichar_type(next_uch) : G_UNICODE_UNASSIGNED;
+                        goto retry;
+                    }
+
+                }
+                post_break_p = pre_break_p = !in_url_p || next_type != G_UNICODE_SPACE_SEPARATOR;
+                if (escape_p)
+                    substitute_p = L"&amp;";
+                break;
+            case gunichar(L'\''):
+                if (english_p) {
+                    if (!in_url_p) {
+                        bool next_letter_p = next_type == G_UNICODE_LOWERCASE_LETTER
+                            || next_type == G_UNICODE_UPPERCASE_LETTER;
+                        pre_break_p = true;
+                        if (next_letter_p && refined_p) {
+                            // break sha n't instead of shan 't:
+                            if (prev_uch == gunichar(L'n') || prev_uch == gunichar(L'N')) {
+                                *(uptr - 1) = gunichar(L' ');
+                                *(uptr++) = prev_uch;
+                                pre_break_p = false;
+                            }
+                        }
+                        post_break_p = since_start == 0
+                            || (!next_letter_p && next_type != G_UNICODE_DECIMAL_NUMBER);
+                    }
+                } else if (latin_p) {
+                    post_break_p = !in_url_p;
+                    pre_break_p = !in_url_p && prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
+                } else {
+                    post_break_p = pre_break_p = !in_url_p;
+                }
+                if (escape_p)
+                    substitute_p = L"&apos;";
+                break;
+            case gunichar(L'"'):
+                post_break_p = pre_break_p = true;
+                if (escape_p)
+                    substitute_p = L"&quot;";
+                break;
+            case gunichar(L','):
+                pre_break_p = !in_num_p || next_type != G_UNICODE_DECIMAL_NUMBER;
+                post_break_p = !in_num_p && next_type != G_UNICODE_DECIMAL_NUMBER;
+                break;
+            case gunichar(L'%'):
+                if (refined_p) {
+                    pre_break_p = !in_num_p;
+                    post_break_p = !in_num_p && next_type != G_UNICODE_DECIMAL_NUMBER;
+                } else {
+                    post_break_p = pre_break_p = true;
+                }
+                break;
+            case gunichar(L'.'):
+                if (prev_uch != '.') {
+                    if (!in_num_p) {
+                        switch (next_type) {
+                        case G_UNICODE_DECIMAL_NUMBER:
+                        case G_UNICODE_LOWERCASE_LETTER:
+                        case G_UNICODE_UPPERCASE_LETTER:
+                            break;
+                        default:
+                            if (since_start > 0) {
+                                switch (prev_type) {
+                                case G_UNICODE_LOWERCASE_LETTER:
+                                case G_UNICODE_UPPERCASE_LETTER: {
+                                    std::wstring k((wchar_t *)(uptr-since_start),since_start);
+                                    if (nbpre_gen_ucs4.find(k) != nbpre_gen_ucs4.end()) {
+                                        // general non-breaking prefix
+                                    } else if (nbpre_num_ucs4.find(k) != nbpre_num_ucs4.end() && class_follows_p(nxt4,lim4,G_UNICODE_DECIMAL_NUMBER)) {
+                                        // non-breaking before numeric
+                                    } else if (k.find(curr_uch) != std::wstring::npos) {
+                                        if (since_start > 1) {
+                                            GUnicodeType tclass = g_unichar_type(*(uptr-2));
+                                            switch (tclass) {
+                                            case G_UNICODE_UPPERCASE_LETTER:
+                                            case G_UNICODE_LOWERCASE_LETTER:
+                                                pre_break_p = true;
+                                                break;
+                                            default:
+                                                break;
+                                            }
+                                        }
+                                        // terminal isolated letter does not break
+                                    } else if (class_follows_p(nxt4,lim4,G_UNICODE_LOWERCASE_LETTER) ||
+                                               g_unichar_type(*nxt4) == G_UNICODE_DASH_PUNCTUATION) {
+                                        // lower-case look-ahead does not break
+                                    } else {
+                                        pre_break_p = true;
+                                    }
+                                    break;
+                                }
+                                default:
+                                    pre_break_p = true;
+                                    break;
+                                }
+                            }
+                            break;
+                        }
+                    } else {
+                        switch (next_type) {
+                        case G_UNICODE_DECIMAL_NUMBER:
+                        case G_UNICODE_LOWERCASE_LETTER:
+                        case G_UNICODE_UPPERCASE_LETTER:
+                            break;
+                        default:
+                            pre_break_p = true;
+                        }
+                    }
+                } else if (next_uch != '.') {
+                    post_break_p = true;
+                }
+                break;
+            default:
+                post_break_p = pre_break_p = true;
+                break;
+            }
+            break;
+        case G_UNICODE_CLOSE_PUNCTUATION:
+        case G_UNICODE_FINAL_PUNCTUATION:
+        case G_UNICODE_INITIAL_PUNCTUATION:
+        case G_UNICODE_OPEN_PUNCTUATION:
+            switch (curr_uch) {
+            case gunichar(L'('):
+            case gunichar(L')'):
+                break;
+            case gunichar(L'['):
+                if (escape_p)
+                    substitute_p = L"&#91;";
+                break;
+            case gunichar(L']'):
+                if (escape_p)
+                    substitute_p = L"&#93;";
+                break;
+            default:
+                in_url_p = false;
+            }
+            post_break_p = pre_break_p = !in_url_p;
+            break;
+        case G_UNICODE_CURRENCY_SYMBOL:
+            if (refined_p) {
+                post_break_p = in_num_p; // was in number, so break it
+                pre_break_p = !in_num_p;
+                in_num_p = in_num_p || next_type == G_UNICODE_DECIMAL_NUMBER || next_uch == gunichar(L'.') || next_uch == gunichar(L',');
+            } else {
+                post_break_p = pre_break_p = true;
+                in_num_p = false;
+            }
+            if (curr_uch != gunichar(L'$'))
+                in_url_p = false;
+            break;
+        case G_UNICODE_MODIFIER_SYMBOL:
+        case G_UNICODE_MATH_SYMBOL:
+            switch (curr_uch) {
+            case gunichar(L'`'):
+                if (english_p) {
+                    if (!in_url_p) {
+                        pre_break_p = true;
+                        post_break_p = since_start == 0 ||
+                            (next_type != G_UNICODE_LOWERCASE_LETTER && next_type != G_UNICODE_UPPERCASE_LETTER && next_type != G_UNICODE_DECIMAL_NUMBER);
+                    }
+                } else if (latin_p) {
+                    post_break_p = !in_url_p;
+                    pre_break_p = !in_url_p && prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
+                } else {
+                    post_break_p = pre_break_p = !in_url_p;
+                }
+                if (escape_p)
+                    substitute_p = L"&apos;";
+                else
+                    curr_uch = gunichar(L'\'');
+                break;
+            case gunichar(L'|'):
+                if (escape_p)
+                    substitute_p = L"&#124;";
+                post_break_p = pre_break_p = true;
+                break;
+            case gunichar(L'<'):
+                if (escape_p)
+                    substitute_p = L"&lt;";
+                post_break_p = pre_break_p = true;
+                break;
+            case gunichar(L'>'):
+                if (escape_p)
+                    substitute_p = L"&gt;";
+                post_break_p = pre_break_p = true;
+                break;
+            case gunichar(L'%'):
+                post_break_p = in_num_p;
+                pre_break_p = !in_num_p && !in_url_p;
+                in_num_p = false;
+                break;
+            case gunichar(L'='):
+            case gunichar(L'~'):
+                in_num_p = false;
+            post_break_p = pre_break_p = !in_url_p;
+            break;
+            case gunichar(L'+'):
+                post_break_p = pre_break_p = !in_url_p;
+                if (in_url_p) {
+                    in_num_p = false;
+                } else if (refined_p) {
+                    // handle floating point as e.g. 1.2e+3.4
+                    bool next_digit_p = next_type == G_UNICODE_DECIMAL_NUMBER ||
+                        next_uch == gunichar(L'.');
+                    pre_break_p = !in_num_p;
+                    in_num_p = next_digit_p && prev_type != G_UNICODE_DECIMAL_NUMBER;
+                    post_break_p = !in_num_p;
+                } else {
+                    in_num_p = in_num_p || since_start == 0;
+                }
+                break;
+            default:
+                post_break_p = pre_break_p = true;
+                break;
+            }
+            break;
+        case G_UNICODE_OTHER_SYMBOL:
+            post_break_p = pre_break_p = true;
+            break;
+        case G_UNICODE_CONTROL:
+            if (drop_bad_p) {
+                curr_uch = gunichar(L' ');
+            } else if (curr_uch < gunichar(L' ')) {
+                curr_uch = gunichar(L' ');
+            } else if (curr_uch == gunichar(L'\u0092') &&
+                       (next_type == G_UNICODE_LOWERCASE_LETTER || next_type == G_UNICODE_UPPERCASE_LETTER)) {
+                // observed corpus corruption case
+                if (english_p) {
+                    pre_break_p = true;
+                    post_break_p = since_start == 0 ||
+                        (next_type != G_UNICODE_LOWERCASE_LETTER && next_type != G_UNICODE_UPPERCASE_LETTER && next_type != G_UNICODE_DECIMAL_NUMBER);
+                } else if (latin_p) {
+                    post_break_p = true;
+                    pre_break_p = prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
+                } else {
+                    post_break_p = pre_break_p = true;
+                }
+                if (escape_p)
+                    substitute_p = L"&apos;";
+                else
+                    curr_uch = gunichar(L'\'');
+            } else {
+                post_break_p = pre_break_p = true;
+            }
+            in_url_p = in_num_p = false;
+            break;
+        case G_UNICODE_LINE_SEPARATOR:
+        case G_UNICODE_SPACE_SEPARATOR:
+            curr_uch = gunichar(L' ');
+            in_url_p = in_num_p = false;
+            break;
+        case G_UNICODE_ENCLOSING_MARK:
+            in_url_p = false;
+            break;
+        case G_UNICODE_NON_SPACING_MARK:
+        case G_UNICODE_PRIVATE_USE:
+        case G_UNICODE_SURROGATE:
+            in_url_p = in_num_p = false;
+            break;
+        case G_UNICODE_UNASSIGNED:
+        default:
+            // malformed bytes are dropped (invalid utf8 unicode)
+            if (drop_bad_p) {
+                curr_uch = 0;
+            } else {
+                pre_break_p = since_start > 0 && bad_length == 0;
+                curr_type = G_UNICODE_UNASSIGNED;
+            }
+            in_url_p = in_num_p = false;
+            break;
+        }
+
+        if (pre_break_p || curr_uch == gunichar(L' ') || (bad_length && curr_type != G_UNICODE_UNASSIGNED)) {
+            if (since_start) {
+                // non-empty token emitted previously, so pre-break must emit token separator
+                *uptr++ = gunichar(L' ');
+                since_start = bad_length = 0;
+            }
+            if (curr_uch == gunichar(L' '))
+                // suppress emission below, fall-through to substitute logic
+                curr_uch = 0;
+        }
+
+        if (substitute_p) {
+            for (gunichar *sptr = (gunichar *)substitute_p; *sptr; ++sptr) {
+                *uptr++ = *sptr;
+                since_start++;
+            }
+            in_url_p = in_num_p = false;
+        } else if (curr_uch) {
+            *uptr++ = curr_uch;
+            since_start++;
+            if (curr_type == G_UNICODE_UNASSIGNED)
+                bad_length++;
+        }
+
+        ucs4 = nxt4;
+    }
+
+    glong nbytes = 0;
+    gchar *utf8 = g_ucs4_to_utf8(ubuf,uptr-ubuf,0,&nbytes,0); // g_free
+    if (utf8[nbytes-1] == ' ')
+        --nbytes;
+    text.assign((const char *)utf8,(const char *)(utf8 + nbytes));
+    g_free(utf8);
+    g_free(usrc);
+    g_free(ubuf);
+
+    // terminate token at superscript or subscript sequence when followed by lower-case
+    if (supersub_p)
+        RE2::GlobalReplace(&text,numscript_x,"\\1\\2 \\3");
+
+    // restore prefix-protected strings
+    num = 0;
+    for (auto& prot : prot_stack) {
+        char subst[32];
+        snprintf(subst,sizeof(subst),"THISISPROTECTED%.3d",num++);
+        size_t loc = text.find(subst);
+        while (loc != std::string::npos) {
+            text.replace(loc,18,prot.data(),prot.size());
+            loc = text.find(subst,loc+18);
+        }
+    }
+
+    // escape moses meta-characters
+    if (escape_p)
+        escape(text);
+
+    return text;
+}
+
+
+std::size_t
+Tokenizer::tokenize(std::istream& is, std::ostream& os)
+{
+    std::size_t line_no = 0;
+    std::size_t perchunk = chunksize ? chunksize : 2000;
+    std::vector< std::vector< std::string > > lines(nthreads);
+    std::vector< std::vector< std::string > > results(nthreads);
+    std::vector< boost::thread > workers(nthreads);
+    bool done_p = !(is.good() && os.good());
+
+
+    for (std::size_t tranche = 0; !done_p; ++tranche) {
+
+        // for loop starting threads for chunks of input
+        for (std::size_t ithread = 0; ithread < nthreads; ++ithread) {
+
+            lines[ithread].resize(perchunk);
+            std::size_t line_pos = 0;
+
+            for ( ; line_pos < perchunk; ++line_pos) {
+
+                std::string istr;
+                std::getline(is,istr);
+
+                if (skip_alltags_p) {
+                    RE2::GlobalReplace(&istr,genl_tags_x,SPC_BYTE);
+                    istr = trim(istr);
+                }
+                line_no++;
+
+                if (istr.empty()) {
+                    if (is.eof()) {
+                        done_p = true;
+                        lines[ithread].resize(line_pos);
+                        results[ithread].resize(line_pos);
+                        break;
+                    }
+                    lines[ithread][line_pos].clear();
+                } else if (skip_xml_p &&
+                           (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) {
+                    lines[ithread][line_pos].clear();
+                } else {
+                    lines[ithread][line_pos] =
+                        std::string(SPC_BYTE).append(istr).append(SPC_BYTE);
+                }
+            }
+
+            if (line_pos) {
+                workers[ithread] =
+                    boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread]));
+            }
+        } // end for loop starting threads
+
+        for (std::size_t ithread = 0; ithread < nthreads; ++ithread) {
+            if (!workers[ithread].joinable())
+                continue;
+
+            workers[ithread].join();
+
+            std::size_t nres = results[ithread].size();
+            std::size_t nlin = lines[ithread].size();
+
+            if (nlin != nres) {
+                std::ostringstream emsg;
+                emsg << "Tranche " << tranche
+                     << " worker " << ithread << "/" << nthreads
+                     << " |lines|==" << nlin << " != |results|==" << nres;
+                throw std::runtime_error(emsg.str());
+            }
+
+            for (std::size_t ires = 0; ires < nres; ++ires)
+                os << results[ithread][ires] << std::endl;
+
+        } // end loop over joined results
+
+        if (verbose_p) {
+            std::cerr << line_no << ' ';
+            std::cerr.flush();
+        }
+
+    } // end loop over chunks
+
+    return line_no;
+}
+
+
+std::string
+Tokenizer::detokenize(const std::string& buf)
+{
+    std::vector<std::string> words = split(trim(buf));
+
+    std::size_t squotes = 0;
+    std::size_t dquotes = 0;
+    std::string prepends("");
+
+    std::ostringstream oss;
+
+    std::size_t nwords = words.size();
+    std::size_t iword = 0;
+
+    if (unescape_p)
+        for (auto &word: words)
+            unescape(word);
+
+    for (auto &word: words) {
+        if (RE2::FullMatch(word,right_x)) {
+            if (iword)
+                oss << SPC_BYTE;
+            oss << word;
+            prepends.clear();
+        } else if (RE2::FullMatch(word,left_x)) {
+            oss << word;
+            prepends = SPC_BYTE;
+        } else if (english_p && iword
+                   && RE2::FullMatch(word,curr_en_x)
+                   && RE2::FullMatch(words[iword-1],pre_en_x)) {
+            oss << word;
+            prepends = SPC_BYTE;
+        } else if (latin_p && iword < nwords - 2
+                   && RE2::FullMatch(word,curr_fr_x)
+                   && RE2::FullMatch(words[iword+1],post_fr_x)) {
+            oss << prepends << word;
+            prepends.clear();
+        } else if (word.size() == 1) {
+            if ((word.at(0) == '\'' && ((squotes % 2) == 0 )) ||
+                (word.at(0) == '"' && ((dquotes % 2) == 0))) {
+                if (english_p && iword
+                    && word.at(0) == '\''
+                    && std::tolower(words[iword-1].at(words[iword-1].size()-1)) == 's') {
+                    oss << word;
+                    prepends = SPC_BYTE;
+				} else {
+                    oss << prepends << word;
+                    prepends.clear();
+                    if (word.at(0) == '\'')
+                        squotes++;
+                    else
+                        dquotes++;
+                }
+			} else {
+                if (std::isalnum(word.at(0)))
+                    oss << prepends;
+                oss << word;
+                prepends = SPC_BYTE;
+                if (word.at(0) == '\'')
+                    squotes++;
+                else if (word.at(0) == '"')
+                    dquotes++;
+			}
+		} else {
+            oss << prepends << word;
+            prepends = SPC_BYTE;
+		}
+        iword++;
+	}
+
+
+    std::string text(oss.str());
+    RE2::GlobalReplace(&text," +",SPC_BYTE);
+    RE2::GlobalReplace(&text,"\n ","\n");
+    RE2::GlobalReplace(&text," \n","\n");
+    return trim(text);
+}
+
+
+std::size_t
+Tokenizer::detokenize(std::istream& is, std::ostream& os)
+{
+    size_t line_no = 0;
+    while (is.good() && os.good()) {
+        std::string istr;
+        std::getline(is,istr);
+        line_no ++;
+        if (istr.empty())
+            continue;
+        if (skip_xml_p && (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) {
+            os << istr << std::endl;
+        } else {
+            os << detokenize(istr) << std::endl;
+        }
+    }
+    return line_no;
+}
+
+
+std::vector<std::string>
+Tokenizer::splitter(const std::string &istr, bool *continuation_ptr) {
+    std::vector<std::string> parts;
+    glong ncp = 0;
+    glong ocp = 0;
+    glong icp = 0;
+    gunichar *ucs4 = g_utf8_to_ucs4_fast((gchar *)istr.c_str(),istr.size(),&ncp);
+    if (ncp == 0) {
+        g_free(ucs4);
+        return parts;
+    }
+    gunichar *uout = (gunichar *)g_malloc0(2*ncp*sizeof(gunichar));
+
+    const wchar_t GENL_HYPH = L'\u2010';
+    const wchar_t IDEO_STOP = L'\u3002';
+    const wchar_t KANA_MDOT = L'\u30FB';
+    const wchar_t WAVE_DASH = L'\u301C';
+    //const wchar_t WAVY_DASH = L'\u3030';
+    const wchar_t KANA_DHYP = L'\u30A0';
+    const wchar_t SMAL_HYPH = L'\uFE63';
+    const wchar_t WIDE_EXCL = L'\uFF01';
+    const wchar_t WIDE_PCTS = L'\uFF05';
+    //const wchar_t WIDE_HYPH = L'\uFF0D';
+    const wchar_t WIDE_STOP = L'\uFF0E';
+    const wchar_t WIDE_QUES = L'\uFF1F';
+    const wchar_t INVERT_QM = L'\u00BF';
+    const wchar_t INVERT_EX = L'\u00A1';
+
+    wchar_t currwc = 0;
+
+    std::size_t init_word = 0;
+    std::size_t fini_word = 0;
+    std::size_t finilen = 0;
+    std::size_t dotslen = 0;
+
+	  const std::size_t SEQ_LIM = 6;
+
+    charclass_t prev_class = empty;
+    charclass_t curr_class = empty;
+    std::vector<charclass_t> seq(SEQ_LIM, empty);
+    std::vector<std::size_t> pos(SEQ_LIM, 0);
+    std::size_t seqpos = 0;
+
+    GUnicodeType curr_type = G_UNICODE_UNASSIGNED;
+    //bool prev_word_p = false;
+    bool curr_word_p = false;
+
+    std::vector<std::size_t> breaks;
+    std::set<std::size_t> suppress;
+
+    for (; icp <= ncp; ++icp) {
+        currwc = wchar_t(ucs4[icp]);
+        curr_type = g_unichar_type(currwc);
+        prev_class = curr_class;
+        //prev_word_p = curr_word_p;
+
+        switch (curr_type) {
+        case G_UNICODE_DECIMAL_NUMBER:
+        case G_UNICODE_OTHER_NUMBER:
+            curr_class = numba;
+            curr_word_p = true;
+            break;
+        case G_UNICODE_LOWERCASE_LETTER:
+        case G_UNICODE_MODIFIER_LETTER:
+        case G_UNICODE_OTHER_LETTER:
+            curr_class = letta;
+            curr_word_p = true;
+            break;
+        case G_UNICODE_UPPERCASE_LETTER:
+        case G_UNICODE_TITLECASE_LETTER:
+            curr_class = upper;
+            curr_word_p = true;
+            break;
+        case G_UNICODE_OPEN_PUNCTUATION:
+        case G_UNICODE_INITIAL_PUNCTUATION:
+            curr_class = pinit;
+            curr_word_p = false;
+            break;
+        case G_UNICODE_DASH_PUNCTUATION:
+            curr_class = hyphn;
+            if (currwc <= GENL_HYPH) {
+                curr_word_p = true;
+            } else if (currwc >= SMAL_HYPH) {
+                curr_word_p = true;
+            } else {
+                curr_word_p = (currwc >= WAVE_DASH) && (currwc <= KANA_DHYP);
+            }
+            break;
+        case G_UNICODE_CLOSE_PUNCTUATION:
+        case G_UNICODE_FINAL_PUNCTUATION:
+            curr_class = pfini;
+            curr_word_p = false;
+            break;
+        case G_UNICODE_OTHER_PUNCTUATION:
+            if (currwc == L'\'' || currwc == L'"') {
+                curr_class = quote;
+                curr_word_p = false;
+            } else if (currwc == L'.' || currwc == IDEO_STOP || currwc == WIDE_STOP || currwc == KANA_MDOT) {
+                curr_class = stops;
+                curr_word_p = true;
+            } else if (currwc == L'?' || currwc == '!' || currwc == WIDE_EXCL || currwc == WIDE_QUES) {
+                curr_class = marks;
+                curr_word_p = false;
+            } else if (currwc == INVERT_QM || currwc == INVERT_EX) {
+                curr_class = pinit;
+                curr_word_p = false;
+            } else if ( currwc == L'%' || currwc == WIDE_PCTS) {
+                curr_class = pfpct;
+                curr_word_p = true;
+            } else {
+                curr_class = empty;
+                curr_word_p = false;
+            }
+            break;
+        default:
+            if (!g_unichar_isgraph(currwc)) {
+                curr_class = blank;
+            } else {
+                curr_class = empty;
+            }
+            curr_word_p = false;
+            break;
+        }
+
+        //  # condition for prefix test
+        //  $words[$i] =~ /([\p{IsAlnum}\.\-]*)([\'\"\)\]\%\p{IsPf}]*)(\.+)$/
+        //  $words[$i+1] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}]*[ ]*[\p{IsUpper}0-9])/
+
+        bool check_abbr_p = false;
+        if (curr_class == stops) {
+            if (prev_class != stops) {
+                dotslen = 1;
+            } else {
+                dotslen++;
+            }
+        } else if (curr_word_p) {
+            if (!fini_word) {
+                init_word = ocp;
+            }
+            fini_word = ocp+1;
+            dotslen = finilen = 0;
+        } else if (curr_class >= quote && curr_class <= pfpct && curr_class != pinit) {
+            finilen++;
+            dotslen = 0;
+            init_word = fini_word = 0;
+        } else if (dotslen) {
+            if (fini_word > init_word) {
+                if (prev_class!=stops || seqpos<1 || (ocp-pos[seqpos-1])<dotslen)
+                    check_abbr_p = false;
+                else
+                    check_abbr_p = dotslen < 2;
+            }
+            dotslen = 0;
+        } else {
+            init_word = fini_word = 0;
+        }
+
+        if (check_abbr_p) {
+            // not a valid word character or post-word punctuation character:  check word
+            std::wstring k((wchar_t *)uout+init_word,fini_word-init_word);
+            if (finilen == 0 && nbpre_gen_ucs4.find(k) != nbpre_gen_ucs4.end()) {
+                suppress.insert(std::size_t(ocp));
+                seqpos = 0;
+            } else {
+                bool acro_p = false;
+                bool found_upper_p = false;
+                for (glong ii = init_word; ii < ocp; ++ii) {
+                    if (uout[ii] == L'.') {
+                        acro_p = true;
+                    } else if (acro_p) {
+                        if (uout[ii] != L'.' && uout[ii] != L'-') {
+                            GUnicodeType i_type = g_unichar_type(uout[ii]);
+                            if (i_type != G_UNICODE_UPPERCASE_LETTER) {
+                                acro_p = false;
+                            } else {
+                                found_upper_p = true;
+                            }
+                        }
+                    }
+                }
+                if (acro_p && found_upper_p) {
+                    suppress.insert(std::size_t(ocp));
+                    seqpos = 0;
+                } else {
+                    // check forward:
+                    // $words[$i+1] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}]*[ ]*[\p{IsUpper}0-9])/
+                    int fcp = icp;
+                    int state = (curr_class == pinit || curr_class == quote) ? 1 : 0;
+                    bool num_p = true;
+                    while (fcp < ncp) {
+                        GUnicodeType f_type = g_unichar_type(ucs4[fcp]);
+                        bool f_white = g_unichar_isgraph(ucs4[fcp]);
+                        switch (state) {
+                        case 0:
+                            if (!f_white) {
+                                ++fcp;
+                                continue;
+                            } else if (f_type == G_UNICODE_INITIAL_PUNCTUATION || f_type == G_UNICODE_OPEN_PUNCTUATION ||
+                                       ucs4[fcp] == L'"'|| ucs4[fcp] == '\'' || ucs4[fcp] == INVERT_QM || ucs4[fcp] == INVERT_EX) {
+                                num_p = false;
+                                state = 1;
+                                ++fcp;
+                                continue;
+                            } else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
+                                if (num_p)
+                                    num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
+                                state = 3;
+                                ++fcp;
+                            }
+                            break;
+                        case 1:
+                            if (!f_white) {
+                                ++fcp;
+                                state = 2;
+                                continue;
+                            } else if (f_type == G_UNICODE_INITIAL_PUNCTUATION || f_type == G_UNICODE_OPEN_PUNCTUATION ||
+                                       ucs4[fcp] == L'"'|| ucs4[fcp] == '\'' || ucs4[fcp] == INVERT_QM || ucs4[fcp] == INVERT_EX) {
+                                ++fcp;
+                                continue;
+                            } else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
+                                if (num_p)
+                                    num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
+                                state = 3;
+                                ++fcp;
+                            }
+                            break;
+                        case 2:
+                            if (!f_white) {
+                                ++fcp;
+                                continue;
+                            } else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
+                                if (num_p)
+                                    num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
+                                state = 3;
+                                ++fcp;
+                                break;
+                            }
+                            break;
+                        }
+                        break;
+                    }
+                    if (num_p && state == 3 && nbpre_num_ucs4.find(k) != nbpre_num_ucs4.end()) {
+                        suppress.insert(std::size_t(ocp));
+                        seqpos = 0;
+                    }
+                }
+            }
+            init_word = fini_word = 0;
+        }
+
+        if (seqpos >= SEQ_LIM) {
+            seqpos = 0;
+        }
+
+        if (curr_class == stops || curr_class == marks) {
+            if (!seqpos) {
+                seq[seqpos] = curr_class;
+                pos[seqpos] = ocp;
+                seqpos++;
+                uout[ocp++] = gunichar(currwc);
+                continue;
+            } else if (seqpos>1 && (seq[seqpos-1]==blank || seq[seqpos-1]==quote || seq[seqpos-1]==pfini)) {
+                // handle "[?!.] ..." which is common in some corpora
+                if (seq[seqpos-2] == curr_class || seq[seqpos-2] == marks) {
+                    seqpos--;
+                    uout[ocp++] = gunichar(currwc);
+                    continue;
+                }
+                seqpos = 0;
+            } else if (seq[seqpos-1] != curr_class) {
+                seqpos = 0;
+            } else if (curr_class == marks) {
+                seqpos = 0;
+            } else {
+                uout[ocp++] = gunichar(currwc);
+                continue;
+            }
+        }
+
+        if (!seqpos) {
+            if (curr_class != blank) {
+                uout[ocp++] = gunichar(currwc);
+            } else if (curr_class != prev_class) {
+                uout[ocp++] = L' ';
+            }
+            continue;
+        }
+
+        if (curr_class == blank) {
+            if (prev_class != blank) {
+                seq[seqpos] = blank;
+                pos[seqpos] = ocp;
+                seqpos++;
+                uout[ocp++] = L' ';
+            }
+            if (icp < ncp)
+                continue;
+        }
+
+        if (curr_class >= quote && curr_class <= pfini) {
+            if (prev_class < quote || prev_class > pfini) {
+                seq[seqpos] = curr_class;
+                pos[seqpos] = ocp;
+                seqpos++;
+            } else if (curr_class == quote && prev_class != curr_class) {
+                curr_class = prev_class;
+            } else if (prev_class == quote) {
+                seq[seqpos] = prev_class = curr_class;
+            }
+            uout[ocp++] = gunichar(currwc);
+            continue;
+        }
+
+        //	$text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
+        //	#multi-dots followed by sentence starters 2
+        //  $text =~ s/(\.[\.]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
+        //  # add breaks for sentences that end with some sort of punctuation inside a quote or parenthetical and are followed by a possible sentence starter punctuation and upper case 4
+        //  $text =~ s/([?!\.][\ ]*[\'\"\)\]\p{IsPf}]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\ ]*[\p{IsUpper}])/$1\n$2/g;
+        //  # add breaks for sentences that end with some sort of punctuation are followed by a sentence starter punctuation and upper case 8
+        //  $text =~ s/([?!\.]) +([\'\"\(\[\¿\¡\p{IsPi}]+[\ ]*[\p{IsUpper}])/$1\n$2/g;
+
+        std::size_t iblank = 0;
+        if (curr_class == upper || icp == ncp) {
+            if (seqpos && (seq[0] == stops || seq[0] == marks)) {
+                switch (seqpos) {
+                case 2:
+                    if (seq[1] == blank)
+                        iblank = 1;
+                    break;
+                case 3:
+                    switch (seq[1]) {
+                    case blank:
+                        if (seq[2] == quote || seq[2] == pinit)
+                            iblank = 1;
+                        break;
+                    case quote:
+                    case pfini:
+                        if (seq[2] == blank)
+                            iblank = 2;
+                        break;
+                    default:
+                        break;
+                    }
+                    break;
+                case 4:
+                    switch (seq[1]) {
+                    case blank:
+                        iblank = 1;
+                        switch (seq[2]) {
+                        case quote:
+                            switch (seq[3]) {
+                            case quote:
+                            case pinit:
+                                break;
+                            case blank:
+                                iblank = 3;
+                                break;
+                            default:
+                                iblank = 0; // invalid
+                                break;
+                            }
+                            break;
+                        case pinit:
+                            if (seq[3] != blank)
+                                iblank = 0; // invalid
+                            break;
+                        case pfini:
+                            if (seq[3] == blank)
+                                iblank = 3;
+                            break;
+                        default:
+                            iblank = 0; // invalid
+                            break;
+                        }
+                        break;
+                    case quote:
+                    case pfini:
+                        iblank = (seq[2] == blank && (seq[3] == quote || seq[3] == pinit)) ? 2 : 0;
+                        break;
+                    default:
+                        iblank = 0; // invalid
+                        break;
+                    }
+                    break;
+                case 5:
+                    iblank = (seq[1] == blank) ? 2 : 1;
+                    if (seq[iblank] == quote || seq[iblank] == pfini)
+                        iblank++;
+                    if (seq[iblank] != blank) {
+                        iblank = 0; // invalid
+                    } else {
+                        if (seq[iblank+1] != quote && seq[iblank+1] != pinit) {
+                            iblank = 0; // invalid
+                        } else if (iblank+2 < seqpos) {
+                            if (seq[iblank+2] != blank)
+                                iblank = 0; // invalid
+                        }
+                    }
+                    break;
+                }
+            }
+            if (iblank && suppress.find(pos[iblank]) == suppress.end()) {
+                breaks.push_back(pos[iblank]);
+                suppress.insert(pos[iblank]);
+            }
+        }
+
+        uout[ocp++] = gunichar(currwc);
+        seqpos = 0;
+    }
+
+    std::vector<std::size_t>::iterator it = breaks.begin();
+    glong iop = 0;
+    while (iop < ocp) {
+        glong endpos = it == breaks.end() ? ocp : *it++;
+        glong nextpos = endpos + 1;
+        while (endpos > iop) {
+            std::size_t chkpos = endpos-1;
+            if (uout[chkpos] == L'\n' || uout[chkpos] == L' ') {
+                endpos = chkpos;
+                continue;
+            }
+            if (g_unichar_isgraph(uout[chkpos]))
+                break;
+            endpos = chkpos;
+        }
+        if (endpos > iop) {
+            gchar *pre = g_ucs4_to_utf8(uout+iop,endpos-iop,0,0,0);
+            parts.push_back(std::string(pre));
+            g_free(pre);
+        }
+        if (continuation_ptr)
+            *continuation_ptr = endpos > iop;
+        iop = nextpos;
+    }
+
+    g_free(uout);
+    g_free(ucs4);
+
+    return parts;
+}
+
+
+std::pair<std::size_t,std::size_t>
+Tokenizer::splitter(std::istream& is, std::ostream& os)
+{
+    std::pair<std::size_t,std::size_t> counts = { 0, 0 };
+    bool continuation_p = false;
+    bool pending_gap = false;
+    bool paragraph_p = false;
+
+    while (is.good() && os.good()) {
+        std::string istr;
+
+        std::getline(is,istr);
+        counts.first++;
+
+        if (istr.empty() && (is.eof() ||!para_marks_p))
+            continue;
+
+        if (skip_xml_p && (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x)))
+            continue;
+
+        std::vector<std::string> sentences(splitter(istr,&continuation_p));
+        if (sentences.empty()) {
+            if (!paragraph_p) {
+                if (pending_gap)
+                    os << std::endl;
+                pending_gap = false;
+                if (para_marks_p)
+                    os << "<P>" << std::endl;
+                paragraph_p = true;
+            }
+            continue;
+        }
+
+        paragraph_p = false;
+        std::size_t nsents = sentences.size();
+        counts.second += nsents;
+
+        if (pending_gap) {
+            os << " ";
+            pending_gap = false;
+        }
+
+        for (std::size_t ii = 0; ii < nsents-1; ++ii)
+            os << sentences[ii] << std::endl;
+
+        os << sentences[nsents-1];
+
+        if (continuation_p)
+            pending_gap = !split_breaks_p;
+        if (!pending_gap)
+            os << std::endl;
+    }
+
+    if (pending_gap)
+        os << std::endl;
+
+    return counts;
+}
+
+
+#ifdef TOKENIZER_NAMESPACE
+}; // namespace
+#endif
+

mosesdecoder/contrib/c++tokenizer/tokenizer.h

@@ -0,0 +1,205 @@
+#include <string>
+#include <iostream>
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+#include <unordered_map>
+#include <set>
+#include <vector>
+#include <iterator>
+#include <stdexcept>
+
+#include <re2/re2.h>
+#include <unistd.h>
+
+#include "Parameters.h"
+
+#ifdef TOKENIZER_NAMESPACE
+namespace TOKENIZER_NAMESPACE {
+#endif
+
+//
+// @about
+// Tokenizer implements the process of Koehn's tokenizer.perl via RE2
+//
+class Tokenizer {
+
+private:
+
+    typedef enum {
+        empty = 0,
+        blank,
+        upper, // upper case
+        letta, // extended word class (includes number, hyphen)
+        numba,
+        hyphn,
+        stops, // blank to stops are "extended word class" variants
+        quote, // init & fini = {',"}
+        pinit, // init (includes INVERT_*)
+        pfini, // fini
+        pfpct, // fini + pct
+        marks,
+        limit
+    } charclass_t;
+
+    std::size_t nthreads;
+    std::size_t chunksize;
+    std::string cfg_dir;
+
+    // non-breaking prefixes (numeric) utf8
+    std::set<std::string> nbpre_num_set;
+    // non-breaking prefixes (other) utf8
+    std::set<std::string> nbpre_gen_set;
+
+    // non-breaking prefixes (numeric) ucs4
+    std::set<std::wstring> nbpre_num_ucs4;
+    // non-breaking prefixes (other) ucs4
+    std::set<std::wstring> nbpre_gen_ucs4;
+
+    // compiled protected patterns
+    std::vector<re2::RE2 *> prot_pat_vec;
+
+protected:
+
+    // language
+    std::string lang_iso;
+    bool english_p; // is lang_iso "en"
+    bool latin_p; // is lang_iso "fr" or "it"
+    bool skip_xml_p;
+    bool skip_alltags_p;
+    bool entities_p;
+    bool escape_p;
+    bool unescape_p;
+    bool aggressive_hyphen_p;
+    bool supersub_p;
+    bool url_p;
+    bool downcase_p;
+    bool normalize_p;
+    bool penn_p;
+    bool narrow_latin_p;
+    bool narrow_kana_p;
+    bool refined_p;
+    bool drop_bad_p;
+    bool splits_p;
+    bool verbose_p;
+    bool para_marks_p;
+    bool split_breaks_p;
+
+    // return counts of general and numeric prefixes loaded
+    std::pair<int,int> load_prefixes(std::ifstream& ifs); // used by init(), parameterized by lang_iso
+
+    // in-place 1 line tokenizer, replaces input string, depends on wrapper to set-up invariants
+    void protected_tokenize(std::string& inplace);
+
+    // used for boost::thread
+    struct VectorTokenizerCallable {
+        Tokenizer *tokenizer;
+        std::vector<std::string>& in;
+        std::vector<std::string>& out;
+
+        VectorTokenizerCallable(Tokenizer *_tokenizer,
+                                std::vector<std::string>& _in,
+                                std::vector<std::string>& _out)
+        : tokenizer(_tokenizer)
+        , in(_in)
+        , out(_out) {
+        };
+
+        void operator()() {
+            out.resize(in.size());
+            for (std::size_t ii = 0; ii < in.size(); ++ii)
+                if (in[ii].empty())
+                    out[ii] = in[ii];
+                else if (tokenizer->penn_p)
+                    out[ii] = tokenizer->penn_tokenize(in[ii]);
+                else
+                    out[ii] = tokenizer->quik_tokenize(in[ii]);
+        };
+    };
+
+public:
+
+    Tokenizer(); // UNIMPL
+
+    // no throw
+    Tokenizer(const Parameters& _params);
+
+    // frees dynamically compiled expressions
+    ~Tokenizer();
+
+    // required before other methods, may throw
+    void init(const char *cfg_dir_path = 0);
+
+    void set_config_dir(const std::string& _cfg_dir);
+
+    // required after processing a contiguous sequence of lines when sentence splitting is on
+    void reset();
+
+    // simultaneous sentence splitting not yet implemented
+    bool splitting() const { return splits_p; }
+
+    // escapes chars the set &|"'<> after tokenization (moses special characters)
+    bool escape(std::string& inplace);
+
+    // used in detokenizer, converts entities into characters
+    // if escape_p is set, does not unescape moses special tokens, thus
+    // escape_p and unescape_p can be used together usefully
+    bool unescape(std::string& inplace);
+
+    // streaming select-tokenizer reads from is, writes to os, preserving line breaks (unless splitting)
+    std::size_t tokenize(std::istream& is, std::ostream& os);
+
+    // quik-tokenize padded line buffer to return string
+    std::string quik_tokenize(const std::string& buf);
+
+    // penn-tokenize padded line buffer to return string // untested
+    std::string penn_tokenize(const std::string& buf);
+
+    // select-tokenize padded line buffer to return string
+    std::string tokenize(const std::string& buf) {
+        return penn_p ? penn_tokenize(buf) : quik_tokenize(buf);
+    }
+
+    // tokenize with output argument
+    void tokenize(const std::string& buf, std::string& outs) {
+        outs = tokenize(buf);
+    }
+
+    // tokenize to a vector
+    std::vector<std::string> tokens(const std::string& in) {
+        std::istringstream tokss(penn_p ? penn_tokenize(in) : tokenize(in));
+        std::vector<std::string> outv;
+        std::copy(std::istream_iterator<std::string>(tokss),
+                  std::istream_iterator<std::string>(),
+                  std::back_inserter(outv));
+        return outv;
+    }
+
+    // streaming detokenizer reads from is, writes to os, preserving breaks
+    std::size_t detokenize(std::istream& is, std::ostream &os);
+
+    // detokenize padded line buffer to return string
+    std::string detokenize(const std::string& buf);
+
+    void detokenize(const std::string& buf, std::string& outs) {
+        outs = detokenize(buf);
+    }
+
+    // detokenize from a vector
+    std::string detokenize(const std::vector<std::string>& inv) {
+        std::ostringstream oss;
+        std::copy(inv.begin(), inv.end(), std::ostream_iterator<std::string>(oss," "));
+        return detokenize(oss.str());
+    }
+
+    // split a string on sentence boundaries (approximately)
+    std::vector<std::string> splitter(const std::string &istr,bool *continuation_p = 0);
+
+    // split sentences from input stream and write one per line on output stream
+    std::pair<std::size_t,std::size_t> splitter(std::istream& is, std::ostream& os);
+
+}; // end class Tokenizer
+
+#ifdef TOKENIZER_NAMESPACE
+};
+#endif

mosesdecoder/contrib/c++tokenizer/tokenizer_main.cpp

@@ -0,0 +1,352 @@
+#include "tokenizer.h"
+#include "Parameters.h"
+#include <memory>
+#include <vector>
+#include <cctype>
+#include <cstring>
+
+#ifdef TOKENIZER_NAMESPACE
+using namespace TOKENIZER_NAMESPACE ;
+#endif
+
+
+void
+usage(const char *path)
+{
+    std::cerr << "Usage: " << path << "[-{v|x|p|a|e|s|u|n|N]* [LL] [-{c|o} PATH]* INFILE*" << std::endl;
+    std::cerr << " -a -- aggressive hyphenization" << std::endl;
+    std::cerr << " -b -- drop bad bytes" << std::endl;
+    std::cerr << " -B -- splitter will split on linebreak" << std::endl;
+    std::cerr << " -c DIR -- config (pattern) file directory" << std::endl;
+    std::cerr << " -d -- downcase" << std::endl;
+    std::cerr << " -D -- detokenize" << std::endl;
+    std::cerr << " -e -- do not escape entities during tokenization" << std::endl;
+    std::cerr << " -E -- preserve entities during tokenization" << std::endl;
+    std::cerr << " -k -- narrow kana" << std::endl;
+    std::cerr << " -n -- narrow latin" << std::endl;
+    std::cerr << " -N -- normalize" << std::endl;
+    std::cerr << " -o OUT -- output file path" << std::endl;
+    std::cerr << " -p -- penn treebank style" << std::endl;
+    std::cerr << " -r -- refined contraction and quantity conjoining" << std::endl;
+    std::cerr << " -s -- super- and sub-script conjoining" << std::endl;
+    std::cerr << " -S -- buffer and sentence-split lines" << std::endl;
+    std::cerr << " -T -- do not tokenize, just split, no <P> marks" << std::endl;
+    std::cerr << " -t N[,C] -- use N threads (1), chunksize C lines" << std::endl;
+    std::cerr << " -u -- disable url handling" << std::endl;
+    std::cerr << " -U -- unescape entities before tokenization, after detokenization" << std::endl;
+    std::cerr << " -v -- verbose" << std::endl;
+    std::cerr << " -w -- word filter" << std::endl;
+    std::cerr << " -x -- skip xml tag lines" << std::endl;
+    std::cerr << " -y -- skip all xml tags" << std::endl;
+    std::cerr << " -X -- split only, with <P> marks" << std::endl;
+    std::cerr << "Default is -c ., stdin, stdout." << std::endl;
+    std::cerr << "LL in en,fr,it affect contraction.  LL selects nonbreaking prefix file" << std::endl;
+    std::cerr << "nonbreaking_prefix.LL is sought in getenv('TOKENIZER_SHARED_DIR')." << std::endl;
+    return;
+}
+
+
+std::string token_word(const std::string& in) {
+    int pos = -1;
+    int digits_prefixed = 0;
+    int nalpha = 0;
+    int len = in.size();
+    std::vector<char> cv;
+    int last_quirk = -1;
+    while (++pos < len) {
+        char ch = in.at(pos);
+        if (std::isdigit(ch)) {
+            if (digits_prefixed > 0) {
+                last_quirk = pos;
+                break;
+            }
+            digits_prefixed--;
+            cv.push_back(std::tolower(ch));
+        } else if (std::isalpha(ch)) {
+            if (digits_prefixed < 0)
+                digits_prefixed = -digits_prefixed;
+            cv.push_back(std::tolower(ch));
+            nalpha++;
+        } else {
+            if (digits_prefixed < 0)
+                digits_prefixed = -digits_prefixed;
+            last_quirk = pos;
+            if ((ch == '-' || ch == '\'') && pos != 0) {
+                cv.push_back(ch);
+            } else {
+                break;
+            }
+        }
+    }
+    if (last_quirk == pos || (digits_prefixed > 0 && nalpha == 0))
+        cv.clear(); // invalid word
+    return std::string(cv.begin(),cv.end());
+}
+
+
+int
+copy_words(Tokenizer& tize, std::istream& ifs, std::ostream& ofs) {
+    int nlines = 0;
+    std::string line;
+    while (ifs.good() && std::getline(ifs,line)) {
+        if (line.empty())
+            continue;
+        std::vector<std::string> tokens(tize.tokens(line));
+        int count = 0;
+        bool was_break = false;
+
+        for (auto& token: tokens) {
+            if (token.empty()) {
+                if (count || was_break) {
+                    ofs << std::endl;
+                    count = 0;
+                    nlines++;
+                    was_break = true;
+                    continue;
+                }
+            }
+            was_break = false;
+
+            std::string word(token_word(token));
+            if (word.empty()) {
+                continue;
+            }
+
+            if (count++) {
+                ofs << ' ';
+            }
+            ofs << word;
+        }
+
+        if (count) {
+            ofs << std::endl;
+            nlines++;
+        }
+    }
+    return nlines;
+}
+
+
+int main(int ac, char **av)
+{
+    int rc = 0;
+    Parameters params;
+
+    const char *prog = av[0];
+    bool next_cfg_p = false;
+    bool next_output_p = false;
+    bool next_threads_p = false;
+    bool detokenize_p = std::strstr(av[0],"detokenize") != 0;
+    if (!detokenize_p)
+        params.split_p = std::strstr(av[0],"splitter") != 0;
+
+    while (++av,--ac) {
+        if (**av == '-') {
+            switch (av[0][1]) {
+            case 'a':
+                params.aggro_p = true;
+                break;
+            case 'b':
+                params.drop_bad_p = true;
+                break;
+            case 'B':
+                params.split_breaks_p = true;
+                break;
+            case 'c':
+                next_cfg_p = true;
+                break;
+            case 'd':
+                params.downcase_p = true;
+                break;
+            case 'D':
+                detokenize_p = !detokenize_p;
+                break;
+            case 'e':
+                params.escape_p = !params.escape_p;
+                break;
+            case 'E':
+                params.entities_p = true;
+                break;
+            case 'h':
+                usage(prog);
+                exit(0);
+            case 'k':
+                params.narrow_kana_p = true;
+                break;
+            case 'n':
+                params.narrow_latin_p = true;
+                break;
+            case 'N':
+                params.normalize_p = true;
+                break;
+            case 'o':
+                next_output_p = true;
+                break;
+            case 'p':
+                params.penn_p = true;
+                break;
+            case 'r':
+                params.refined_p = true;
+                break;
+            case 's':
+                params.supersub_p = true;
+                break;
+            case 'S':
+                params.split_p = !params.split_p;
+                break;
+            case 'T':
+                params.notokenization_p = true;
+                params.para_marks_p = false;
+                break;
+            case 't':
+                next_threads_p = true;
+                break;
+            case 'U':
+                params.unescape_p = true;
+                break;
+            case 'u':
+                params.url_p = false;
+                break;
+            case 'v':
+                params.verbose_p = true;
+                break;
+            case 'w':
+                params.words_p = true;
+                break;
+            case 'x':
+                params.detag_p = true;
+                break;
+            case 'X':
+                params.notokenization_p = true;
+                params.para_marks_p = true;
+                break;
+            case 'y':
+                params.alltag_p = true;
+                break;
+            case 'l':
+                // ignored
+                break;
+            default:
+                std::cerr << "Unknown option: " << *av << std::endl;
+                ::exit(1);
+            }
+        } else if (params.lang_iso.empty() && strlen(*av) == 2 && !isdigit(**av)) {
+            params.lang_iso = *av;
+        } else if (next_output_p) {
+            next_output_p = false;
+            params.out_path = *av;
+        } else if (next_cfg_p) {
+            next_cfg_p = false;
+            params.cfg_path = *av;
+        } else if (next_threads_p) {
+            next_threads_p = false;
+            char *comma = strchr(*av,',');
+            if (comma) {
+                *comma++ = 0;
+                params.chunksize = std::strtoul(comma,0,0);
+            }
+            params.nthreads = std::strtoul(*av,0,0);
+        } else {
+            params.args.push_back(std::string(*av));
+        }
+    }
+
+    if (!params.cfg_path) {
+        params.cfg_path = getenv("TOKENIZER_SHARED_DIR");
+    }
+    if (!params.cfg_path) {
+        if (!::access("../share/.",X_OK)) {
+            if (!::access("../share/moses/.",X_OK)) {
+                params.cfg_path = "../share/moses";
+            } else {
+                params.cfg_path = "../share";
+            }
+        } else if (!::access("./scripts/share/.",X_OK)) {
+            params.cfg_path = "./scripts/share";
+        } else if (!::access("./nonbreaking_prefix.en",R_OK)) {
+            params.cfg_path = ".";
+        } else {
+            const char *slash = std::strrchr(prog,'/');
+            if (slash) {
+                std::string cfg_dir_str(prog,slash-prog);
+                std::string cfg_shr_str(cfg_dir_str);
+                cfg_shr_str.append("/shared");
+                std::string cfg_mos_str(cfg_shr_str);
+                cfg_mos_str.append("/moses");
+                if (!::access(cfg_mos_str.c_str(),X_OK)) {
+                    params.cfg_path = strdup(cfg_mos_str.c_str());
+                } else if (!::access(cfg_shr_str.c_str(),X_OK)) {
+                    params.cfg_path = strdup(cfg_shr_str.c_str());
+                } else if (!::access(cfg_dir_str.c_str(),X_OK)) {
+                    params.cfg_path = strdup(cfg_dir_str.c_str());
+                }
+            }
+        }
+    }
+    if (params.cfg_path) {
+        if (params.verbose_p) {
+            std::cerr << "config path: " << params.cfg_path << std::endl;
+        }
+    }
+
+    std::unique_ptr<std::ofstream> pofs = 0;
+    if (!params.out_path.empty()) {
+        pofs.reset(new std::ofstream(params.out_path.c_str()));
+    }
+    std::ostream& ofs(pofs ? *pofs : std::cout);
+
+    if (params.lang_iso.empty())
+        params.lang_iso = "en";
+
+    Tokenizer tize(params);
+    tize.init();
+    std::pair<std::size_t,std::size_t> plines = { 0, 0 };
+
+    if (params.words_p) {
+        if (params.args.empty()) {
+            plines.first += copy_words(tize,std::cin,ofs);
+        } else {
+            for (std::string& arg : params.args) {
+                try {
+                    std::ifstream ifs(arg.c_str());
+                    plines.first += copy_words(tize,ifs,ofs);
+                } catch (...) {
+                    std::cerr << "Exception extracting words from path " << arg << std::endl;
+                }
+            }
+        }
+    } else if (params.args.empty()) {
+        if (detokenize_p) {
+            plines.first = tize.detokenize(std::cin,ofs);
+        } else if (params.notokenization_p) {
+            plines = tize.splitter(std::cin,ofs);
+        } else {
+            plines.first = tize.tokenize(std::cin,ofs);
+        }
+    } else {
+        for (std::string& arg : params.args) {
+            try {
+                std::ifstream ifs(arg.c_str());
+                if (detokenize_p) {
+                    plines.first = tize.detokenize(ifs,ofs);
+                } else if (params.notokenization_p) {
+                    plines = tize.splitter(ifs,ofs);
+                } else {
+                    plines.first = tize.tokenize(ifs,ofs);
+                }
+            } catch (...) {
+                std::cerr << "Exception tokenizing from path " << arg << std::endl;
+            }
+        }
+    }
+
+    if (params.verbose_p) {
+        std::cerr << "%%% " << plines.first << " lines." << std::endl;
+        if (plines.second) {
+            std::cerr << "%%% " << plines.second << " sentences." << std::endl;
+        }
+    }
+    return rc;
+}
+
+