| |
| |
| |
| |
| |
|
|
| from __future__ import print_function, unicode_literals |
|
|
| import logging |
| import argparse |
| import subprocess |
| import sys |
| import os |
| import codecs |
|
|
| |
| sys.path.append(os.path.join(os.path.dirname(sys.path[0]), 'bilingual-lm')) |
| import train_nplm |
| import extract_vocab |
| import extract_syntactic_ngrams |
|
|
| logging.basicConfig( |
| format='%(asctime)s %(levelname)s: %(message)s', |
| datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG) |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--working-dir", dest="working_dir", metavar="PATH") |
| parser.add_argument( |
| "--corpus", '-text', dest="corpus_stem", metavar="PATH", help="Input file.") |
| parser.add_argument( |
| "--nplm-home", dest="nplm_home", metavar="PATH", required=True, |
| help="Location of NPLM.") |
| parser.add_argument( |
| "--epochs", dest="epochs", type=int, metavar="INT", |
| help="Number of training epochs (default: %(default)s).") |
| parser.add_argument( |
| "--up-context-size", dest="up_context_size", type=int, metavar="INT", |
| help="Size of ancestor context (default: %(default)s).") |
| parser.add_argument( |
| "--left-context-size", dest="left_context_size", type=int, metavar="INT", |
| help="Size of sibling context (left) (default: %(default)s).") |
| parser.add_argument( |
| "--right-context-size", dest="right_context_size", type=int, |
| metavar="INT", |
| help="Size of sibling context (right) (default: %(default)s).") |
| parser.add_argument( |
| "--mode", dest="mode", choices=['head', 'label'], required=True, |
| help="Type of RDLM to train (both are required for decoding).") |
| parser.add_argument( |
| "--minibatch-size", dest="minibatch_size", type=int, metavar="INT", |
| help="Minibatch size (default: %(default)s).") |
| parser.add_argument( |
| "--noise", dest="noise", type=int, metavar="INT", |
| help="Number of noise samples for NCE (default: %(default)s).") |
| parser.add_argument( |
| "--hidden", dest="hidden", type=int, metavar="INT", |
| help=( |
| "Size of hidden layer (0 for single hidden layer) " |
| "(default: %(default)s)")) |
| parser.add_argument( |
| "--input-embedding", dest="input_embedding", type=int, metavar="INT", |
| help="Size of input embedding layer (default: %(default)s).") |
| parser.add_argument( |
| "--output-embedding", dest="output_embedding", type=int, metavar="INT", |
| help="Size of output embedding layer (default: %(default)s).") |
| parser.add_argument( |
| "--threads", "-t", dest="threads", type=int, metavar="INT", |
| help="Number of threads (default: %(default)s).") |
| parser.add_argument( |
| "--output-model", dest="output_model", metavar="PATH", |
| help="Name of output model (default: %(default)s).") |
| parser.add_argument( |
| "--output-dir", dest="output_dir", metavar="PATH", |
| help="Output directory (default: same as working-dir).") |
| parser.add_argument( |
| "--config-options-file", dest="config_options_file", metavar="PATH") |
| parser.add_argument( |
| "--log-file", dest="log_file", metavar="PATH", |
| help="Log file to write to (default: %(default)s).") |
| parser.add_argument( |
| "--validation-corpus", dest="validation_corpus", metavar="PATH", |
| help="Validation file (default: %(default)s).") |
| parser.add_argument( |
| "--activation-function", dest="activation_fn", |
| choices=['identity', 'rectifier', 'tanh', 'hardtanh'], |
| help="Activation function (default: %(default)s).") |
| parser.add_argument( |
| "--learning-rate", dest="learning_rate", type=float, metavar="FLOAT", |
| help="Learning rate (default: %(default)s).") |
| parser.add_argument( |
| "--input-words-file", dest="input_words_file", metavar="PATH", |
| help="Input vocabulary (default: %(default)s).") |
| parser.add_argument( |
| "--output-words-file", dest="output_words_file", metavar="PATH", |
| help="Output vocabulary (default: %(default)s).") |
| parser.add_argument( |
| "--input-vocab-size", dest="input_vocab_size", type=int, metavar="INT", |
| help="Input vocabulary size (default: %(default)s).") |
| parser.add_argument( |
| "--output-vocab-size", dest="output_vocab_size", type=int, metavar="INT", |
| help="Output vocabulary size (default: %(default)s).") |
| parser.add_argument( |
| "--mmap", dest="mmap", action="store_true", |
| help="Use memory-mapped file (for lower memory consumption).") |
| parser.add_argument( |
| "--train-host", dest="train_host", |
| help="Execute nplm training on this host, via ssh") |
| parser.add_argument("--extra-settings", dest="extra_settings", |
| help="Extra settings to be passed to NPLM") |
|
|
|
|
| parser.set_defaults( |
| working_dir="working", |
| corpus_stem="train", |
| nplm_home="/home/bhaddow/tools/nplm", |
| epochs=2, |
| up_context_size=2, |
| left_context_size=3, |
| right_context_size=0, |
| minibatch_size=1000, |
| noise=100, |
| hidden=0, |
| mode='head', |
| input_embedding=150, |
| output_embedding=750, |
| threads=4, |
| output_model="train", |
| output_dir=None, |
| config_options_file="config", |
| log_file="log", |
| validation_corpus=None, |
| activation_fn="rectifier", |
| learning_rate=1, |
| input_words_file=None, |
| output_words_file=None, |
| input_vocab_size=500000, |
| output_vocab_size=500000) |
|
|
|
|
| def prepare_vocabulary(options): |
| vocab_prefix = os.path.join(options.working_dir, 'vocab') |
| extract_vocab_options = extract_vocab.create_parser().parse_args( |
| ['--input', options.corpus_stem, '--output', vocab_prefix]) |
| extract_vocab.main(extract_vocab_options) |
|
|
| if options.input_words_file is None: |
| options.input_words_file = vocab_prefix + '.input' |
| orig = vocab_prefix + '.all' |
| filtered_vocab = open(orig).readlines() |
| if options.input_vocab_size: |
| filtered_vocab = filtered_vocab[:options.input_vocab_size] |
| open(options.input_words_file, 'w').writelines(filtered_vocab) |
|
|
| if options.output_words_file is None: |
| options.output_words_file = vocab_prefix + '.output' |
| if options.mode == 'label': |
| blacklist = [ |
| '<null', |
| '<root', |
| '<start_head', |
| '<dummy', |
| '<head_head', |
| '<stop_head', |
| ] |
| orig = vocab_prefix + '.special' |
| filtered_vocab = open(orig).readlines() |
| orig = vocab_prefix + '.nonterminals' |
| filtered_vocab += open(orig).readlines() |
| filtered_vocab = [ |
| word |
| for word in filtered_vocab |
| if not any(word.startswith(prefix) for prefix in blacklist)] |
| if options.output_vocab_size: |
| filtered_vocab = filtered_vocab[:options.output_vocab_size] |
| else: |
| orig = vocab_prefix + '.all' |
| filtered_vocab = open(orig).readlines()[:options.output_vocab_size] |
| open(options.output_words_file, 'w').writelines(filtered_vocab) |
|
|
|
|
| def main(options): |
|
|
| if options.output_dir is None: |
| options.output_dir = options.working_dir |
| else: |
| |
| if not os.path.exists(options.output_dir): |
| os.makedirs(options.output_dir) |
|
|
| options.ngram_size = ( |
| 2 * options.up_context_size + |
| 2 * options.left_context_size + |
| 2 * options.right_context_size |
| ) |
| if options.mode == 'head': |
| options.ngram_size += 2 |
| elif options.mode == 'label': |
| options.ngram_size += 1 |
|
|
| if options.input_words_file is None or options.output_words_file is None: |
| sys.stderr.write( |
| "Either input vocabulary or output vocabulary not specified: " |
| "extracting vocabulary from training text.\n") |
| prepare_vocabulary(options) |
|
|
| numberized_file = os.path.basename(options.corpus_stem) + '.numberized' |
| train_file = numberized_file |
| if options.mmap: |
| train_file += '.mmap' |
|
|
| extract_options = extract_syntactic_ngrams.create_parser().parse_args([ |
| '--input', options.corpus_stem, |
| '--output', os.path.join(options.working_dir, numberized_file), |
| '--vocab', options.input_words_file, |
| '--output_vocab', options.output_words_file, |
| '--right_context', str(options.right_context_size), |
| '--left_context', str(options.left_context_size), |
| '--up_context', str(options.up_context_size), |
| '--mode', options.mode |
| ]) |
| sys.stderr.write('extracting syntactic n-grams\n') |
| extract_syntactic_ngrams.main(extract_options) |
|
|
| if options.validation_corpus: |
| extract_options.input = open(options.validation_corpus) |
| options.validation_file = os.path.join( |
| options.working_dir, os.path.basename(options.validation_corpus)) |
| extract_options.output = open( |
| options.validation_file + '.numberized', 'w') |
| sys.stderr.write('extracting syntactic n-grams (validation file)\n') |
| extract_syntactic_ngrams.main(extract_options) |
| extract_options.output.close() |
| else: |
| options.validation_file = None |
|
|
| if options.mmap: |
| try: |
| os.remove(os.path.join(options.working_dir, train_file)) |
| except OSError: |
| pass |
| mmap_cmd = [os.path.join(options.nplm_home, 'src', 'createMmap'), |
| '--input_file', |
| os.path.join(options.working_dir, numberized_file), |
| '--output_file', |
| os.path.join(options.working_dir, train_file) |
| ] |
| sys.stderr.write('creating memory-mapped file\n') |
| sys.stderr.write('executing: ' + ', '.join(mmap_cmd) + '\n') |
| ret = subprocess.call(mmap_cmd) |
| if ret: |
| raise Exception("creating memory-mapped file failed") |
|
|
| sys.stderr.write('training neural network\n') |
| train_nplm.main(options) |
|
|
| sys.stderr.write('averaging null words\n') |
| ret = subprocess.call([ |
| os.path.join(sys.path[0], 'average_null_embedding.py'), |
| options.nplm_home, |
| os.path.join( |
| options.output_dir, |
| options.output_model + '.model.nplm.' + str(options.epochs)), |
| os.path.join( |
| options.working_dir, |
| numberized_file), |
| os.path.join(options.output_dir, options.output_model + '.model.nplm') |
| ]) |
| if ret: |
| raise Exception("averaging null words failed") |
|
|
|
|
| if __name__ == "__main__": |
| if sys.version_info < (3, 0): |
| sys.stderr = codecs.getwriter('UTF-8')(sys.stderr) |
| sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) |
| sys.stdin = codecs.getreader('UTF-8')(sys.stdin) |
|
|
| options = parser.parse_known_args()[0] |
| if parser.parse_known_args()[1]: |
| sys.stderr.write('Warning: unknown arguments: {0}\n'.format(parser.parse_known_args()[1])) |
| main(options) |
|
|
