| |
| |
| |
| |
| |
| |
|
|
| """ |
| (Hacky) wrapper around Stanford CoreNLP to produce CoNLL dependency format. |
| Assumes tokenized and sentence-split text. |
| |
| To get Moses XML format, first projectivize the trees, then use |
| conll2mosesxml.py. |
| """ |
|
|
| from __future__ import print_function, unicode_literals |
| import os |
| import sys |
| import codecs |
| import argparse |
|
|
| from subprocess import Popen, PIPE |
|
|
| |
| from io import open |
| argparse.open = open |
|
|
|
|
| def create_parser(): |
| parser = argparse.ArgumentParser( |
| description=( |
| """Wrapper around Stanford CoreNLP to produce CoNLL dependency format. |
| Assumes that text is tokenized and has one sentence per line.""")) |
|
|
| parser.add_argument( |
| '--stanford', type=str, |
| metavar='PATH', required=True, |
| help='path to Stanford CoreNLP') |
|
|
| parser.add_argument( |
| '--java', type=str, default='java', |
| metavar='PATH', |
| help='path to java executable') |
|
|
| parser.add_argument( |
| '--input', '-i', type=argparse.FileType('r'), default=sys.stdin, |
| metavar='PATH', |
| help="Input text (default: standard input).") |
| parser.add_argument( |
| '--output', '-o', type=argparse.FileType('w'), default=sys.stdout, |
| metavar='PATH', |
| help="Output text (default: standard output).") |
|
|
| return parser |
|
|
|
|
| def process_stanford(infile, javacmd, stanfordpath): |
|
|
| corenlp_jar = os.path.join(stanfordpath, 'stanford-corenlp-3.5.0.jar') |
| corenlp_models_jar = os.path.join( |
| stanfordpath, 'stanford-corenlp-3.5.0-models.jar') |
| stanford = Popen( |
| [ |
| javacmd, |
| '-cp', "%s:%s" % (corenlp_jar, corenlp_models_jar), |
| 'edu.stanford.nlp.pipeline.StanfordCoreNLP', |
| '-annotators', 'tokenize, ssplit, pos, depparse, lemma', |
| '-ssplit.eolonly', 'true', |
| '-tokenize.whitespace', 'true', |
| '-numThreads', '8', |
| '-textFile', '-', |
| 'outFile', '-', |
| ], |
| stdin=infile, stdout=PIPE) |
| return stanford.stdout |
|
|
|
|
| def get_sentences(instream): |
| sentence = [] |
| expect = 0 |
|
|
| for line in instream: |
| if expect == 0 and line.startswith('Sentence #'): |
| if sentence: |
| yield sentence |
| sentence = [] |
| expect = 1 |
|
|
| elif line == '\n': |
| expect = 0 |
|
|
| elif expect == 3: |
| rel, remainder = line.split('(') |
| head, dep = remainder.split() |
| head_int = int(head.split('-')[-1][:-1]) |
| dep_int = int(dep.split('-')[-1][:-1]) |
| sentence[dep_int - 1]['head'] = head_int |
| sentence[dep_int - 1]['label'] = rel |
|
|
| elif expect == 2: |
| linesplit = line.split('[', 1)[1].rsplit(']', 1)[0].split('] [') |
| if len(linesplit) != len(sentence): |
| sys.stderr.write( |
| "Warning: mismatch in number of words in sentence\n") |
| sys.stderr.write(' '.join(w['word'] for w in sentence)) |
| for i in range(len(sentence)): |
| sentence[i]['pos'] = '-' |
| sentence[i]['lemma'] = '-' |
| sentence[i]['head'] = 0 |
| sentence[i]['label'] = '-' |
| expect = 0 |
| continue |
| for i, w in enumerate(linesplit): |
| sentence[i]['pos'] = w.split(' PartOfSpeech=')[-1].split()[0] |
| sentence[i]['lemma'] = w.split(' Lemma=')[-1] |
| expect = 3 |
|
|
| elif expect == 1: |
| for w in line.split(): |
| sentence.append({'word': w}) |
| expect = 2 |
|
|
| if sentence: |
| yield sentence |
|
|
|
|
| def write(sentence, outstream): |
| for i, w in enumerate(sentence): |
| outstream.write( |
| '{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\n'.format( |
| i + 1, w['word'], w['lemma'], w['pos'], w['pos'], '-', |
| w['head'], w['label'])) |
|
|
|
|
| if __name__ == '__main__': |
| if sys.version_info < (3, 0): |
| sys.stderr = codecs.getwriter('UTF-8')(sys.stderr) |
| sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) |
| sys.stdin = codecs.getreader('UTF-8')(sys.stdin) |
|
|
| parser = create_parser() |
| options = parser.parse_args() |
|
|
| stanford = process_stanford(options.input, options.java, options.stanford) |
| for sentence in get_sentences(codecs.getreader('UTF-8')(stanford)): |
| write(sentence, options.output) |
| options.output.write('\n') |
|
|
