| """ |
| A module for interfacing with ``split-sentences.perl`` from Moses toolkit. |
| |
| Copyright ® 2016-2017, Luís Gomes <luismsgomes@gmail.com> |
| """ |
|
|
| usage = """ |
| Usage: |
| moses-sentence-splitter [options] <lang> [<inputfile> [<outputfile>]] |
| moses-sentence-splitter --selftest [--verbose] |
| |
| Options: |
| --selftest, -t Run selftests. |
| --verbose, -v Be more verbose. |
| --unwrap, -u Assume that the text is wrapped and try to unwrap it. |
| Note that this option will cause all consecutive non-empty |
| lines to be buffered in memory. If you give this option |
| make sure that you have empty lines separating paragraphs. |
| When this option is not given, each line is assumed to be |
| an independent paragraph or sentence and thus will not be |
| joined with other lines. |
| --more Also split on colons and semi-colons. |
| |
| 2016, Luís Gomes <luismsgomes@gmail.com> |
| """ |
|
|
|
|
| from docopt import docopt |
| from openfile import openfile |
| from os import path |
| from toolwrapper import ToolWrapper |
| import sys |
|
|
|
|
| class MosesSentenceSplitter(ToolWrapper): |
| """ |
| A class for interfacing with ``split-sentences.perl`` from Moses toolkit. |
| |
| This class communicates with split-sentences.perl process via pipes. When |
| the MosesSentenceSplitter object is no longer needed, the close() method |
| should be called to free system resources. The class supports the context |
| manager interface. If used in a with statement, the close() method is |
| invoked automatically. |
| |
| When attribute ``more`` is True, colons and semi-colons are considered |
| sentence separators. |
| |
| >>> split_sents = MosesSentenceSplitter('en') |
| >>> split_sents(['Hello World! Hello', 'again.']) |
| ['Hello World!', 'Hello again.'] |
| |
| """ |
|
|
| def __init__(self, lang="en", more=True): |
| self.lang = lang |
| program = path.join( |
| path.dirname(__file__), |
| "split-sentences.perl" |
| ) |
| argv = ["perl", program, "-q", "-b", "-l", self.lang] |
| if more: |
| argv.append("-m") |
| super().__init__(argv) |
|
|
| def __str__(self): |
| return "MosesSentenceSplitter(lang=\"{lang}\")".format(lang=self.lang) |
|
|
| def __call__(self, paragraph): |
| """Splits sentences within a paragraph. |
| The paragraph is a list of non-empty lines. XML-like tags are not |
| allowed. |
| """ |
| assert isinstance(paragraph, (list, tuple)) |
| if not paragraph: |
| return [] |
| assert all(isinstance(line, str) for line in paragraph) |
| paragraph = [line.strip() for line in paragraph] |
| assert all(paragraph), "blank lines are not allowed" |
| for line in paragraph: |
| self.writeline(line) |
| self.writeline("<P>") |
| sentences = [] |
| while True: |
| sentence = self.readline().strip() |
| if sentence == "<P>": |
| break |
| sentences.append(sentence) |
| return sentences |
|
|
|
|
| def read_paragraphs(inputfile, wrapped=True): |
| lines = map(str.strip, inputfile) |
| if wrapped: |
| paragraph = [] |
| for line in lines: |
| if line: |
| paragraph.append(line) |
| elif paragraph: |
| yield paragraph |
| paragraph = [] |
| if paragraph: |
| yield paragraph |
| else: |
| for line in lines: |
| yield [line] if line else [] |
|
|
|
|
| def write_paragraphs(paragraphs, outputfile, blank_sep=True): |
| for paragraph in paragraphs: |
| for sentence in paragraph: |
| print(sentence, file=outputfile) |
| if blank_sep or not paragraph: |
| print(file=outputfile) |
|
|
|
|
| def main(): |
| args = docopt(usage) |
| if args["--selftest"]: |
| import doctest |
| import mosestokenizer.sentsplitter |
| doctest.testmod(mosestokenizer.sentsplitter) |
| if not args["<lang>"]: |
| sys.exit(0) |
| split_sents = MosesSentenceSplitter(args["<lang>"], more=args["--more"]) |
| inputfile = openfile(args["<inputfile>"]) |
| outputfile = openfile(args["<outputfile>"], "wt") |
| with inputfile, outputfile: |
| paragraphs = read_paragraphs(inputfile, wrapped=args["--unwrap"]) |
| paragraphs = map(split_sents, paragraphs) |
| write_paragraphs(paragraphs, outputfile, blank_sep=args["--unwrap"]) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
