Add files using upload-large-folder tool

.eggs/nltk-3.8-py3.10.egg/nltk/app/chunkparser_app.py

@@ -0,0 +1,1500 @@
+# Natural Language Toolkit: Regexp Chunk Parser Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the regular expression based chunk
+parser ``nltk.chunk.RegexpChunkParser``.
+"""
+
+# Todo: Add a way to select the development set from the menubar.  This
+# might just need to be a selection box (conll vs treebank etc) plus
+# configuration parameters to select what's being chunked (eg VP vs NP)
+# and what part of the data is being used as the development set.
+
+import random
+import re
+import textwrap
+import time
+from tkinter import (
+    Button,
+    Canvas,
+    Checkbutton,
+    Frame,
+    IntVar,
+    Label,
+    Menu,
+    Scrollbar,
+    Text,
+    Tk,
+)
+from tkinter.filedialog import askopenfilename, asksaveasfilename
+from tkinter.font import Font
+
+from nltk.chunk import ChunkScore, RegexpChunkParser
+from nltk.chunk.regexp import RegexpChunkRule
+from nltk.corpus import conll2000, treebank_chunk
+from nltk.draw.util import ShowText
+from nltk.tree import Tree
+from nltk.util import in_idle
+
+
+class RegexpChunkApp:
+    """
+    A graphical tool for exploring the regular expression based chunk
+    parser ``nltk.chunk.RegexpChunkParser``.
+
+    See ``HELP`` for instructional text.
+    """
+
+    ##/////////////////////////////////////////////////////////////////
+    ##  Help Text
+    ##/////////////////////////////////////////////////////////////////
+
+    #: A dictionary mapping from part of speech tags to descriptions,
+    #: which is used in the help text.  (This should probably live with
+    #: the conll and/or treebank corpus instead.)
+    TAGSET = {
+        "CC": "Coordinating conjunction",
+        "PRP$": "Possessive pronoun",
+        "CD": "Cardinal number",
+        "RB": "Adverb",
+        "DT": "Determiner",
+        "RBR": "Adverb, comparative",
+        "EX": "Existential there",
+        "RBS": "Adverb, superlative",
+        "FW": "Foreign word",
+        "RP": "Particle",
+        "JJ": "Adjective",
+        "TO": "to",
+        "JJR": "Adjective, comparative",
+        "UH": "Interjection",
+        "JJS": "Adjective, superlative",
+        "VB": "Verb, base form",
+        "LS": "List item marker",
+        "VBD": "Verb, past tense",
+        "MD": "Modal",
+        "NNS": "Noun, plural",
+        "NN": "Noun, singular or masps",
+        "VBN": "Verb, past participle",
+        "VBZ": "Verb,3rd ps. sing. present",
+        "NNP": "Proper noun, singular",
+        "NNPS": "Proper noun plural",
+        "WDT": "wh-determiner",
+        "PDT": "Predeterminer",
+        "WP": "wh-pronoun",
+        "POS": "Possessive ending",
+        "WP$": "Possessive wh-pronoun",
+        "PRP": "Personal pronoun",
+        "WRB": "wh-adverb",
+        "(": "open parenthesis",
+        ")": "close parenthesis",
+        "``": "open quote",
+        ",": "comma",
+        "''": "close quote",
+        ".": "period",
+        "#": "pound sign (currency marker)",
+        "$": "dollar sign (currency marker)",
+        "IN": "Preposition/subord. conjunction",
+        "SYM": "Symbol (mathematical or scientific)",
+        "VBG": "Verb, gerund/present participle",
+        "VBP": "Verb, non-3rd ps. sing. present",
+        ":": "colon",
+    }
+
+    #: Contents for the help box.  This is a list of tuples, one for
+    #: each help page, where each tuple has four elements:
+    #:   - A title (displayed as a tab)
+    #:   - A string description of tabstops (see Tkinter.Text for details)
+    #:   - The text contents for the help page.  You can use expressions
+    #:     like <red>...</red> to colorize the text; see ``HELP_AUTOTAG``
+    #:     for a list of tags you can use for colorizing.
+    HELP = [
+        (
+            "Help",
+            "20",
+            "Welcome to the regular expression chunk-parser grammar editor.  "
+            "You can use this editor to develop and test chunk parser grammars "
+            "based on NLTK's RegexpChunkParser class.\n\n"
+            # Help box.
+            "Use this box ('Help') to learn more about the editor; click on the "
+            "tabs for help on specific topics:"
+            "<indent>\n"
+            "Rules: grammar rule types\n"
+            "Regexps: regular expression syntax\n"
+            "Tags: part of speech tags\n</indent>\n"
+            # Grammar.
+            "Use the upper-left box ('Grammar') to edit your grammar.  "
+            "Each line of your grammar specifies a single 'rule', "
+            "which performs an action such as creating a chunk or merging "
+            "two chunks.\n\n"
+            # Dev set.
+            "The lower-left box ('Development Set') runs your grammar on the "
+            "development set, and displays the results.  "
+            "Your grammar's chunks are <highlight>highlighted</highlight>, and "
+            "the correct (gold standard) chunks are "
+            "<underline>underlined</underline>.  If they "
+            "match, they are displayed in <green>green</green>; otherwise, "
+            "they are displayed in <red>red</red>.  The box displays a single "
+            "sentence from the development set at a time; use the scrollbar or "
+            "the next/previous buttons view additional sentences.\n\n"
+            # Performance
+            "The lower-right box ('Evaluation') tracks the performance of "
+            "your grammar on the development set.  The 'precision' axis "
+            "indicates how many of your grammar's chunks are correct; and "
+            "the 'recall' axis indicates how many of the gold standard "
+            "chunks your system generated.  Typically, you should try to "
+            "design a grammar that scores high on both metrics.  The "
+            "exact precision and recall of the current grammar, as well "
+            "as their harmonic mean (the 'f-score'), are displayed in "
+            "the status bar at the bottom of the window.",
+        ),
+        (
+            "Rules",
+            "10",
+            "<h1>{...regexp...}</h1>"
+            "<indent>\nChunk rule: creates new chunks from words matching "
+            "regexp.</indent>\n\n"
+            "<h1>}...regexp...{</h1>"
+            "<indent>\nStrip rule: removes words matching regexp from existing "
+            "chunks.</indent>\n\n"
+            "<h1>...regexp1...}{...regexp2...</h1>"
+            "<indent>\nSplit rule: splits chunks that match regexp1 followed by "
+            "regexp2 in two.</indent>\n\n"
+            "<h1>...regexp...{}...regexp...</h1>"
+            "<indent>\nMerge rule: joins consecutive chunks that match regexp1 "
+            "and regexp2</indent>\n",
+        ),
+        (
+            "Regexps",
+            "10 60",
+            # "Regular Expression Syntax Summary:\n\n"
+            "<h1>Pattern\t\tMatches...</h1>\n"
+            "<hangindent>"
+            "\t<<var>T</var>>\ta word with tag <var>T</var> "
+            "(where <var>T</var> may be a regexp).\n"
+            "\t<var>x</var>?\tan optional <var>x</var>\n"
+            "\t<var>x</var>+\ta sequence of 1 or more <var>x</var>'s\n"
+            "\t<var>x</var>*\ta sequence of 0 or more <var>x</var>'s\n"
+            "\t<var>x</var>|<var>y</var>\t<var>x</var> or <var>y</var>\n"
+            "\t.\tmatches any character\n"
+            "\t(<var>x</var>)\tTreats <var>x</var> as a group\n"
+            "\t# <var>x...</var>\tTreats <var>x...</var> "
+            "(to the end of the line) as a comment\n"
+            "\t\\<var>C</var>\tmatches character <var>C</var> "
+            "(useful when <var>C</var> is a special character "
+            "like + or #)\n"
+            "</hangindent>"
+            "\n<h1>Examples:</h1>\n"
+            "<hangindent>"
+            "\t<regexp><NN></regexp>\n"
+            '\t\tMatches <match>"cow/NN"</match>\n'
+            '\t\tMatches <match>"green/NN"</match>\n'
+            "\t<regexp><VB.*></regexp>\n"
+            '\t\tMatches <match>"eating/VBG"</match>\n'
+            '\t\tMatches <match>"ate/VBD"</match>\n'
+            "\t<regexp><IN><DT><NN></regexp>\n"
+            '\t\tMatches <match>"on/IN the/DT car/NN"</match>\n'
+            "\t<regexp><RB>?<VBD></regexp>\n"
+            '\t\tMatches <match>"ran/VBD"</match>\n'
+            '\t\tMatches <match>"slowly/RB ate/VBD"</match>\n'
+            r"\t<regexp><\#><CD> # This is a comment...</regexp>\n"
+            '\t\tMatches <match>"#/# 100/CD"</match>\n'
+            "</hangindent>",
+        ),
+        (
+            "Tags",
+            "10 60",
+            "<h1>Part of Speech Tags:</h1>\n"
+            + "<hangindent>"
+            + "<<TAGSET>>"
+            + "</hangindent>\n",  # this gets auto-substituted w/ self.TAGSET
+        ),
+    ]
+
+    HELP_AUTOTAG = [
+        ("red", dict(foreground="#a00")),
+        ("green", dict(foreground="#080")),
+        ("highlight", dict(background="#ddd")),
+        ("underline", dict(underline=True)),
+        ("h1", dict(underline=True)),
+        ("indent", dict(lmargin1=20, lmargin2=20)),
+        ("hangindent", dict(lmargin1=0, lmargin2=60)),
+        ("var", dict(foreground="#88f")),
+        ("regexp", dict(foreground="#ba7")),
+        ("match", dict(foreground="#6a6")),
+    ]
+
+    ##/////////////////////////////////////////////////////////////////
+    ##  Config Parameters
+    ##/////////////////////////////////////////////////////////////////
+
+    _EVAL_DELAY = 1
+    """If the user has not pressed any key for this amount of time (in
+       seconds), and the current grammar has not been evaluated, then
+       the eval demon will evaluate it."""
+
+    _EVAL_CHUNK = 15
+    """The number of sentences that should be evaluated by the eval
+       demon each time it runs."""
+    _EVAL_FREQ = 0.2
+    """The frequency (in seconds) at which the eval demon is run"""
+    _EVAL_DEMON_MIN = 0.02
+    """The minimum amount of time that the eval demon should take each time
+       it runs -- if it takes less than this time, _EVAL_CHUNK will be
+       modified upwards."""
+    _EVAL_DEMON_MAX = 0.04
+    """The maximum amount of time that the eval demon should take each time
+       it runs -- if it takes more than this time, _EVAL_CHUNK will be
+       modified downwards."""
+
+    _GRAMMARBOX_PARAMS = dict(
+        width=40,
+        height=12,
+        background="#efe",
+        highlightbackground="#efe",
+        highlightthickness=1,
+        relief="groove",
+        border=2,
+        wrap="word",
+    )
+    _HELPBOX_PARAMS = dict(
+        width=15,
+        height=15,
+        background="#efe",
+        highlightbackground="#efe",
+        foreground="#555",
+        highlightthickness=1,
+        relief="groove",
+        border=2,
+        wrap="word",
+    )
+    _DEVSETBOX_PARAMS = dict(
+        width=70,
+        height=10,
+        background="#eef",
+        highlightbackground="#eef",
+        highlightthickness=1,
+        relief="groove",
+        border=2,
+        wrap="word",
+        tabs=(30,),
+    )
+    _STATUS_PARAMS = dict(background="#9bb", relief="groove", border=2)
+    _FONT_PARAMS = dict(family="helvetica", size=-20)
+    _FRAME_PARAMS = dict(background="#777", padx=2, pady=2, border=3)
+    _EVALBOX_PARAMS = dict(
+        background="#eef",
+        highlightbackground="#eef",
+        highlightthickness=1,
+        relief="groove",
+        border=2,
+        width=300,
+        height=280,
+    )
+    _BUTTON_PARAMS = dict(
+        background="#777", activebackground="#777", highlightbackground="#777"
+    )
+    _HELPTAB_BG_COLOR = "#aba"
+    _HELPTAB_FG_COLOR = "#efe"
+
+    _HELPTAB_FG_PARAMS = dict(background="#efe")
+    _HELPTAB_BG_PARAMS = dict(background="#aba")
+    _HELPTAB_SPACER = 6
+
+    def normalize_grammar(self, grammar):
+        # Strip comments
+        grammar = re.sub(r"((\\.|[^#])*)(#.*)?", r"\1", grammar)
+        # Normalize whitespace
+        grammar = re.sub(" +", " ", grammar)
+        grammar = re.sub(r"\n\s+", r"\n", grammar)
+        grammar = grammar.strip()
+        # [xx] Hack: automatically backslash $!
+        grammar = re.sub(r"([^\\])\$", r"\1\\$", grammar)
+        return grammar
+
+    def __init__(
+        self,
+        devset_name="conll2000",
+        devset=None,
+        grammar="",
+        chunk_label="NP",
+        tagset=None,
+    ):
+        """
+        :param devset_name: The name of the development set; used for
+            display & for save files.  If either the name 'treebank'
+            or the name 'conll2000' is used, and devset is None, then
+            devset will be set automatically.
+        :param devset: A list of chunked sentences
+        :param grammar: The initial grammar to display.
+        :param tagset: Dictionary from tags to string descriptions, used
+            for the help page.  Defaults to ``self.TAGSET``.
+        """
+        self._chunk_label = chunk_label
+
+        if tagset is None:
+            tagset = self.TAGSET
+        self.tagset = tagset
+
+        # Named development sets:
+        if devset is None:
+            if devset_name == "conll2000":
+                devset = conll2000.chunked_sents("train.txt")  # [:100]
+            elif devset == "treebank":
+                devset = treebank_chunk.chunked_sents()  # [:100]
+            else:
+                raise ValueError("Unknown development set %s" % devset_name)
+
+        self.chunker = None
+        """The chunker built from the grammar string"""
+
+        self.grammar = grammar
+        """The unparsed grammar string"""
+
+        self.normalized_grammar = None
+        """A normalized version of ``self.grammar``."""
+
+        self.grammar_changed = 0
+        """The last time() that the grammar was changed."""
+
+        self.devset = devset
+        """The development set -- a list of chunked sentences."""
+
+        self.devset_name = devset_name
+        """The name of the development set (for save files)."""
+
+        self.devset_index = -1
+        """The index into the development set of the first instance
+           that's currently being viewed."""
+
+        self._last_keypress = 0
+        """The time() when a key was most recently pressed"""
+
+        self._history = []
+        """A list of (grammar, precision, recall, fscore) tuples for
+           grammars that the user has already tried."""
+
+        self._history_index = 0
+        """When the user is scrolling through previous grammars, this
+           is used to keep track of which grammar they're looking at."""
+
+        self._eval_grammar = None
+        """The grammar that is being currently evaluated by the eval
+           demon."""
+
+        self._eval_normalized_grammar = None
+        """A normalized copy of ``_eval_grammar``."""
+
+        self._eval_index = 0
+        """The index of the next sentence in the development set that
+           should be looked at by the eval demon."""
+
+        self._eval_score = ChunkScore(chunk_label=chunk_label)
+        """The ``ChunkScore`` object that's used to keep track of the score
+        of the current grammar on the development set."""
+
+        # Set up the main window.
+        top = self.top = Tk()
+        top.geometry("+50+50")
+        top.title("Regexp Chunk Parser App")
+        top.bind("<Control-q>", self.destroy)
+
+        # Variable that restricts how much of the devset we look at.
+        self._devset_size = IntVar(top)
+        self._devset_size.set(100)
+
+        # Set up all the tkinter widgets
+        self._init_fonts(top)
+        self._init_widgets(top)
+        self._init_bindings(top)
+        self._init_menubar(top)
+        self.grammarbox.focus()
+
+        # If a grammar was given, then display it.
+        if grammar:
+            self.grammarbox.insert("end", grammar + "\n")
+            self.grammarbox.mark_set("insert", "1.0")
+
+        # Display the first item in the development set
+        self.show_devset(0)
+        self.update()
+
+    def _init_bindings(self, top):
+        top.bind("<Control-n>", self._devset_next)
+        top.bind("<Control-p>", self._devset_prev)
+        top.bind("<Control-t>", self.toggle_show_trace)
+        top.bind("<KeyPress>", self.update)
+        top.bind("<Control-s>", lambda e: self.save_grammar())
+        top.bind("<Control-o>", lambda e: self.load_grammar())
+        self.grammarbox.bind("<Control-t>", self.toggle_show_trace)
+        self.grammarbox.bind("<Control-n>", self._devset_next)
+        self.grammarbox.bind("<Control-p>", self._devset_prev)
+
+        # Redraw the eval graph when the window size changes
+        self.evalbox.bind("<Configure>", self._eval_plot)
+
+    def _init_fonts(self, top):
+        # TWhat's our font size (default=same as sysfont)
+        self._size = IntVar(top)
+        self._size.set(20)
+        self._font = Font(family="helvetica", size=-self._size.get())
+        self._smallfont = Font(
+            family="helvetica", size=-(int(self._size.get() * 14 // 20))
+        )
+
+    def _init_menubar(self, parent):
+        menubar = Menu(parent)
+
+        filemenu = Menu(menubar, tearoff=0)
+        filemenu.add_command(label="Reset Application", underline=0, command=self.reset)
+        filemenu.add_command(
+            label="Save Current Grammar",
+            underline=0,
+            accelerator="Ctrl-s",
+            command=self.save_grammar,
+        )
+        filemenu.add_command(
+            label="Load Grammar",
+            underline=0,
+            accelerator="Ctrl-o",
+            command=self.load_grammar,
+        )
+
+        filemenu.add_command(
+            label="Save Grammar History", underline=13, command=self.save_history
+        )
+
+        filemenu.add_command(
+            label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-q"
+        )
+        menubar.add_cascade(label="File", underline=0, menu=filemenu)
+
+        viewmenu = Menu(menubar, tearoff=0)
+        viewmenu.add_radiobutton(
+            label="Tiny",
+            variable=self._size,
+            underline=0,
+            value=10,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Small",
+            variable=self._size,
+            underline=0,
+            value=16,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Medium",
+            variable=self._size,
+            underline=0,
+            value=20,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Large",
+            variable=self._size,
+            underline=0,
+            value=24,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Huge",
+            variable=self._size,
+            underline=0,
+            value=34,
+            command=self.resize,
+        )
+        menubar.add_cascade(label="View", underline=0, menu=viewmenu)
+
+        devsetmenu = Menu(menubar, tearoff=0)
+        devsetmenu.add_radiobutton(
+            label="50 sentences",
+            variable=self._devset_size,
+            value=50,
+            command=self.set_devset_size,
+        )
+        devsetmenu.add_radiobutton(
+            label="100 sentences",
+            variable=self._devset_size,
+            value=100,
+            command=self.set_devset_size,
+        )
+        devsetmenu.add_radiobutton(
+            label="200 sentences",
+            variable=self._devset_size,
+            value=200,
+            command=self.set_devset_size,
+        )
+        devsetmenu.add_radiobutton(
+            label="500 sentences",
+            variable=self._devset_size,
+            value=500,
+            command=self.set_devset_size,
+        )
+        menubar.add_cascade(label="Development-Set", underline=0, menu=devsetmenu)
+
+        helpmenu = Menu(menubar, tearoff=0)
+        helpmenu.add_command(label="About", underline=0, command=self.about)
+        menubar.add_cascade(label="Help", underline=0, menu=helpmenu)
+
+        parent.config(menu=menubar)
+
+    def toggle_show_trace(self, *e):
+        if self._showing_trace:
+            self.show_devset()
+        else:
+            self.show_trace()
+        return "break"
+
+    _SCALE_N = 5  # center on the last 5 examples.
+    _DRAW_LINES = False
+
+    def _eval_plot(self, *e, **config):
+        width = config.get("width", self.evalbox.winfo_width())
+        height = config.get("height", self.evalbox.winfo_height())
+
+        # Clear the canvas
+        self.evalbox.delete("all")
+
+        # Draw the precision & recall labels.
+        tag = self.evalbox.create_text(
+            10, height // 2 - 10, justify="left", anchor="w", text="Precision"
+        )
+        left, right = self.evalbox.bbox(tag)[2] + 5, width - 10
+        tag = self.evalbox.create_text(
+            left + (width - left) // 2,
+            height - 10,
+            anchor="s",
+            text="Recall",
+            justify="center",
+        )
+        top, bot = 10, self.evalbox.bbox(tag)[1] - 10
+
+        # Draw masks for clipping the plot.
+        bg = self._EVALBOX_PARAMS["background"]
+        self.evalbox.lower(
+            self.evalbox.create_rectangle(0, 0, left - 1, 5000, fill=bg, outline=bg)
+        )
+        self.evalbox.lower(
+            self.evalbox.create_rectangle(0, bot + 1, 5000, 5000, fill=bg, outline=bg)
+        )
+
+        # Calculate the plot's scale.
+        if self._autoscale.get() and len(self._history) > 1:
+            max_precision = max_recall = 0
+            min_precision = min_recall = 1
+            for i in range(1, min(len(self._history), self._SCALE_N + 1)):
+                grammar, precision, recall, fmeasure = self._history[-i]
+                min_precision = min(precision, min_precision)
+                min_recall = min(recall, min_recall)
+                max_precision = max(precision, max_precision)
+                max_recall = max(recall, max_recall)
+            #             if max_precision-min_precision > max_recall-min_recall:
+            #                 min_recall -= (max_precision-min_precision)/2
+            #                 max_recall += (max_precision-min_precision)/2
+            #             else:
+            #                 min_precision -= (max_recall-min_recall)/2
+            #                 max_precision += (max_recall-min_recall)/2
+            #             if min_recall < 0:
+            #                 max_recall -= min_recall
+            #                 min_recall = 0
+            #             if min_precision < 0:
+            #                 max_precision -= min_precision
+            #                 min_precision = 0
+            min_precision = max(min_precision - 0.01, 0)
+            min_recall = max(min_recall - 0.01, 0)
+            max_precision = min(max_precision + 0.01, 1)
+            max_recall = min(max_recall + 0.01, 1)
+        else:
+            min_precision = min_recall = 0
+            max_precision = max_recall = 1
+
+        # Draw the axis lines & grid lines
+        for i in range(11):
+            x = left + (right - left) * (
+                (i / 10.0 - min_recall) / (max_recall - min_recall)
+            )
+            y = bot - (bot - top) * (
+                (i / 10.0 - min_precision) / (max_precision - min_precision)
+            )
+            if left < x < right:
+                self.evalbox.create_line(x, top, x, bot, fill="#888")
+            if top < y < bot:
+                self.evalbox.create_line(left, y, right, y, fill="#888")
+        self.evalbox.create_line(left, top, left, bot)
+        self.evalbox.create_line(left, bot, right, bot)
+
+        # Display the plot's scale
+        self.evalbox.create_text(
+            left - 3,
+            bot,
+            justify="right",
+            anchor="se",
+            text="%d%%" % (100 * min_precision),
+        )
+        self.evalbox.create_text(
+            left - 3,
+            top,
+            justify="right",
+            anchor="ne",
+            text="%d%%" % (100 * max_precision),
+        )
+        self.evalbox.create_text(
+            left,
+            bot + 3,
+            justify="center",
+            anchor="nw",
+            text="%d%%" % (100 * min_recall),
+        )
+        self.evalbox.create_text(
+            right,
+            bot + 3,
+            justify="center",
+            anchor="ne",
+            text="%d%%" % (100 * max_recall),
+        )
+
+        # Display the scores.
+        prev_x = prev_y = None
+        for i, (_, precision, recall, fscore) in enumerate(self._history):
+            x = left + (right - left) * (
+                (recall - min_recall) / (max_recall - min_recall)
+            )
+            y = bot - (bot - top) * (
+                (precision - min_precision) / (max_precision - min_precision)
+            )
+            if i == self._history_index:
+                self.evalbox.create_oval(
+                    x - 2, y - 2, x + 2, y + 2, fill="#0f0", outline="#000"
+                )
+                self.status["text"] = (
+                    "Precision: %.2f%%\t" % (precision * 100)
+                    + "Recall: %.2f%%\t" % (recall * 100)
+                    + "F-score: %.2f%%" % (fscore * 100)
+                )
+            else:
+                self.evalbox.lower(
+                    self.evalbox.create_oval(
+                        x - 2, y - 2, x + 2, y + 2, fill="#afa", outline="#8c8"
+                    )
+                )
+            if prev_x is not None and self._eval_lines.get():
+                self.evalbox.lower(
+                    self.evalbox.create_line(prev_x, prev_y, x, y, fill="#8c8")
+                )
+            prev_x, prev_y = x, y
+
+    _eval_demon_running = False
+
+    def _eval_demon(self):
+        if self.top is None:
+            return
+        if self.chunker is None:
+            self._eval_demon_running = False
+            return
+
+        # Note our starting time.
+        t0 = time.time()
+
+        # If are still typing, then wait for them to finish.
+        if (
+            time.time() - self._last_keypress < self._EVAL_DELAY
+            and self.normalized_grammar != self._eval_normalized_grammar
+        ):
+            self._eval_demon_running = True
+            return self.top.after(int(self._EVAL_FREQ * 1000), self._eval_demon)
+
+        # If the grammar changed, restart the evaluation.
+        if self.normalized_grammar != self._eval_normalized_grammar:
+            # Check if we've seen this grammar already.  If so, then
+            # just use the old evaluation values.
+            for (g, p, r, f) in self._history:
+                if self.normalized_grammar == self.normalize_grammar(g):
+                    self._history.append((g, p, r, f))
+                    self._history_index = len(self._history) - 1
+                    self._eval_plot()
+                    self._eval_demon_running = False
+                    self._eval_normalized_grammar = None
+                    return
+            self._eval_index = 0
+            self._eval_score = ChunkScore(chunk_label=self._chunk_label)
+            self._eval_grammar = self.grammar
+            self._eval_normalized_grammar = self.normalized_grammar
+
+        # If the grammar is empty, the don't bother evaluating it, or
+        # recording it in history -- the score will just be 0.
+        if self.normalized_grammar.strip() == "":
+            # self._eval_index = self._devset_size.get()
+            self._eval_demon_running = False
+            return
+
+        # Score the next set of examples
+        for gold in self.devset[
+            self._eval_index : min(
+                self._eval_index + self._EVAL_CHUNK, self._devset_size.get()
+            )
+        ]:
+            guess = self._chunkparse(gold.leaves())
+            self._eval_score.score(gold, guess)
+
+        # update our index in the devset.
+        self._eval_index += self._EVAL_CHUNK
+
+        # Check if we're done
+        if self._eval_index >= self._devset_size.get():
+            self._history.append(
+                (
+                    self._eval_grammar,
+                    self._eval_score.precision(),
+                    self._eval_score.recall(),
+                    self._eval_score.f_measure(),
+                )
+            )
+            self._history_index = len(self._history) - 1
+            self._eval_plot()
+            self._eval_demon_running = False
+            self._eval_normalized_grammar = None
+        else:
+            progress = 100 * self._eval_index / self._devset_size.get()
+            self.status["text"] = "Evaluating on Development Set (%d%%)" % progress
+            self._eval_demon_running = True
+            self._adaptively_modify_eval_chunk(time.time() - t0)
+            self.top.after(int(self._EVAL_FREQ * 1000), self._eval_demon)
+
+    def _adaptively_modify_eval_chunk(self, t):
+        """
+        Modify _EVAL_CHUNK to try to keep the amount of time that the
+        eval demon takes between _EVAL_DEMON_MIN and _EVAL_DEMON_MAX.
+
+        :param t: The amount of time that the eval demon took.
+        """
+        if t > self._EVAL_DEMON_MAX and self._EVAL_CHUNK > 5:
+            self._EVAL_CHUNK = min(
+                self._EVAL_CHUNK - 1,
+                max(
+                    int(self._EVAL_CHUNK * (self._EVAL_DEMON_MAX / t)),
+                    self._EVAL_CHUNK - 10,
+                ),
+            )
+        elif t < self._EVAL_DEMON_MIN:
+            self._EVAL_CHUNK = max(
+                self._EVAL_CHUNK + 1,
+                min(
+                    int(self._EVAL_CHUNK * (self._EVAL_DEMON_MIN / t)),
+                    self._EVAL_CHUNK + 10,
+                ),
+            )
+
+    def _init_widgets(self, top):
+        frame0 = Frame(top, **self._FRAME_PARAMS)
+        frame0.grid_columnconfigure(0, weight=4)
+        frame0.grid_columnconfigure(3, weight=2)
+        frame0.grid_rowconfigure(1, weight=1)
+        frame0.grid_rowconfigure(5, weight=1)
+
+        # The grammar
+        self.grammarbox = Text(frame0, font=self._font, **self._GRAMMARBOX_PARAMS)
+        self.grammarlabel = Label(
+            frame0,
+            font=self._font,
+            text="Grammar:",
+            highlightcolor="black",
+            background=self._GRAMMARBOX_PARAMS["background"],
+        )
+        self.grammarlabel.grid(column=0, row=0, sticky="SW")
+        self.grammarbox.grid(column=0, row=1, sticky="NEWS")
+
+        # Scroll bar for grammar
+        grammar_scrollbar = Scrollbar(frame0, command=self.grammarbox.yview)
+        grammar_scrollbar.grid(column=1, row=1, sticky="NWS")
+        self.grammarbox.config(yscrollcommand=grammar_scrollbar.set)
+
+        # grammar buttons
+        bg = self._FRAME_PARAMS["background"]
+        frame3 = Frame(frame0, background=bg)
+        frame3.grid(column=0, row=2, sticky="EW")
+        Button(
+            frame3,
+            text="Prev Grammar",
+            command=self._history_prev,
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+        Button(
+            frame3,
+            text="Next Grammar",
+            command=self._history_next,
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+
+        # Help box
+        self.helpbox = Text(frame0, font=self._smallfont, **self._HELPBOX_PARAMS)
+        self.helpbox.grid(column=3, row=1, sticky="NEWS")
+        self.helptabs = {}
+        bg = self._FRAME_PARAMS["background"]
+        helptab_frame = Frame(frame0, background=bg)
+        helptab_frame.grid(column=3, row=0, sticky="SW")
+        for i, (tab, tabstops, text) in enumerate(self.HELP):
+            label = Label(helptab_frame, text=tab, font=self._smallfont)
+            label.grid(column=i * 2, row=0, sticky="S")
+            # help_frame.grid_columnconfigure(i, weight=1)
+            # label.pack(side='left')
+            label.bind("<ButtonPress>", lambda e, tab=tab: self.show_help(tab))
+            self.helptabs[tab] = label
+            Frame(
+                helptab_frame, height=1, width=self._HELPTAB_SPACER, background=bg
+            ).grid(column=i * 2 + 1, row=0)
+        self.helptabs[self.HELP[0][0]].configure(font=self._font)
+        self.helpbox.tag_config("elide", elide=True)
+        for (tag, params) in self.HELP_AUTOTAG:
+            self.helpbox.tag_config("tag-%s" % tag, **params)
+        self.show_help(self.HELP[0][0])
+
+        # Scroll bar for helpbox
+        help_scrollbar = Scrollbar(frame0, command=self.helpbox.yview)
+        self.helpbox.config(yscrollcommand=help_scrollbar.set)
+        help_scrollbar.grid(column=4, row=1, sticky="NWS")
+
+        # The dev set
+        frame4 = Frame(frame0, background=self._FRAME_PARAMS["background"])
+        self.devsetbox = Text(frame4, font=self._font, **self._DEVSETBOX_PARAMS)
+        self.devsetbox.pack(expand=True, fill="both")
+        self.devsetlabel = Label(
+            frame0,
+            font=self._font,
+            text="Development Set:",
+            justify="right",
+            background=self._DEVSETBOX_PARAMS["background"],
+        )
+        self.devsetlabel.grid(column=0, row=4, sticky="SW")
+        frame4.grid(column=0, row=5, sticky="NEWS")
+
+        # dev set scrollbars
+        self.devset_scroll = Scrollbar(frame0, command=self._devset_scroll)
+        self.devset_scroll.grid(column=1, row=5, sticky="NWS")
+        self.devset_xscroll = Scrollbar(
+            frame4, command=self.devsetbox.xview, orient="horiz"
+        )
+        self.devsetbox["xscrollcommand"] = self.devset_xscroll.set
+        self.devset_xscroll.pack(side="bottom", fill="x")
+
+        # dev set buttons
+        bg = self._FRAME_PARAMS["background"]
+        frame1 = Frame(frame0, background=bg)
+        frame1.grid(column=0, row=7, sticky="EW")
+        Button(
+            frame1,
+            text="Prev Example (Ctrl-p)",
+            command=self._devset_prev,
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+        Button(
+            frame1,
+            text="Next Example (Ctrl-n)",
+            command=self._devset_next,
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+        self.devset_button = Button(
+            frame1,
+            text="Show example",
+            command=self.show_devset,
+            state="disabled",
+            **self._BUTTON_PARAMS,
+        )
+        self.devset_button.pack(side="right")
+        self.trace_button = Button(
+            frame1, text="Show trace", command=self.show_trace, **self._BUTTON_PARAMS
+        )
+        self.trace_button.pack(side="right")
+
+        # evaluation box
+        self.evalbox = Canvas(frame0, **self._EVALBOX_PARAMS)
+        label = Label(
+            frame0,
+            font=self._font,
+            text="Evaluation:",
+            justify="right",
+            background=self._EVALBOX_PARAMS["background"],
+        )
+        label.grid(column=3, row=4, sticky="SW")
+        self.evalbox.grid(column=3, row=5, sticky="NEWS", columnspan=2)
+
+        # evaluation box buttons
+        bg = self._FRAME_PARAMS["background"]
+        frame2 = Frame(frame0, background=bg)
+        frame2.grid(column=3, row=7, sticky="EW")
+        self._autoscale = IntVar(self.top)
+        self._autoscale.set(False)
+        Checkbutton(
+            frame2,
+            variable=self._autoscale,
+            command=self._eval_plot,
+            text="Zoom",
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+        self._eval_lines = IntVar(self.top)
+        self._eval_lines.set(False)
+        Checkbutton(
+            frame2,
+            variable=self._eval_lines,
+            command=self._eval_plot,
+            text="Lines",
+            **self._BUTTON_PARAMS,
+        ).pack(side="left")
+        Button(frame2, text="History", **self._BUTTON_PARAMS).pack(side="right")
+
+        # The status label
+        self.status = Label(frame0, font=self._font, **self._STATUS_PARAMS)
+        self.status.grid(column=0, row=9, sticky="NEW", padx=3, pady=2, columnspan=5)
+
+        # Help box & devset box can't be edited.
+        self.helpbox["state"] = "disabled"
+        self.devsetbox["state"] = "disabled"
+
+        # Spacers
+        bg = self._FRAME_PARAMS["background"]
+        Frame(frame0, height=10, width=0, background=bg).grid(column=0, row=3)
+        Frame(frame0, height=0, width=10, background=bg).grid(column=2, row=0)
+        Frame(frame0, height=6, width=0, background=bg).grid(column=0, row=8)
+
+        # pack the frame.
+        frame0.pack(fill="both", expand=True)
+
+        # Set up colors for the devset box
+        self.devsetbox.tag_config("true-pos", background="#afa", underline="True")
+        self.devsetbox.tag_config("false-neg", underline="True", foreground="#800")
+        self.devsetbox.tag_config("false-pos", background="#faa")
+        self.devsetbox.tag_config("trace", foreground="#666", wrap="none")
+        self.devsetbox.tag_config("wrapindent", lmargin2=30, wrap="none")
+        self.devsetbox.tag_config("error", foreground="#800")
+
+        # And for the grammarbox
+        self.grammarbox.tag_config("error", background="#fec")
+        self.grammarbox.tag_config("comment", foreground="#840")
+        self.grammarbox.tag_config("angle", foreground="#00f")
+        self.grammarbox.tag_config("brace", foreground="#0a0")
+        self.grammarbox.tag_config("hangindent", lmargin1=0, lmargin2=40)
+
+    _showing_trace = False
+
+    def show_trace(self, *e):
+        self._showing_trace = True
+        self.trace_button["state"] = "disabled"
+        self.devset_button["state"] = "normal"
+
+        self.devsetbox["state"] = "normal"
+        # self.devsetbox['wrap'] = 'none'
+        self.devsetbox.delete("1.0", "end")
+        self.devsetlabel["text"] = "Development Set (%d/%d)" % (
+            (self.devset_index + 1, self._devset_size.get())
+        )
+
+        if self.chunker is None:
+            self.devsetbox.insert("1.0", "Trace: waiting for a valid grammar.")
+            self.devsetbox.tag_add("error", "1.0", "end")
+            return  # can't do anything more
+
+        gold_tree = self.devset[self.devset_index]
+        rules = self.chunker.rules()
+
+        # Calculate the tag sequence
+        tagseq = "\t"
+        charnum = [1]
+        for wordnum, (word, pos) in enumerate(gold_tree.leaves()):
+            tagseq += "%s " % pos
+            charnum.append(len(tagseq))
+        self.charnum = {
+            (i, j): charnum[j]
+            for i in range(len(rules) + 1)
+            for j in range(len(charnum))
+        }
+        self.linenum = {i: i * 2 + 2 for i in range(len(rules) + 1)}
+
+        for i in range(len(rules) + 1):
+            if i == 0:
+                self.devsetbox.insert("end", "Start:\n")
+                self.devsetbox.tag_add("trace", "end -2c linestart", "end -2c")
+            else:
+                self.devsetbox.insert("end", "Apply %s:\n" % rules[i - 1])
+                self.devsetbox.tag_add("trace", "end -2c linestart", "end -2c")
+            # Display the tag sequence.
+            self.devsetbox.insert("end", tagseq + "\n")
+            self.devsetbox.tag_add("wrapindent", "end -2c linestart", "end -2c")
+            # Run a partial parser, and extract gold & test chunks
+            chunker = RegexpChunkParser(rules[:i])
+            test_tree = self._chunkparse(gold_tree.leaves())
+            gold_chunks = self._chunks(gold_tree)
+            test_chunks = self._chunks(test_tree)
+            # Compare them.
+            for chunk in gold_chunks.intersection(test_chunks):
+                self._color_chunk(i, chunk, "true-pos")
+            for chunk in gold_chunks - test_chunks:
+                self._color_chunk(i, chunk, "false-neg")
+            for chunk in test_chunks - gold_chunks:
+                self._color_chunk(i, chunk, "false-pos")
+        self.devsetbox.insert("end", "Finished.\n")
+        self.devsetbox.tag_add("trace", "end -2c linestart", "end -2c")
+
+        # This is a hack, because the x-scrollbar isn't updating its
+        # position right -- I'm not sure what the underlying cause is
+        # though.  (This is on OS X w/ python 2.5)
+        self.top.after(100, self.devset_xscroll.set, 0, 0.3)
+
+    def show_help(self, tab):
+        self.helpbox["state"] = "normal"
+        self.helpbox.delete("1.0", "end")
+        for (name, tabstops, text) in self.HELP:
+            if name == tab:
+                text = text.replace(
+                    "<<TAGSET>>",
+                    "\n".join(
+                        "\t%s\t%s" % item
+                        for item in sorted(
+                            list(self.tagset.items()),
+                            key=lambda t_w: re.match(r"\w+", t_w[0])
+                            and (0, t_w[0])
+                            or (1, t_w[0]),
+                        )
+                    ),
+                )
+
+                self.helptabs[name].config(**self._HELPTAB_FG_PARAMS)
+                self.helpbox.config(tabs=tabstops)
+                self.helpbox.insert("1.0", text + "\n" * 20)
+                C = "1.0 + %d chars"
+                for (tag, params) in self.HELP_AUTOTAG:
+                    pattern = f"(?s)(<{tag}>)(.*?)(</{tag}>)"
+                    for m in re.finditer(pattern, text):
+                        self.helpbox.tag_add("elide", C % m.start(1), C % m.end(1))
+                        self.helpbox.tag_add(
+                            "tag-%s" % tag, C % m.start(2), C % m.end(2)
+                        )
+                        self.helpbox.tag_add("elide", C % m.start(3), C % m.end(3))
+            else:
+                self.helptabs[name].config(**self._HELPTAB_BG_PARAMS)
+        self.helpbox["state"] = "disabled"
+
+    def _history_prev(self, *e):
+        self._view_history(self._history_index - 1)
+        return "break"
+
+    def _history_next(self, *e):
+        self._view_history(self._history_index + 1)
+        return "break"
+
+    def _view_history(self, index):
+        # Bounds & sanity checking:
+        index = max(0, min(len(self._history) - 1, index))
+        if not self._history:
+            return
+        # Already viewing the requested history item?
+        if index == self._history_index:
+            return
+        # Show the requested grammar.  It will get added to _history
+        # only if they edit it (causing self.update() to get run.)
+        self.grammarbox["state"] = "normal"
+        self.grammarbox.delete("1.0", "end")
+        self.grammarbox.insert("end", self._history[index][0])
+        self.grammarbox.mark_set("insert", "1.0")
+        self._history_index = index
+        self._syntax_highlight_grammar(self._history[index][0])
+        # Record the normalized grammar & regenerate the chunker.
+        self.normalized_grammar = self.normalize_grammar(self._history[index][0])
+        if self.normalized_grammar:
+            rules = [
+                RegexpChunkRule.fromstring(line)
+                for line in self.normalized_grammar.split("\n")
+            ]
+        else:
+            rules = []
+        self.chunker = RegexpChunkParser(rules)
+        # Show the score.
+        self._eval_plot()
+        # Update the devset box
+        self._highlight_devset()
+        if self._showing_trace:
+            self.show_trace()
+        # Update the grammar label
+        if self._history_index < len(self._history) - 1:
+            self.grammarlabel["text"] = "Grammar {}/{}:".format(
+                self._history_index + 1,
+                len(self._history),
+            )
+        else:
+            self.grammarlabel["text"] = "Grammar:"
+
+    def _devset_next(self, *e):
+        self._devset_scroll("scroll", 1, "page")
+        return "break"
+
+    def _devset_prev(self, *e):
+        self._devset_scroll("scroll", -1, "page")
+        return "break"
+
+    def destroy(self, *e):
+        if self.top is None:
+            return
+        self.top.destroy()
+        self.top = None
+
+    def _devset_scroll(self, command, *args):
+        N = 1  # size of a page -- one sentence.
+        showing_trace = self._showing_trace
+        if command == "scroll" and args[1].startswith("unit"):
+            self.show_devset(self.devset_index + int(args[0]))
+        elif command == "scroll" and args[1].startswith("page"):
+            self.show_devset(self.devset_index + N * int(args[0]))
+        elif command == "moveto":
+            self.show_devset(int(float(args[0]) * self._devset_size.get()))
+        else:
+            assert 0, f"bad scroll command {command} {args}"
+        if showing_trace:
+            self.show_trace()
+
+    def show_devset(self, index=None):
+        if index is None:
+            index = self.devset_index
+
+        # Bounds checking
+        index = min(max(0, index), self._devset_size.get() - 1)
+
+        if index == self.devset_index and not self._showing_trace:
+            return
+        self.devset_index = index
+
+        self._showing_trace = False
+        self.trace_button["state"] = "normal"
+        self.devset_button["state"] = "disabled"
+
+        # Clear the text box.
+        self.devsetbox["state"] = "normal"
+        self.devsetbox["wrap"] = "word"
+        self.devsetbox.delete("1.0", "end")
+        self.devsetlabel["text"] = "Development Set (%d/%d)" % (
+            (self.devset_index + 1, self._devset_size.get())
+        )
+
+        # Add the sentences
+        sample = self.devset[self.devset_index : self.devset_index + 1]
+        self.charnum = {}
+        self.linenum = {0: 1}
+        for sentnum, sent in enumerate(sample):
+            linestr = ""
+            for wordnum, (word, pos) in enumerate(sent.leaves()):
+                self.charnum[sentnum, wordnum] = len(linestr)
+                linestr += f"{word}/{pos} "
+                self.charnum[sentnum, wordnum + 1] = len(linestr)
+            self.devsetbox.insert("end", linestr[:-1] + "\n\n")
+
+        # Highlight chunks in the dev set
+        if self.chunker is not None:
+            self._highlight_devset()
+        self.devsetbox["state"] = "disabled"
+
+        # Update the scrollbar
+        first = self.devset_index / self._devset_size.get()
+        last = (self.devset_index + 2) / self._devset_size.get()
+        self.devset_scroll.set(first, last)
+
+    def _chunks(self, tree):
+        chunks = set()
+        wordnum = 0
+        for child in tree:
+            if isinstance(child, Tree):
+                if child.label() == self._chunk_label:
+                    chunks.add((wordnum, wordnum + len(child)))
+                wordnum += len(child)
+            else:
+                wordnum += 1
+        return chunks
+
+    def _syntax_highlight_grammar(self, grammar):
+        if self.top is None:
+            return
+        self.grammarbox.tag_remove("comment", "1.0", "end")
+        self.grammarbox.tag_remove("angle", "1.0", "end")
+        self.grammarbox.tag_remove("brace", "1.0", "end")
+        self.grammarbox.tag_add("hangindent", "1.0", "end")
+        for lineno, line in enumerate(grammar.split("\n")):
+            if not line.strip():
+                continue
+            m = re.match(r"(\\.|[^#])*(#.*)?", line)
+            comment_start = None
+            if m.group(2):
+                comment_start = m.start(2)
+                s = "%d.%d" % (lineno + 1, m.start(2))
+                e = "%d.%d" % (lineno + 1, m.end(2))
+                self.grammarbox.tag_add("comment", s, e)
+            for m in re.finditer("[<>{}]", line):
+                if comment_start is not None and m.start() >= comment_start:
+                    break
+                s = "%d.%d" % (lineno + 1, m.start())
+                e = "%d.%d" % (lineno + 1, m.end())
+                if m.group() in "<>":
+                    self.grammarbox.tag_add("angle", s, e)
+                else:
+                    self.grammarbox.tag_add("brace", s, e)
+
+    def _grammarcheck(self, grammar):
+        if self.top is None:
+            return
+        self.grammarbox.tag_remove("error", "1.0", "end")
+        self._grammarcheck_errs = []
+        for lineno, line in enumerate(grammar.split("\n")):
+            line = re.sub(r"((\\.|[^#])*)(#.*)?", r"\1", line)
+            line = line.strip()
+            if line:
+                try:
+                    RegexpChunkRule.fromstring(line)
+                except ValueError as e:
+                    self.grammarbox.tag_add(
+                        "error", "%s.0" % (lineno + 1), "%s.0 lineend" % (lineno + 1)
+                    )
+        self.status["text"] = ""
+
+    def update(self, *event):
+        # Record when update was called (for grammarcheck)
+        if event:
+            self._last_keypress = time.time()
+
+        # Read the grammar from the Text box.
+        self.grammar = grammar = self.grammarbox.get("1.0", "end")
+
+        # If the grammar hasn't changed, do nothing:
+        normalized_grammar = self.normalize_grammar(grammar)
+        if normalized_grammar == self.normalized_grammar:
+            return
+        else:
+            self.normalized_grammar = normalized_grammar
+
+        # If the grammar has changed, and we're looking at history,
+        # then stop looking at history.
+        if self._history_index < len(self._history) - 1:
+            self.grammarlabel["text"] = "Grammar:"
+
+        self._syntax_highlight_grammar(grammar)
+
+        # The grammar has changed; try parsing it.  If it doesn't
+        # parse, do nothing.  (flag error location?)
+        try:
+            # Note: the normalized grammar has no blank lines.
+            if normalized_grammar:
+                rules = [
+                    RegexpChunkRule.fromstring(line)
+                    for line in normalized_grammar.split("\n")
+                ]
+            else:
+                rules = []
+        except ValueError as e:
+            # Use the un-normalized grammar for error highlighting.
+            self._grammarcheck(grammar)
+            self.chunker = None
+            return
+
+        self.chunker = RegexpChunkParser(rules)
+        self.grammarbox.tag_remove("error", "1.0", "end")
+        self.grammar_changed = time.time()
+        # Display the results
+        if self._showing_trace:
+            self.show_trace()
+        else:
+            self._highlight_devset()
+        # Start the eval demon
+        if not self._eval_demon_running:
+            self._eval_demon()
+
+    def _highlight_devset(self, sample=None):
+        if sample is None:
+            sample = self.devset[self.devset_index : self.devset_index + 1]
+
+        self.devsetbox.tag_remove("true-pos", "1.0", "end")
+        self.devsetbox.tag_remove("false-neg", "1.0", "end")
+        self.devsetbox.tag_remove("false-pos", "1.0", "end")
+
+        # Run the grammar on the test cases.
+        for sentnum, gold_tree in enumerate(sample):
+            # Run the chunk parser
+            test_tree = self._chunkparse(gold_tree.leaves())
+            # Extract gold & test chunks
+            gold_chunks = self._chunks(gold_tree)
+            test_chunks = self._chunks(test_tree)
+            # Compare them.
+            for chunk in gold_chunks.intersection(test_chunks):
+                self._color_chunk(sentnum, chunk, "true-pos")
+            for chunk in gold_chunks - test_chunks:
+                self._color_chunk(sentnum, chunk, "false-neg")
+            for chunk in test_chunks - gold_chunks:
+                self._color_chunk(sentnum, chunk, "false-pos")
+
+    def _chunkparse(self, words):
+        try:
+            return self.chunker.parse(words)
+        except (ValueError, IndexError) as e:
+            # There's an error somewhere in the grammar, but we're not sure
+            # exactly where, so just mark the whole grammar as bad.
+            # E.g., this is caused by: "({<NN>})"
+            self.grammarbox.tag_add("error", "1.0", "end")
+            # Treat it as tagging nothing:
+            return words
+
+    def _color_chunk(self, sentnum, chunk, tag):
+        start, end = chunk
+        self.devsetbox.tag_add(
+            tag,
+            f"{self.linenum[sentnum]}.{self.charnum[sentnum, start]}",
+            f"{self.linenum[sentnum]}.{self.charnum[sentnum, end] - 1}",
+        )
+
+    def reset(self):
+        # Clear various variables
+        self.chunker = None
+        self.grammar = None
+        self.normalized_grammar = None
+        self.grammar_changed = 0
+        self._history = []
+        self._history_index = 0
+        # Update the on-screen display.
+        self.grammarbox.delete("1.0", "end")
+        self.show_devset(0)
+        self.update()
+        # self._eval_plot()
+
+    SAVE_GRAMMAR_TEMPLATE = (
+        "# Regexp Chunk Parsing Grammar\n"
+        "# Saved %(date)s\n"
+        "#\n"
+        "# Development set: %(devset)s\n"
+        "#   Precision: %(precision)s\n"
+        "#   Recall:    %(recall)s\n"
+        "#   F-score:   %(fscore)s\n\n"
+        "%(grammar)s\n"
+    )
+
+    def save_grammar(self, filename=None):
+        if not filename:
+            ftypes = [("Chunk Gramamr", ".chunk"), ("All files", "*")]
+            filename = asksaveasfilename(filetypes=ftypes, defaultextension=".chunk")
+            if not filename:
+                return
+        if self._history and self.normalized_grammar == self.normalize_grammar(
+            self._history[-1][0]
+        ):
+            precision, recall, fscore = (
+                "%.2f%%" % (100 * v) for v in self._history[-1][1:]
+            )
+        elif self.chunker is None:
+            precision = recall = fscore = "Grammar not well formed"
+        else:
+            precision = recall = fscore = "Not finished evaluation yet"
+
+        with open(filename, "w") as outfile:
+            outfile.write(
+                self.SAVE_GRAMMAR_TEMPLATE
+                % dict(
+                    date=time.ctime(),
+                    devset=self.devset_name,
+                    precision=precision,
+                    recall=recall,
+                    fscore=fscore,
+                    grammar=self.grammar.strip(),
+                )
+            )
+
+    def load_grammar(self, filename=None):
+        if not filename:
+            ftypes = [("Chunk Gramamr", ".chunk"), ("All files", "*")]
+            filename = askopenfilename(filetypes=ftypes, defaultextension=".chunk")
+            if not filename:
+                return
+        self.grammarbox.delete("1.0", "end")
+        self.update()
+        with open(filename) as infile:
+            grammar = infile.read()
+        grammar = re.sub(
+            r"^\# Regexp Chunk Parsing Grammar[\s\S]*" "F-score:.*\n", "", grammar
+        ).lstrip()
+        self.grammarbox.insert("1.0", grammar)
+        self.update()
+
+    def save_history(self, filename=None):
+        if not filename:
+            ftypes = [("Chunk Gramamr History", ".txt"), ("All files", "*")]
+            filename = asksaveasfilename(filetypes=ftypes, defaultextension=".txt")
+            if not filename:
+                return
+
+        with open(filename, "w") as outfile:
+            outfile.write("# Regexp Chunk Parsing Grammar History\n")
+            outfile.write("# Saved %s\n" % time.ctime())
+            outfile.write("# Development set: %s\n" % self.devset_name)
+            for i, (g, p, r, f) in enumerate(self._history):
+                hdr = (
+                    "Grammar %d/%d (precision=%.2f%%, recall=%.2f%%, "
+                    "fscore=%.2f%%)"
+                    % (i + 1, len(self._history), p * 100, r * 100, f * 100)
+                )
+                outfile.write("\n%s\n" % hdr)
+                outfile.write("".join("  %s\n" % line for line in g.strip().split()))
+
+            if not (
+                self._history
+                and self.normalized_grammar
+                == self.normalize_grammar(self._history[-1][0])
+            ):
+                if self.chunker is None:
+                    outfile.write("\nCurrent Grammar (not well-formed)\n")
+                else:
+                    outfile.write("\nCurrent Grammar (not evaluated)\n")
+                outfile.write(
+                    "".join("  %s\n" % line for line in self.grammar.strip().split())
+                )
+
+    def about(self, *e):
+        ABOUT = "NLTK RegExp Chunk Parser Application\n" + "Written by Edward Loper"
+        TITLE = "About: Regular Expression Chunk Parser Application"
+        try:
+            from tkinter.messagebox import Message
+
+            Message(message=ABOUT, title=TITLE).show()
+        except:
+            ShowText(self.top, TITLE, ABOUT)
+
+    def set_devset_size(self, size=None):
+        if size is not None:
+            self._devset_size.set(size)
+        self._devset_size.set(min(len(self.devset), self._devset_size.get()))
+        self.show_devset(1)
+        self.show_devset(0)
+        # what about history?  Evaluated at diff dev set sizes!
+
+    def resize(self, size=None):
+        if size is not None:
+            self._size.set(size)
+        size = self._size.get()
+        self._font.configure(size=-(abs(size)))
+        self._smallfont.configure(size=min(-10, -(abs(size)) * 14 // 20))
+
+    def mainloop(self, *args, **kwargs):
+        """
+        Enter the Tkinter mainloop.  This function must be called if
+        this demo is created from a non-interactive program (e.g.
+        from a secript); otherwise, the demo will close as soon as
+        the script completes.
+        """
+        if in_idle():
+            return
+        self.top.mainloop(*args, **kwargs)
+
+
+def app():
+    RegexpChunkApp().mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/collocations_app.py

@@ -0,0 +1,438 @@
+# Natural Language Toolkit: Collocations Application
+# Much of the GUI code is imported from concordance.py; We intend to merge these tools together
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Sumukh Ghodke <sghodke@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+
+import queue as q
+import threading
+from tkinter import (
+    END,
+    LEFT,
+    SUNKEN,
+    Button,
+    Frame,
+    IntVar,
+    Label,
+    Menu,
+    OptionMenu,
+    Scrollbar,
+    StringVar,
+    Text,
+    Tk,
+)
+from tkinter.font import Font
+
+from nltk.corpus import (
+    alpino,
+    brown,
+    cess_cat,
+    cess_esp,
+    floresta,
+    indian,
+    mac_morpho,
+    machado,
+    nps_chat,
+    sinica_treebank,
+    treebank,
+)
+from nltk.probability import FreqDist
+from nltk.util import in_idle
+
+CORPUS_LOADED_EVENT = "<<CL_EVENT>>"
+ERROR_LOADING_CORPUS_EVENT = "<<ELC_EVENT>>"
+POLL_INTERVAL = 100
+
+_DEFAULT = "English: Brown Corpus (Humor)"
+_CORPORA = {
+    "Catalan: CESS-CAT Corpus": lambda: cess_cat.words(),
+    "English: Brown Corpus": lambda: brown.words(),
+    "English: Brown Corpus (Press)": lambda: brown.words(
+        categories=["news", "editorial", "reviews"]
+    ),
+    "English: Brown Corpus (Religion)": lambda: brown.words(categories="religion"),
+    "English: Brown Corpus (Learned)": lambda: brown.words(categories="learned"),
+    "English: Brown Corpus (Science Fiction)": lambda: brown.words(
+        categories="science_fiction"
+    ),
+    "English: Brown Corpus (Romance)": lambda: brown.words(categories="romance"),
+    "English: Brown Corpus (Humor)": lambda: brown.words(categories="humor"),
+    "English: NPS Chat Corpus": lambda: nps_chat.words(),
+    "English: Wall Street Journal Corpus": lambda: treebank.words(),
+    "Chinese: Sinica Corpus": lambda: sinica_treebank.words(),
+    "Dutch: Alpino Corpus": lambda: alpino.words(),
+    "Hindi: Indian Languages Corpus": lambda: indian.words(files="hindi.pos"),
+    "Portuguese: Floresta Corpus (Portugal)": lambda: floresta.words(),
+    "Portuguese: MAC-MORPHO Corpus (Brazil)": lambda: mac_morpho.words(),
+    "Portuguese: Machado Corpus (Brazil)": lambda: machado.words(),
+    "Spanish: CESS-ESP Corpus": lambda: cess_esp.words(),
+}
+
+
+class CollocationsView:
+    _BACKGROUND_COLOUR = "#FFF"  # white
+
+    def __init__(self):
+        self.queue = q.Queue()
+        self.model = CollocationsModel(self.queue)
+        self.top = Tk()
+        self._init_top(self.top)
+        self._init_menubar()
+        self._init_widgets(self.top)
+        self.load_corpus(self.model.DEFAULT_CORPUS)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+    def _init_top(self, top):
+        top.geometry("550x650+50+50")
+        top.title("NLTK Collocations List")
+        top.bind("<Control-q>", self.destroy)
+        top.protocol("WM_DELETE_WINDOW", self.destroy)
+        top.minsize(550, 650)
+
+    def _init_widgets(self, parent):
+        self.main_frame = Frame(
+            parent, dict(background=self._BACKGROUND_COLOUR, padx=1, pady=1, border=1)
+        )
+        self._init_corpus_select(self.main_frame)
+        self._init_results_box(self.main_frame)
+        self._init_paging(self.main_frame)
+        self._init_status(self.main_frame)
+        self.main_frame.pack(fill="both", expand=True)
+
+    def _init_corpus_select(self, parent):
+        innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+        self.var = StringVar(innerframe)
+        self.var.set(self.model.DEFAULT_CORPUS)
+        Label(
+            innerframe,
+            justify=LEFT,
+            text=" Corpus: ",
+            background=self._BACKGROUND_COLOUR,
+            padx=2,
+            pady=1,
+            border=0,
+        ).pack(side="left")
+
+        other_corpora = list(self.model.CORPORA.keys()).remove(
+            self.model.DEFAULT_CORPUS
+        )
+        om = OptionMenu(
+            innerframe,
+            self.var,
+            self.model.DEFAULT_CORPUS,
+            command=self.corpus_selected,
+            *self.model.non_default_corpora()
+        )
+        om["borderwidth"] = 0
+        om["highlightthickness"] = 1
+        om.pack(side="left")
+        innerframe.pack(side="top", fill="x", anchor="n")
+
+    def _init_status(self, parent):
+        self.status = Label(
+            parent,
+            justify=LEFT,
+            relief=SUNKEN,
+            background=self._BACKGROUND_COLOUR,
+            border=0,
+            padx=1,
+            pady=0,
+        )
+        self.status.pack(side="top", anchor="sw")
+
+    def _init_menubar(self):
+        self._result_size = IntVar(self.top)
+        menubar = Menu(self.top)
+
+        filemenu = Menu(menubar, tearoff=0, borderwidth=0)
+        filemenu.add_command(
+            label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-q"
+        )
+        menubar.add_cascade(label="File", underline=0, menu=filemenu)
+
+        editmenu = Menu(menubar, tearoff=0)
+        rescntmenu = Menu(editmenu, tearoff=0)
+        rescntmenu.add_radiobutton(
+            label="20",
+            variable=self._result_size,
+            underline=0,
+            value=20,
+            command=self.set_result_size,
+        )
+        rescntmenu.add_radiobutton(
+            label="50",
+            variable=self._result_size,
+            underline=0,
+            value=50,
+            command=self.set_result_size,
+        )
+        rescntmenu.add_radiobutton(
+            label="100",
+            variable=self._result_size,
+            underline=0,
+            value=100,
+            command=self.set_result_size,
+        )
+        rescntmenu.invoke(1)
+        editmenu.add_cascade(label="Result Count", underline=0, menu=rescntmenu)
+
+        menubar.add_cascade(label="Edit", underline=0, menu=editmenu)
+        self.top.config(menu=menubar)
+
+    def set_result_size(self, **kwargs):
+        self.model.result_count = self._result_size.get()
+
+    def _init_results_box(self, parent):
+        innerframe = Frame(parent)
+        i1 = Frame(innerframe)
+        i2 = Frame(innerframe)
+        vscrollbar = Scrollbar(i1, borderwidth=1)
+        hscrollbar = Scrollbar(i2, borderwidth=1, orient="horiz")
+        self.results_box = Text(
+            i1,
+            font=Font(family="courier", size="16"),
+            state="disabled",
+            borderwidth=1,
+            yscrollcommand=vscrollbar.set,
+            xscrollcommand=hscrollbar.set,
+            wrap="none",
+            width="40",
+            height="20",
+            exportselection=1,
+        )
+        self.results_box.pack(side="left", fill="both", expand=True)
+        vscrollbar.pack(side="left", fill="y", anchor="e")
+        vscrollbar.config(command=self.results_box.yview)
+        hscrollbar.pack(side="left", fill="x", expand=True, anchor="w")
+        hscrollbar.config(command=self.results_box.xview)
+        # there is no other way of avoiding the overlap of scrollbars while using pack layout manager!!!
+        Label(i2, text="   ", background=self._BACKGROUND_COLOUR).pack(
+            side="left", anchor="e"
+        )
+        i1.pack(side="top", fill="both", expand=True, anchor="n")
+        i2.pack(side="bottom", fill="x", anchor="s")
+        innerframe.pack(side="top", fill="both", expand=True)
+
+    def _init_paging(self, parent):
+        innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+        self.prev = prev = Button(
+            innerframe,
+            text="Previous",
+            command=self.previous,
+            width="10",
+            borderwidth=1,
+            highlightthickness=1,
+            state="disabled",
+        )
+        prev.pack(side="left", anchor="center")
+        self.next = next = Button(
+            innerframe,
+            text="Next",
+            command=self.__next__,
+            width="10",
+            borderwidth=1,
+            highlightthickness=1,
+            state="disabled",
+        )
+        next.pack(side="right", anchor="center")
+        innerframe.pack(side="top", fill="y")
+        self.reset_current_page()
+
+    def reset_current_page(self):
+        self.current_page = -1
+
+    def _poll(self):
+        try:
+            event = self.queue.get(block=False)
+        except q.Empty:
+            pass
+        else:
+            if event == CORPUS_LOADED_EVENT:
+                self.handle_corpus_loaded(event)
+            elif event == ERROR_LOADING_CORPUS_EVENT:
+                self.handle_error_loading_corpus(event)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+    def handle_error_loading_corpus(self, event):
+        self.status["text"] = "Error in loading " + self.var.get()
+        self.unfreeze_editable()
+        self.clear_results_box()
+        self.freeze_editable()
+        self.reset_current_page()
+
+    def handle_corpus_loaded(self, event):
+        self.status["text"] = self.var.get() + " is loaded"
+        self.unfreeze_editable()
+        self.clear_results_box()
+        self.reset_current_page()
+        # self.next()
+        collocations = self.model.next(self.current_page + 1)
+        self.write_results(collocations)
+        self.current_page += 1
+
+    def corpus_selected(self, *args):
+        new_selection = self.var.get()
+        self.load_corpus(new_selection)
+
+    def previous(self):
+        self.freeze_editable()
+        collocations = self.model.prev(self.current_page - 1)
+        self.current_page = self.current_page - 1
+        self.clear_results_box()
+        self.write_results(collocations)
+        self.unfreeze_editable()
+
+    def __next__(self):
+        self.freeze_editable()
+        collocations = self.model.next(self.current_page + 1)
+        self.clear_results_box()
+        self.write_results(collocations)
+        self.current_page += 1
+        self.unfreeze_editable()
+
+    def load_corpus(self, selection):
+        if self.model.selected_corpus != selection:
+            self.status["text"] = "Loading " + selection + "..."
+            self.freeze_editable()
+            self.model.load_corpus(selection)
+
+    def freeze_editable(self):
+        self.prev["state"] = "disabled"
+        self.next["state"] = "disabled"
+
+    def clear_results_box(self):
+        self.results_box["state"] = "normal"
+        self.results_box.delete("1.0", END)
+        self.results_box["state"] = "disabled"
+
+    def fire_event(self, event):
+        # Firing an event so that rendering of widgets happen in the mainloop thread
+        self.top.event_generate(event, when="tail")
+
+    def destroy(self, *e):
+        if self.top is None:
+            return
+        self.top.after_cancel(self.after)
+        self.top.destroy()
+        self.top = None
+
+    def mainloop(self, *args, **kwargs):
+        if in_idle():
+            return
+        self.top.mainloop(*args, **kwargs)
+
+    def unfreeze_editable(self):
+        self.set_paging_button_states()
+
+    def set_paging_button_states(self):
+        if self.current_page == -1 or self.current_page == 0:
+            self.prev["state"] = "disabled"
+        else:
+            self.prev["state"] = "normal"
+        if self.model.is_last_page(self.current_page):
+            self.next["state"] = "disabled"
+        else:
+            self.next["state"] = "normal"
+
+    def write_results(self, results):
+        self.results_box["state"] = "normal"
+        row = 1
+        for each in results:
+            self.results_box.insert(str(row) + ".0", each[0] + " " + each[1] + "\n")
+            row += 1
+        self.results_box["state"] = "disabled"
+
+
+class CollocationsModel:
+    def __init__(self, queue):
+        self.result_count = None
+        self.selected_corpus = None
+        self.collocations = None
+        self.CORPORA = _CORPORA
+        self.DEFAULT_CORPUS = _DEFAULT
+        self.queue = queue
+        self.reset_results()
+
+    def reset_results(self):
+        self.result_pages = []
+        self.results_returned = 0
+
+    def load_corpus(self, name):
+        self.selected_corpus = name
+        self.collocations = None
+        runner_thread = self.LoadCorpus(name, self)
+        runner_thread.start()
+        self.reset_results()
+
+    def non_default_corpora(self):
+        copy = []
+        copy.extend(list(self.CORPORA.keys()))
+        copy.remove(self.DEFAULT_CORPUS)
+        copy.sort()
+        return copy
+
+    def is_last_page(self, number):
+        if number < len(self.result_pages):
+            return False
+        return self.results_returned + (
+            number - len(self.result_pages)
+        ) * self.result_count >= len(self.collocations)
+
+    def next(self, page):
+        if (len(self.result_pages) - 1) < page:
+            for i in range(page - (len(self.result_pages) - 1)):
+                self.result_pages.append(
+                    self.collocations[
+                        self.results_returned : self.results_returned
+                        + self.result_count
+                    ]
+                )
+                self.results_returned += self.result_count
+        return self.result_pages[page]
+
+    def prev(self, page):
+        if page == -1:
+            return []
+        return self.result_pages[page]
+
+    class LoadCorpus(threading.Thread):
+        def __init__(self, name, model):
+            threading.Thread.__init__(self)
+            self.model, self.name = model, name
+
+        def run(self):
+            try:
+                words = self.model.CORPORA[self.name]()
+                from operator import itemgetter
+
+                text = [w for w in words if len(w) > 2]
+                fd = FreqDist(tuple(text[i : i + 2]) for i in range(len(text) - 1))
+                vocab = FreqDist(text)
+                scored = [
+                    ((w1, w2), fd[(w1, w2)] ** 3 / (vocab[w1] * vocab[w2]))
+                    for w1, w2 in fd
+                ]
+                scored.sort(key=itemgetter(1), reverse=True)
+                self.model.collocations = list(map(itemgetter(0), scored))
+                self.model.queue.put(CORPUS_LOADED_EVENT)
+            except Exception as e:
+                print(e)
+                self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
+
+
+# def collocations():
+#    colloc_strings = [w1 + ' ' + w2 for w1, w2 in self._collocations[:num]]
+
+
+def app():
+    c = CollocationsView()
+    c.mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/concordance_app.py

@@ -0,0 +1,709 @@
+# Natural Language Toolkit: Concordance Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Sumukh Ghodke <sghodke@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import queue as q
+import re
+import threading
+from tkinter import (
+    END,
+    LEFT,
+    SUNKEN,
+    Button,
+    Entry,
+    Frame,
+    IntVar,
+    Label,
+    Menu,
+    OptionMenu,
+    Scrollbar,
+    StringVar,
+    Text,
+    Tk,
+)
+from tkinter.font import Font
+
+from nltk.corpus import (
+    alpino,
+    brown,
+    cess_cat,
+    cess_esp,
+    floresta,
+    indian,
+    mac_morpho,
+    nps_chat,
+    sinica_treebank,
+    treebank,
+)
+from nltk.draw.util import ShowText
+from nltk.util import in_idle
+
+WORD_OR_TAG = "[^/ ]+"
+BOUNDARY = r"\b"
+
+CORPUS_LOADED_EVENT = "<<CL_EVENT>>"
+SEARCH_TERMINATED_EVENT = "<<ST_EVENT>>"
+SEARCH_ERROR_EVENT = "<<SE_EVENT>>"
+ERROR_LOADING_CORPUS_EVENT = "<<ELC_EVENT>>"
+
+POLL_INTERVAL = 50
+
+# NB All corpora must be specified in a lambda expression so as not to be
+# loaded when the module is imported.
+
+_DEFAULT = "English: Brown Corpus (Humor, simplified)"
+_CORPORA = {
+    "Catalan: CESS-CAT Corpus (simplified)": lambda: cess_cat.tagged_sents(
+        tagset="universal"
+    ),
+    "English: Brown Corpus": lambda: brown.tagged_sents(),
+    "English: Brown Corpus (simplified)": lambda: brown.tagged_sents(
+        tagset="universal"
+    ),
+    "English: Brown Corpus (Press, simplified)": lambda: brown.tagged_sents(
+        categories=["news", "editorial", "reviews"], tagset="universal"
+    ),
+    "English: Brown Corpus (Religion, simplified)": lambda: brown.tagged_sents(
+        categories="religion", tagset="universal"
+    ),
+    "English: Brown Corpus (Learned, simplified)": lambda: brown.tagged_sents(
+        categories="learned", tagset="universal"
+    ),
+    "English: Brown Corpus (Science Fiction, simplified)": lambda: brown.tagged_sents(
+        categories="science_fiction", tagset="universal"
+    ),
+    "English: Brown Corpus (Romance, simplified)": lambda: brown.tagged_sents(
+        categories="romance", tagset="universal"
+    ),
+    "English: Brown Corpus (Humor, simplified)": lambda: brown.tagged_sents(
+        categories="humor", tagset="universal"
+    ),
+    "English: NPS Chat Corpus": lambda: nps_chat.tagged_posts(),
+    "English: NPS Chat Corpus (simplified)": lambda: nps_chat.tagged_posts(
+        tagset="universal"
+    ),
+    "English: Wall Street Journal Corpus": lambda: treebank.tagged_sents(),
+    "English: Wall Street Journal Corpus (simplified)": lambda: treebank.tagged_sents(
+        tagset="universal"
+    ),
+    "Chinese: Sinica Corpus": lambda: sinica_treebank.tagged_sents(),
+    "Chinese: Sinica Corpus (simplified)": lambda: sinica_treebank.tagged_sents(
+        tagset="universal"
+    ),
+    "Dutch: Alpino Corpus": lambda: alpino.tagged_sents(),
+    "Dutch: Alpino Corpus (simplified)": lambda: alpino.tagged_sents(
+        tagset="universal"
+    ),
+    "Hindi: Indian Languages Corpus": lambda: indian.tagged_sents(files="hindi.pos"),
+    "Hindi: Indian Languages Corpus (simplified)": lambda: indian.tagged_sents(
+        files="hindi.pos", tagset="universal"
+    ),
+    "Portuguese: Floresta Corpus (Portugal)": lambda: floresta.tagged_sents(),
+    "Portuguese: Floresta Corpus (Portugal, simplified)": lambda: floresta.tagged_sents(
+        tagset="universal"
+    ),
+    "Portuguese: MAC-MORPHO Corpus (Brazil)": lambda: mac_morpho.tagged_sents(),
+    "Portuguese: MAC-MORPHO Corpus (Brazil, simplified)": lambda: mac_morpho.tagged_sents(
+        tagset="universal"
+    ),
+    "Spanish: CESS-ESP Corpus (simplified)": lambda: cess_esp.tagged_sents(
+        tagset="universal"
+    ),
+}
+
+
+class ConcordanceSearchView:
+    _BACKGROUND_COLOUR = "#FFF"  # white
+
+    # Colour of highlighted results
+    _HIGHLIGHT_WORD_COLOUR = "#F00"  # red
+    _HIGHLIGHT_WORD_TAG = "HL_WRD_TAG"
+
+    _HIGHLIGHT_LABEL_COLOUR = "#C0C0C0"  # dark grey
+    _HIGHLIGHT_LABEL_TAG = "HL_LBL_TAG"
+
+    # Percentage of text left of the scrollbar position
+    _FRACTION_LEFT_TEXT = 0.30
+
+    def __init__(self):
+        self.queue = q.Queue()
+        self.model = ConcordanceSearchModel(self.queue)
+        self.top = Tk()
+        self._init_top(self.top)
+        self._init_menubar()
+        self._init_widgets(self.top)
+        self.load_corpus(self.model.DEFAULT_CORPUS)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+    def _init_top(self, top):
+        top.geometry("950x680+50+50")
+        top.title("NLTK Concordance Search")
+        top.bind("<Control-q>", self.destroy)
+        top.protocol("WM_DELETE_WINDOW", self.destroy)
+        top.minsize(950, 680)
+
+    def _init_widgets(self, parent):
+        self.main_frame = Frame(
+            parent, dict(background=self._BACKGROUND_COLOUR, padx=1, pady=1, border=1)
+        )
+        self._init_corpus_select(self.main_frame)
+        self._init_query_box(self.main_frame)
+        self._init_results_box(self.main_frame)
+        self._init_paging(self.main_frame)
+        self._init_status(self.main_frame)
+        self.main_frame.pack(fill="both", expand=True)
+
+    def _init_menubar(self):
+        self._result_size = IntVar(self.top)
+        self._cntx_bf_len = IntVar(self.top)
+        self._cntx_af_len = IntVar(self.top)
+        menubar = Menu(self.top)
+
+        filemenu = Menu(menubar, tearoff=0, borderwidth=0)
+        filemenu.add_command(
+            label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-q"
+        )
+        menubar.add_cascade(label="File", underline=0, menu=filemenu)
+
+        editmenu = Menu(menubar, tearoff=0)
+        rescntmenu = Menu(editmenu, tearoff=0)
+        rescntmenu.add_radiobutton(
+            label="20",
+            variable=self._result_size,
+            underline=0,
+            value=20,
+            command=self.set_result_size,
+        )
+        rescntmenu.add_radiobutton(
+            label="50",
+            variable=self._result_size,
+            underline=0,
+            value=50,
+            command=self.set_result_size,
+        )
+        rescntmenu.add_radiobutton(
+            label="100",
+            variable=self._result_size,
+            underline=0,
+            value=100,
+            command=self.set_result_size,
+        )
+        rescntmenu.invoke(1)
+        editmenu.add_cascade(label="Result Count", underline=0, menu=rescntmenu)
+
+        cntxmenu = Menu(editmenu, tearoff=0)
+        cntxbfmenu = Menu(cntxmenu, tearoff=0)
+        cntxbfmenu.add_radiobutton(
+            label="60 characters",
+            variable=self._cntx_bf_len,
+            underline=0,
+            value=60,
+            command=self.set_cntx_bf_len,
+        )
+        cntxbfmenu.add_radiobutton(
+            label="80 characters",
+            variable=self._cntx_bf_len,
+            underline=0,
+            value=80,
+            command=self.set_cntx_bf_len,
+        )
+        cntxbfmenu.add_radiobutton(
+            label="100 characters",
+            variable=self._cntx_bf_len,
+            underline=0,
+            value=100,
+            command=self.set_cntx_bf_len,
+        )
+        cntxbfmenu.invoke(1)
+        cntxmenu.add_cascade(label="Before", underline=0, menu=cntxbfmenu)
+
+        cntxafmenu = Menu(cntxmenu, tearoff=0)
+        cntxafmenu.add_radiobutton(
+            label="70 characters",
+            variable=self._cntx_af_len,
+            underline=0,
+            value=70,
+            command=self.set_cntx_af_len,
+        )
+        cntxafmenu.add_radiobutton(
+            label="90 characters",
+            variable=self._cntx_af_len,
+            underline=0,
+            value=90,
+            command=self.set_cntx_af_len,
+        )
+        cntxafmenu.add_radiobutton(
+            label="110 characters",
+            variable=self._cntx_af_len,
+            underline=0,
+            value=110,
+            command=self.set_cntx_af_len,
+        )
+        cntxafmenu.invoke(1)
+        cntxmenu.add_cascade(label="After", underline=0, menu=cntxafmenu)
+
+        editmenu.add_cascade(label="Context", underline=0, menu=cntxmenu)
+
+        menubar.add_cascade(label="Edit", underline=0, menu=editmenu)
+
+        self.top.config(menu=menubar)
+
+    def set_result_size(self, **kwargs):
+        self.model.result_count = self._result_size.get()
+
+    def set_cntx_af_len(self, **kwargs):
+        self._char_after = self._cntx_af_len.get()
+
+    def set_cntx_bf_len(self, **kwargs):
+        self._char_before = self._cntx_bf_len.get()
+
+    def _init_corpus_select(self, parent):
+        innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+        self.var = StringVar(innerframe)
+        self.var.set(self.model.DEFAULT_CORPUS)
+        Label(
+            innerframe,
+            justify=LEFT,
+            text=" Corpus: ",
+            background=self._BACKGROUND_COLOUR,
+            padx=2,
+            pady=1,
+            border=0,
+        ).pack(side="left")
+
+        other_corpora = list(self.model.CORPORA.keys()).remove(
+            self.model.DEFAULT_CORPUS
+        )
+        om = OptionMenu(
+            innerframe,
+            self.var,
+            self.model.DEFAULT_CORPUS,
+            command=self.corpus_selected,
+            *self.model.non_default_corpora()
+        )
+        om["borderwidth"] = 0
+        om["highlightthickness"] = 1
+        om.pack(side="left")
+        innerframe.pack(side="top", fill="x", anchor="n")
+
+    def _init_status(self, parent):
+        self.status = Label(
+            parent,
+            justify=LEFT,
+            relief=SUNKEN,
+            background=self._BACKGROUND_COLOUR,
+            border=0,
+            padx=1,
+            pady=0,
+        )
+        self.status.pack(side="top", anchor="sw")
+
+    def _init_query_box(self, parent):
+        innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+        another = Frame(innerframe, background=self._BACKGROUND_COLOUR)
+        self.query_box = Entry(another, width=60)
+        self.query_box.pack(side="left", fill="x", pady=25, anchor="center")
+        self.search_button = Button(
+            another,
+            text="Search",
+            command=self.search,
+            borderwidth=1,
+            highlightthickness=1,
+        )
+        self.search_button.pack(side="left", fill="x", pady=25, anchor="center")
+        self.query_box.bind("<KeyPress-Return>", self.search_enter_keypress_handler)
+        another.pack()
+        innerframe.pack(side="top", fill="x", anchor="n")
+
+    def search_enter_keypress_handler(self, *event):
+        self.search()
+
+    def _init_results_box(self, parent):
+        innerframe = Frame(parent)
+        i1 = Frame(innerframe)
+        i2 = Frame(innerframe)
+        vscrollbar = Scrollbar(i1, borderwidth=1)
+        hscrollbar = Scrollbar(i2, borderwidth=1, orient="horiz")
+        self.results_box = Text(
+            i1,
+            font=Font(family="courier", size="16"),
+            state="disabled",
+            borderwidth=1,
+            yscrollcommand=vscrollbar.set,
+            xscrollcommand=hscrollbar.set,
+            wrap="none",
+            width="40",
+            height="20",
+            exportselection=1,
+        )
+        self.results_box.pack(side="left", fill="both", expand=True)
+        self.results_box.tag_config(
+            self._HIGHLIGHT_WORD_TAG, foreground=self._HIGHLIGHT_WORD_COLOUR
+        )
+        self.results_box.tag_config(
+            self._HIGHLIGHT_LABEL_TAG, foreground=self._HIGHLIGHT_LABEL_COLOUR
+        )
+        vscrollbar.pack(side="left", fill="y", anchor="e")
+        vscrollbar.config(command=self.results_box.yview)
+        hscrollbar.pack(side="left", fill="x", expand=True, anchor="w")
+        hscrollbar.config(command=self.results_box.xview)
+        # there is no other way of avoiding the overlap of scrollbars while using pack layout manager!!!
+        Label(i2, text="   ", background=self._BACKGROUND_COLOUR).pack(
+            side="left", anchor="e"
+        )
+        i1.pack(side="top", fill="both", expand=True, anchor="n")
+        i2.pack(side="bottom", fill="x", anchor="s")
+        innerframe.pack(side="top", fill="both", expand=True)
+
+    def _init_paging(self, parent):
+        innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+        self.prev = prev = Button(
+            innerframe,
+            text="Previous",
+            command=self.previous,
+            width="10",
+            borderwidth=1,
+            highlightthickness=1,
+            state="disabled",
+        )
+        prev.pack(side="left", anchor="center")
+        self.next = next = Button(
+            innerframe,
+            text="Next",
+            command=self.__next__,
+            width="10",
+            borderwidth=1,
+            highlightthickness=1,
+            state="disabled",
+        )
+        next.pack(side="right", anchor="center")
+        innerframe.pack(side="top", fill="y")
+        self.current_page = 0
+
+    def previous(self):
+        self.clear_results_box()
+        self.freeze_editable()
+        self.model.prev(self.current_page - 1)
+
+    def __next__(self):
+        self.clear_results_box()
+        self.freeze_editable()
+        self.model.next(self.current_page + 1)
+
+    def about(self, *e):
+        ABOUT = "NLTK Concordance Search Demo\n"
+        TITLE = "About: NLTK Concordance Search Demo"
+        try:
+            from tkinter.messagebox import Message
+
+            Message(message=ABOUT, title=TITLE, parent=self.main_frame).show()
+        except:
+            ShowText(self.top, TITLE, ABOUT)
+
+    def _bind_event_handlers(self):
+        self.top.bind(CORPUS_LOADED_EVENT, self.handle_corpus_loaded)
+        self.top.bind(SEARCH_TERMINATED_EVENT, self.handle_search_terminated)
+        self.top.bind(SEARCH_ERROR_EVENT, self.handle_search_error)
+        self.top.bind(ERROR_LOADING_CORPUS_EVENT, self.handle_error_loading_corpus)
+
+    def _poll(self):
+        try:
+            event = self.queue.get(block=False)
+        except q.Empty:
+            pass
+        else:
+            if event == CORPUS_LOADED_EVENT:
+                self.handle_corpus_loaded(event)
+            elif event == SEARCH_TERMINATED_EVENT:
+                self.handle_search_terminated(event)
+            elif event == SEARCH_ERROR_EVENT:
+                self.handle_search_error(event)
+            elif event == ERROR_LOADING_CORPUS_EVENT:
+                self.handle_error_loading_corpus(event)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+    def handle_error_loading_corpus(self, event):
+        self.status["text"] = "Error in loading " + self.var.get()
+        self.unfreeze_editable()
+        self.clear_all()
+        self.freeze_editable()
+
+    def handle_corpus_loaded(self, event):
+        self.status["text"] = self.var.get() + " is loaded"
+        self.unfreeze_editable()
+        self.clear_all()
+        self.query_box.focus_set()
+
+    def handle_search_terminated(self, event):
+        # todo: refactor the model such that it is less state sensitive
+        results = self.model.get_results()
+        self.write_results(results)
+        self.status["text"] = ""
+        if len(results) == 0:
+            self.status["text"] = "No results found for " + self.model.query
+        else:
+            self.current_page = self.model.last_requested_page
+        self.unfreeze_editable()
+        self.results_box.xview_moveto(self._FRACTION_LEFT_TEXT)
+
+    def handle_search_error(self, event):
+        self.status["text"] = "Error in query " + self.model.query
+        self.unfreeze_editable()
+
+    def corpus_selected(self, *args):
+        new_selection = self.var.get()
+        self.load_corpus(new_selection)
+
+    def load_corpus(self, selection):
+        if self.model.selected_corpus != selection:
+            self.status["text"] = "Loading " + selection + "..."
+            self.freeze_editable()
+            self.model.load_corpus(selection)
+
+    def search(self):
+        self.current_page = 0
+        self.clear_results_box()
+        self.model.reset_results()
+        query = self.query_box.get()
+        if len(query.strip()) == 0:
+            return
+        self.status["text"] = "Searching for " + query
+        self.freeze_editable()
+        self.model.search(query, self.current_page + 1)
+
+    def write_results(self, results):
+        self.results_box["state"] = "normal"
+        row = 1
+        for each in results:
+            sent, pos1, pos2 = each[0].strip(), each[1], each[2]
+            if len(sent) != 0:
+                if pos1 < self._char_before:
+                    sent, pos1, pos2 = self.pad(sent, pos1, pos2)
+                sentence = sent[pos1 - self._char_before : pos1 + self._char_after]
+                if not row == len(results):
+                    sentence += "\n"
+                self.results_box.insert(str(row) + ".0", sentence)
+                word_markers, label_markers = self.words_and_labels(sent, pos1, pos2)
+                for marker in word_markers:
+                    self.results_box.tag_add(
+                        self._HIGHLIGHT_WORD_TAG,
+                        str(row) + "." + str(marker[0]),
+                        str(row) + "." + str(marker[1]),
+                    )
+                for marker in label_markers:
+                    self.results_box.tag_add(
+                        self._HIGHLIGHT_LABEL_TAG,
+                        str(row) + "." + str(marker[0]),
+                        str(row) + "." + str(marker[1]),
+                    )
+                row += 1
+        self.results_box["state"] = "disabled"
+
+    def words_and_labels(self, sentence, pos1, pos2):
+        search_exp = sentence[pos1:pos2]
+        words, labels = [], []
+        labeled_words = search_exp.split(" ")
+        index = 0
+        for each in labeled_words:
+            if each == "":
+                index += 1
+            else:
+                word, label = each.split("/")
+                words.append(
+                    (self._char_before + index, self._char_before + index + len(word))
+                )
+                index += len(word) + 1
+                labels.append(
+                    (self._char_before + index, self._char_before + index + len(label))
+                )
+                index += len(label)
+            index += 1
+        return words, labels
+
+    def pad(self, sent, hstart, hend):
+        if hstart >= self._char_before:
+            return sent, hstart, hend
+        d = self._char_before - hstart
+        sent = "".join([" "] * d) + sent
+        return sent, hstart + d, hend + d
+
+    def destroy(self, *e):
+        if self.top is None:
+            return
+        self.top.after_cancel(self.after)
+        self.top.destroy()
+        self.top = None
+
+    def clear_all(self):
+        self.query_box.delete(0, END)
+        self.model.reset_query()
+        self.clear_results_box()
+
+    def clear_results_box(self):
+        self.results_box["state"] = "normal"
+        self.results_box.delete("1.0", END)
+        self.results_box["state"] = "disabled"
+
+    def freeze_editable(self):
+        self.query_box["state"] = "disabled"
+        self.search_button["state"] = "disabled"
+        self.prev["state"] = "disabled"
+        self.next["state"] = "disabled"
+
+    def unfreeze_editable(self):
+        self.query_box["state"] = "normal"
+        self.search_button["state"] = "normal"
+        self.set_paging_button_states()
+
+    def set_paging_button_states(self):
+        if self.current_page == 0 or self.current_page == 1:
+            self.prev["state"] = "disabled"
+        else:
+            self.prev["state"] = "normal"
+        if self.model.has_more_pages(self.current_page):
+            self.next["state"] = "normal"
+        else:
+            self.next["state"] = "disabled"
+
+    def fire_event(self, event):
+        # Firing an event so that rendering of widgets happen in the mainloop thread
+        self.top.event_generate(event, when="tail")
+
+    def mainloop(self, *args, **kwargs):
+        if in_idle():
+            return
+        self.top.mainloop(*args, **kwargs)
+
+
+class ConcordanceSearchModel:
+    def __init__(self, queue):
+        self.queue = queue
+        self.CORPORA = _CORPORA
+        self.DEFAULT_CORPUS = _DEFAULT
+        self.selected_corpus = None
+        self.reset_query()
+        self.reset_results()
+        self.result_count = None
+        self.last_sent_searched = 0
+
+    def non_default_corpora(self):
+        copy = []
+        copy.extend(list(self.CORPORA.keys()))
+        copy.remove(self.DEFAULT_CORPUS)
+        copy.sort()
+        return copy
+
+    def load_corpus(self, name):
+        self.selected_corpus = name
+        self.tagged_sents = []
+        runner_thread = self.LoadCorpus(name, self)
+        runner_thread.start()
+
+    def search(self, query, page):
+        self.query = query
+        self.last_requested_page = page
+        self.SearchCorpus(self, page, self.result_count).start()
+
+    def next(self, page):
+        self.last_requested_page = page
+        if len(self.results) < page:
+            self.search(self.query, page)
+        else:
+            self.queue.put(SEARCH_TERMINATED_EVENT)
+
+    def prev(self, page):
+        self.last_requested_page = page
+        self.queue.put(SEARCH_TERMINATED_EVENT)
+
+    def reset_results(self):
+        self.last_sent_searched = 0
+        self.results = []
+        self.last_page = None
+
+    def reset_query(self):
+        self.query = None
+
+    def set_results(self, page, resultset):
+        self.results.insert(page - 1, resultset)
+
+    def get_results(self):
+        return self.results[self.last_requested_page - 1]
+
+    def has_more_pages(self, page):
+        if self.results == [] or self.results[0] == []:
+            return False
+        if self.last_page is None:
+            return True
+        return page < self.last_page
+
+    class LoadCorpus(threading.Thread):
+        def __init__(self, name, model):
+            threading.Thread.__init__(self)
+            self.model, self.name = model, name
+
+        def run(self):
+            try:
+                ts = self.model.CORPORA[self.name]()
+                self.model.tagged_sents = [
+                    " ".join(w + "/" + t for (w, t) in sent) for sent in ts
+                ]
+                self.model.queue.put(CORPUS_LOADED_EVENT)
+            except Exception as e:
+                print(e)
+                self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
+
+    class SearchCorpus(threading.Thread):
+        def __init__(self, model, page, count):
+            self.model, self.count, self.page = model, count, page
+            threading.Thread.__init__(self)
+
+        def run(self):
+            q = self.processed_query()
+            sent_pos, i, sent_count = [], 0, 0
+            for sent in self.model.tagged_sents[self.model.last_sent_searched :]:
+                try:
+                    m = re.search(q, sent)
+                except re.error:
+                    self.model.reset_results()
+                    self.model.queue.put(SEARCH_ERROR_EVENT)
+                    return
+                if m:
+                    sent_pos.append((sent, m.start(), m.end()))
+                    i += 1
+                    if i > self.count:
+                        self.model.last_sent_searched += sent_count - 1
+                        break
+                sent_count += 1
+            if self.count >= len(sent_pos):
+                self.model.last_sent_searched += sent_count - 1
+                self.model.last_page = self.page
+                self.model.set_results(self.page, sent_pos)
+            else:
+                self.model.set_results(self.page, sent_pos[:-1])
+            self.model.queue.put(SEARCH_TERMINATED_EVENT)
+
+        def processed_query(self):
+            new = []
+            for term in self.model.query.split():
+                term = re.sub(r"\.", r"[^/ ]", term)
+                if re.match("[A-Z]+$", term):
+                    new.append(BOUNDARY + WORD_OR_TAG + "/" + term + BOUNDARY)
+                elif "/" in term:
+                    new.append(BOUNDARY + term + BOUNDARY)
+                else:
+                    new.append(BOUNDARY + term + "/" + WORD_OR_TAG + BOUNDARY)
+            return " ".join(new)
+
+
+def app():
+    d = ConcordanceSearchView()
+    d.mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/nemo_app.py

@@ -0,0 +1,163 @@
+# Finding (and Replacing) Nemo, Version 1.1, Aristide Grange 2006/06/06
+# https://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496783
+
+"""
+Finding (and Replacing) Nemo
+
+Instant Regular Expressions
+Created by Aristide Grange
+"""
+import itertools
+import re
+from tkinter import SEL_FIRST, SEL_LAST, Frame, Label, PhotoImage, Scrollbar, Text, Tk
+
+windowTitle = "Finding (and Replacing) Nemo"
+initialFind = r"n(.*?)e(.*?)m(.*?)o"
+initialRepl = r"M\1A\2K\3I"
+initialText = """\
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
+Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+"""
+images = {
+    "FIND": "R0lGODlhMAAiAPcAMf/////37//35//n1v97Off///f/9/f37/fexvfOvfeEQvd7QvdrQvdrKfdaKfdSMfdSIe/v9+/v7+/v5+/n3u/e1u/Wxu/Gre+1lO+tnO+thO+Ua+97Y+97Oe97Me9rOe9rMe9jOe9jMe9jIe9aMefe5+fe3ufezuece+eEWudzQudaIedSIedKMedKIedCKedCId7e1t7Wzt7Oxt7Gvd69vd69rd61pd6ljN6UjN6Ue96EY95zY95rUt5rQt5jMd5SId5KIdbn59be3tbGztbGvda1rdaEa9Z7a9Z7WtZzQtZzOdZzMdZjMdZaQtZSOdZSMdZKMdZCKdZCGNY5Ic7W1s7Oxs7Gtc69xs69tc69rc6tpc6llM6clM6cjM6Ue86EY85zWs5rSs5SKc5KKc5KGMa1tcatrcalvcalnMaUpcZ7c8ZzMcZrUsZrOcZrMcZaQsZSOcZSMcZKMcZCKcZCGMYxIcYxGL3Gxr21tb21rb2lpb2crb2cjL2UnL2UlL2UhL2Ec717Wr17Ur1zWr1rMb1jUr1KMb1KIb1CIb0xGLWlrbWlpbWcnLWEe7V7c7VzY7VzUrVSKbVKMbVCMbVCIbU5KbUxIbUxEK2lta2lpa2clK2UjK2MnK2MlK2Ea617e61za61rY61rMa1jSq1aUq1aSq1SQq1KKa0xEKWlnKWcnKWUnKWUhKWMjKWEa6Vza6VrWqVjMaVaUqVaKaVSMaVCMaU5KaUxIaUxGJyclJyMe5yElJyEhJx7e5x7c5xrOZxaQpxSOZxKQpw5IZSMhJSEjJR7c5Rre5RrY5RrUpRSQpRSKZRCOZRCKZQxKZQxIYyEhIx7hIxza4xzY4xrc4xjUoxaa4xaUoxSSoxKQoxCMYw5GIR7c4Rzc4Rre4RjY4RjWoRaa4RSWoRSUoRSMYRKQoRCOYQ5KYQxIXtra3taY3taSntKOXtCMXtCKXNCMXM5MXMxIWtSUmtKSmtKQmtCOWs5MWs5KWs5IWNCKWMxIVIxKUIQCDkhGAAAACH+AS4ALAAAAAAwACIAAAj/AAEIHEiwoMGDCBMqXMiwoUOHMqxIeEiRoZVp7cpZ29WrF4WKIAd208dGAQEVbiTVChUjZMU9+pYQmPmBZpxgvVw+nDdKwQICNVcIXQEkTgKdDdUJ+/nggVAXK1xI3TEA6UIr2uJ8iBqka1cXXTlkqGoVYRZ7iLyqBSs0iiEtZQVKiDGxBI1u3NR6lUpGDKg8MSgEQCphU7Z22vhg0dILXRCpYLuSCcYJT4wqXASBQaBzU7klHxC127OHD7ZDJFpERqRt0x5OnwQpmZmCLEhrbgg4WIHO1RY+nbQ9WRGEDJlmnXwJ+9FBgXMCIzYMVijBBgYMFxIMqJBMSc0Ht7qh/+Gjpte2rnYsYeNlasWIBgQ6yCewIoPCCp/cyP/wgUGbXVu0QcADZNBDnh98gHMLGXYQUw02w61QU3wdbNWDbQVVIIhMMwFF1DaZiPLBAy7E04kafrjSizaK3LFNNc0AAYRQDsAHHQlJ2IDQJ2zE1+EKDjiAijShkECCC8Qgw4cr7ZgyzC2WaHPNLWWoNeNWPiRAw0QFWQFMhz8C+QQ20yAiVSrY+MGOJCsccsst2GCzoHFxxEGGC+8hgs0MB2kyCpgzrUDCbs1Es41UdtATHFFkWELMOtsoQsYcgvRRQw5RSDgGOjZMR1AvPQIq6KCo9AKOJWDd48owQlHR4DXEKP9iyRrK+DNNBTu4RwIPFeTAGUG7hAomkA84gEg1m6ADljy9PBKGGJY4ig0xlsTBRSn98FOFDUC8pwQOPkgHbCGAzhTkA850s0c7j6Hjix9+gBIrMXLeAccWXUCyiRBcBEECdEJ98KtAqtBCYQc/OvDENnl4gYpUxISCIjjzylkGGV9okYUVNogRhAOBuuAEhjG08wOgDYzAgA5bCjIoCe5uwUk80RKTTSppPREGGGCIISOQ9AXBg6cC6WIywvCpoMHAocRBwhP4bHLFLujYkV42xNxBRhAyGrc113EgYtRBerDDDHMoDCyQEL5sE083EkgwQyBhxGFHMM206DUixGxmE0wssbQjCQ4JCaFKFwgQTVAVVhQUwAVPIFJKrHfYYRwi6OCDzzuIJIFhXAD0EccPsYRiSyqKSDpFcWSMIcZRoBMkQyA2BGZDIKSYcggih8TRRg4VxM5QABVYYLxgwiev/PLMCxQQADs=",
+    "find": "R0lGODlhMAAiAPQAMf////f39+/v7+fn597e3tbW1s7OzsbGxr29vbW1ta2traWlpZycnJSUlIyMjISEhHt7e3Nzc2tra2NjY1paWlJSUkpKSkJCQjk5OSkpKRgYGAAAAAAAAAAAAAAAAAAAACH+AS4ALAAAAAAwACIAAAX/ICCOZGmeaKquY2AGLiuvMCAUBuHWc48Kh0iFInEYCb4kSQCxPBiMxkMigRQEgJiSFVBYHNGG0RiZOHjblWAiiY4fkDhEYoBp06dAWfyAQyKAgAwDaHgnB0RwgYASgQ0IhDuGJDAIFhMRVFSLEX8QCJJ4AQM5AgQHTZqqjBAOCQQEkWkCDRMUFQsICQ4Vm5maEwwHOAsPDTpKMAsUDlO4CssTcb+2DAp8YGCyNFoCEsZwFQ3QDRTTVBRS0g1QbgsCd5QAAwgIBwYFAwStzQ8UEdCKVchky0yVBw7YuXkAKt4IAg74vXHVagqFBRgXSCAyYWAVCH0SNhDTitCJfSL5/4RbAPKPhQYYjVCYYAvCP0BxEDaD8CheAAHNwqh8MMGPSwgLeJWhwHSjqkYI+xg4MMCEgQjtRvZ7UAYCpghMF7CxONOWJkYR+rCpY4JlVpVxKDwYWEactKW9mhYRtqCTgwgWEMArERSK1j5q//6T8KXonFsShpiJkAECgQYVjykooCVA0JGHEWNiYCHThTFeb3UkoiCCBgwGEKQ1kuAJlhFwhA71h5SukwUM5qqeCSGBgicEWkfNiWSERtBad4JNIBaQBaQah1ToyGZBAnsIuIJs1qnqiAIVjIE2gnAB1T5x0icgzXT79ipgMOOEH6HBbREBMJCeGEY08IoLAkzB1YYFwjxwSUGSNULQJnNUwRYlCcyEkALIxECAP9cNMMABYpRhy3ZsSLDaR70oUAiABGCkAxowCGCAAfDYIQACXoElGRsdXWDBdg2Y90IWktDYGYAB9PWHP0PMdFZaF07SQgAFNDAMAQg0QA1UC8xoZQl22JGFPgWkOUCOL1pZQyhjxinnnCWEAAA7",
+    "REPL": "R0lGODlhMAAjAPcAMf/////3//+lOf+UKf+MEPf///f39/f35/fv7/ecQvecOfecKfeUIfeUGPeUEPeUCPeMAO/37+/v9+/v3u/n3u/n1u+9jO+9c++1hO+ta++tY++tWu+tUu+tSu+lUu+lQu+lMe+UMe+UKe+UGO+UEO+UAO+MCOfv5+fvxufn7+fn5+fnzue9lOe9c+e1jOe1e+e1c+e1a+etWuetUuelQuecOeeUUueUCN7e597e3t7e1t7ezt7evd7Wzt7Oxt7Ovd7Otd7Opd7OnN7Gtd7Gpd69lN61hN6ta96lStbextberdbW3tbWztbWxtbOvdbOrda1hNalUtaECM7W1s7Ozs7Oxs7Otc7Gxs7Gvc69tc69rc69pc61jM6lc8bWlMbOvcbGxsbGpca9tca9pca1nMaMAL3OhL3Gtb21vb21tb2tpb2tnL2tlLW9tbW9pbW9e7W1pbWtjLWcKa21nK2tra2tnK2tlK2lpa2llK2ljK2le6WlnKWljKWUe6WUc6WUY5y1QpyclJycjJychJyUc5yMY5StY5SUe5SMhJSMe5SMc5SMWpSEa5SESoyUe4yMhIyEY4SlKYScWoSMe4SEe4SEa4R7c4R7Y3uMY3uEe3t7e3t7c3tza3tzY3trKXtjIXOcAHOUMXOEY3Nzc3NzWnNrSmulCGuUMWuMGGtzWmtrY2taMWtaGGOUOWOMAGNzUmNjWmNjSmNaUmNaQmNaOWNaIWNSCFqcAFpjUlpSMVpSIVpSEFpKKVKMAFJSUlJSSlJSMVJKMVJKGFJKAFI5CEqUAEqEAEpzQkpKIUpCQkpCGEpCAEo5EEoxAEJjOUJCOUJCAEI5IUIxADl7ADlaITlCOTkxMTkxKTkxEDkhADFzADFrGDE5OTExADEpEClrCCkxKSkpKSkpISkpACkhCCkhACkYACFzACFrACEhCCEYGBhjEBhjABghABgYCBgYABgQEBgQABAQABAIAAhjAAhSAAhKAAgIEAgICABaAABCAAAhAAAQAAAIAAAAAAAAACH+AS4ALAAAAAAwACMAAAj/AAEIHEiwoMGDCBMqXMiwocOHAA4cgEixIIIJO3JMmAjADIqKFU/8MHIkg5EgYXx4iaTkI0iHE6wE2TCggYILQayEAgXIy8uGCKz8sDCAQAMRG3iEcXULlJkJPwli3OFjh9UdYYLE6NBhA04UXHoVA2XoTZgfPKBWlOBDphAWOdfMcfMDLloeO3hIMjbWVCQ5Fn6E2UFxgpsgFjYIEBADrZU6luqEEfqjTqpt54z1uuWqTIcgWAk7PECGzIUQDRosDmxlUrVJkwQJkqVuX71v06YZcyUlROAdbnLAJKPFyAYFAhoMwFlnEh0rWkpz8raPHm7dqKKc/KFFkBUrVn1M/ziBcEIeLUEQI8/AYk0i9Be4sqjsrN66c9/OnbobhpR3HkIUoZ0WVnBE0AGLFKKFD0HAFUQe77HQgQI1hRBDEHMcY0899bBzihZuCPILJD8EccEGGzwAQhFaUHHQH82sUkgeNHISDBk8WCCCcsqFUEQWmOyzjz3sUGNNOO5Y48YOEgowAAQhnBScQV00k82V47jzjy9CXZBcjziFoco//4CDiSOyhPMPLkJZkEBqJmRQxA9uZGEQD8Ncmc044/zzDF2IZQBCCDYE8QMZz/iiCSx0neHGI7BIhhhNn+1gxRpokEcQAp7seWU7/PwTyxqG/iCEEVzQmUombnDRxRExzP9nBR2PCKLFD3UJwcMPa/SRqUGNWJmNOVn+M44ukMRB4KGcWDNLVhuUMEIJAlzwA3DJBHMJIXm4sQYhqyxCRQQGLSIsn1qac2UzysQSyzX/hLMGD0F0IMCODYAQBA9W/PKPOcRiw0wzwxTiokF9dLMnuv/Mo+fCZF7jBr0xbDDCACWEYKgb1vzjDp/jZNOMLX0IZxAKq2TZTjtaOjwOsXyG+s8sZJTIQsUdIGHoJPf8w487QI/TDSt5mGwQFZxc406o8HiDJchk/ltLHpSlJwSvz5DpTjvmuGNOM57koelBOaAhiCaaPBLL0wwbm003peRBnBZqJMJL1ECz/HXYYx/NdAIOOVCxQyLorswymU93o0wuwfAiTDNR/xz0MLXU0XdCE+UwSTRZAq2lsSATu+4wkGvt+TjNzPLrQyegAUku2Hij5cd8LhxyM8QIg4w18HgcdC6BTBFSDmfQqsovttveDcG7lFLHI75cE841sARCxeWsnxC4G9HADPK6ywzDCRqBo0EHHWhMgT1IJzziNci1N7PMKnSYfML96/90AiJKey/0KtbLX1QK0rrNnQ541xugQ7SHhkXBghN0SKACWRc4KlAhBwKcIOYymJCAAAA7",
+    "repl": "R0lGODlhMAAjAPQAMf////f39+/v7+fn597e3tbW1s7OzsbGxr29vbW1ta2traWlpZycnJSUlIyMjISEhHt7e3Nzc2tra2NjY1paWlJSUkpKSkJCQjk5OTExMSkpKSEhIRgYGBAQEAgICAAAACH+AS4ALAAAAAAwACMAAAX/ICCOZGmeaKqubOu+gCDANBkIQ1EMQhAghFptYEAkEgjEwXBo7ISvweGgWCwUysPjwTgEoCafTySYIhYMxgLBjEQgCULvCw0QdAZdoVhUIJUFChISEAxYeQM1N1OMTAp+UwZ5eA4TEhFbDWYFdC4ECVMJjwl5BwsQa0umEhUVlhESDgqlBp0rAn5nVpBMDxeZDRQbHBgWFBSWDgtLBnFjKwRYCI9VqQsPs0YKEcMXFq0UEalFDWx4BAO2IwPjppAKDkrTWKYUGd7fEJJFEZpM00cOzCgh4EE8SaoWxKNixQooBRMyZMBwAYIRBhUgLDGS4MoBJeoANMhAgQsaCRZm/5lqaCUJhA4cNHjDoKEDBlJUHqkBlYBTiQUZNGjYMMxDhY3VWk6R4MEDBoMUak5AqoYBqANIBo4wcGGDUKIeLlzVZmWJggsVIkwAZaQSA3kdZzlKkIiEAAlDvW5oOkEBs488JTw44oeUIwdvVTFTUK7uiAAPgubt8GFDhQepqETAQCFU1UMGzlqAgFhUsAcCS0AO6lUDhw8xNRSbENGDhgWSHjWUe6ACbKITizmopZoBa6KvOwj9uuHDhwxyj3xekgDDhw5EvWKo0IB4iQLCOCC/njc7ZQ8UeGvza+ABZZgcxJNc4FO1gc0cOsCUrHevc8tdIMTIAhc4F198G2Qwwd8CBIQUAwEINABBBJUwR9R5wElgVRLwWODBBx4cGB8GEzDQIAo33CGJA8gh+JoH/clUgQU0YvDhdfmJdwEFC6Sjgg8yEPAABsPkh2F22cl2AQbn6QdTghTQ5eAJAQyQAAQV0MSBB9gRVZ4GE1mw5JZOAmiAVi1UWcAZDrDyZXYTeaOhA/bIVuIBPtKQ4h7ViYekUPdcEAEbzTzCRp5CADmAAwj+ORGPBcgwAAHo9ABGCYtm0ChwFHShlRiXhmHlkAcCiOeUodqQw5W0oXLAiamy4MOkjOyAaqxUymApDCEAADs=",
+}
+colors = ["#FF7B39", "#80F121"]
+emphColors = ["#DAFC33", "#F42548"]
+fieldParams = {
+    "height": 3,
+    "width": 70,
+    "font": ("monaco", 14),
+    "highlightthickness": 0,
+    "borderwidth": 0,
+    "background": "white",
+}
+textParams = {
+    "bg": "#F7E0D4",
+    "fg": "#2321F1",
+    "highlightthickness": 0,
+    "width": 1,
+    "height": 10,
+    "font": ("verdana", 16),
+    "wrap": "word",
+}
+
+
+class Zone:
+    def __init__(self, image, initialField, initialText):
+        frm = Frame(root)
+        frm.config(background="white")
+        self.image = PhotoImage(format="gif", data=images[image.upper()])
+        self.imageDimmed = PhotoImage(format="gif", data=images[image])
+        self.img = Label(frm)
+        self.img.config(borderwidth=0)
+        self.img.pack(side="left")
+        self.fld = Text(frm, **fieldParams)
+        self.initScrollText(frm, self.fld, initialField)
+        frm = Frame(root)
+        self.txt = Text(frm, **textParams)
+        self.initScrollText(frm, self.txt, initialText)
+        for i in range(2):
+            self.txt.tag_config(colors[i], background=colors[i])
+            self.txt.tag_config("emph" + colors[i], foreground=emphColors[i])
+
+    def initScrollText(self, frm, txt, contents):
+        scl = Scrollbar(frm)
+        scl.config(command=txt.yview)
+        scl.pack(side="right", fill="y")
+        txt.pack(side="left", expand=True, fill="x")
+        txt.config(yscrollcommand=scl.set)
+        txt.insert("1.0", contents)
+        frm.pack(fill="x")
+        Frame(height=2, bd=1, relief="ridge").pack(fill="x")
+
+    def refresh(self):
+        self.colorCycle = itertools.cycle(colors)
+        try:
+            self.substitute()
+            self.img.config(image=self.image)
+        except re.error:
+            self.img.config(image=self.imageDimmed)
+
+
+class FindZone(Zone):
+    def addTags(self, m):
+        color = next(self.colorCycle)
+        self.txt.tag_add(color, "1.0+%sc" % m.start(), "1.0+%sc" % m.end())
+        try:
+            self.txt.tag_add(
+                "emph" + color, "1.0+%sc" % m.start("emph"), "1.0+%sc" % m.end("emph")
+            )
+        except:
+            pass
+
+    def substitute(self, *args):
+        for color in colors:
+            self.txt.tag_remove(color, "1.0", "end")
+            self.txt.tag_remove("emph" + color, "1.0", "end")
+        self.rex = re.compile("")  # default value in case of malformed regexp
+        self.rex = re.compile(self.fld.get("1.0", "end")[:-1], re.MULTILINE)
+        try:
+            re.compile("(?P<emph>%s)" % self.fld.get(SEL_FIRST, SEL_LAST))
+            self.rexSel = re.compile(
+                "%s(?P<emph>%s)%s"
+                % (
+                    self.fld.get("1.0", SEL_FIRST),
+                    self.fld.get(SEL_FIRST, SEL_LAST),
+                    self.fld.get(SEL_LAST, "end")[:-1],
+                ),
+                re.MULTILINE,
+            )
+        except:
+            self.rexSel = self.rex
+        self.rexSel.sub(self.addTags, self.txt.get("1.0", "end"))
+
+
+class ReplaceZone(Zone):
+    def addTags(self, m):
+        s = sz.rex.sub(self.repl, m.group())
+        self.txt.delete(
+            "1.0+%sc" % (m.start() + self.diff), "1.0+%sc" % (m.end() + self.diff)
+        )
+        self.txt.insert("1.0+%sc" % (m.start() + self.diff), s, next(self.colorCycle))
+        self.diff += len(s) - (m.end() - m.start())
+
+    def substitute(self):
+        self.txt.delete("1.0", "end")
+        self.txt.insert("1.0", sz.txt.get("1.0", "end")[:-1])
+        self.diff = 0
+        self.repl = rex0.sub(r"\\g<\1>", self.fld.get("1.0", "end")[:-1])
+        sz.rex.sub(self.addTags, sz.txt.get("1.0", "end")[:-1])
+
+
+def launchRefresh(_):
+    sz.fld.after_idle(sz.refresh)
+    rz.fld.after_idle(rz.refresh)
+
+
+def app():
+    global root, sz, rz, rex0
+    root = Tk()
+    root.resizable(height=False, width=True)
+    root.title(windowTitle)
+    root.minsize(width=250, height=0)
+    sz = FindZone("find", initialFind, initialText)
+    sz.fld.bind("<Button-1>", launchRefresh)
+    sz.fld.bind("<ButtonRelease-1>", launchRefresh)
+    sz.fld.bind("<B1-Motion>", launchRefresh)
+    sz.rexSel = re.compile("")
+    rz = ReplaceZone("repl", initialRepl, "")
+    rex0 = re.compile(r"(?<!\\)\\([0-9]+)")
+    root.bind_all("<Key>", launchRefresh)
+    launchRefresh(None)
+    root.mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/rdparser_app.py

@@ -0,0 +1,1052 @@
+# Natural Language Toolkit: Recursive Descent Parser Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the recursive descent parser.
+
+The recursive descent parser maintains a tree, which records the
+structure of the portion of the text that has been parsed.  It uses
+CFG productions to expand the fringe of the tree, and matches its
+leaves against the text.  Initially, the tree contains the start
+symbol ("S").  It is shown in the main canvas, to the right of the
+list of available expansions.
+
+The parser builds up a tree structure for the text using three
+operations:
+
+  - "expand" uses a CFG production to add children to a node on the
+    fringe of the tree.
+  - "match" compares a leaf in the tree to a text token.
+  - "backtrack" returns the tree to its state before the most recent
+    expand or match operation.
+
+The parser maintains a list of tree locations called a "frontier" to
+remember which nodes have not yet been expanded and which leaves have
+not yet been matched against the text.  The leftmost frontier node is
+shown in green, and the other frontier nodes are shown in blue.  The
+parser always performs expand and match operations on the leftmost
+element of the frontier.
+
+You can control the parser's operation by using the "expand," "match,"
+and "backtrack" buttons; or you can use the "step" button to let the
+parser automatically decide which operation to apply.  The parser uses
+the following rules to decide which operation to apply:
+
+  - If the leftmost frontier element is a token, try matching it.
+  - If the leftmost frontier element is a node, try expanding it with
+    the first untried expansion.
+  - Otherwise, backtrack.
+
+The "expand" button applies the untried expansion whose CFG production
+is listed earliest in the grammar.  To manually choose which expansion
+to apply, click on a CFG production from the list of available
+expansions, on the left side of the main window.
+
+The "autostep" button will let the parser continue applying
+applications to the tree until it reaches a complete parse.  You can
+cancel an autostep in progress at any time by clicking on the
+"autostep" button again.
+
+Keyboard Shortcuts::
+      [Space]\t Perform the next expand, match, or backtrack operation
+      [a]\t Step through operations until the next complete parse
+      [e]\t Perform an expand operation
+      [m]\t Perform a match operation
+      [b]\t Perform a backtrack operation
+      [Delete]\t Reset the parser
+      [g]\t Show/hide available expansions list
+      [h]\t Help
+      [Ctrl-p]\t Print
+      [q]\t Quit
+"""
+
+from tkinter import Button, Frame, IntVar, Label, Listbox, Menu, Scrollbar, Tk
+from tkinter.font import Font
+
+from nltk.draw import CFGEditor, TreeSegmentWidget, tree_to_treesegment
+from nltk.draw.util import CanvasFrame, EntryDialog, ShowText, TextWidget
+from nltk.parse import SteppingRecursiveDescentParser
+from nltk.tree import Tree
+from nltk.util import in_idle
+
+
+class RecursiveDescentApp:
+    """
+    A graphical tool for exploring the recursive descent parser.  The tool
+    displays the parser's tree and the remaining text, and allows the
+    user to control the parser's operation.  In particular, the user
+    can expand subtrees on the frontier, match tokens on the frontier
+    against the text, and backtrack.  A "step" button simply steps
+    through the parsing process, performing the operations that
+    ``RecursiveDescentParser`` would use.
+    """
+
+    def __init__(self, grammar, sent, trace=0):
+        self._sent = sent
+        self._parser = SteppingRecursiveDescentParser(grammar, trace)
+
+        # Set up the main window.
+        self._top = Tk()
+        self._top.title("Recursive Descent Parser Application")
+
+        # Set up key bindings.
+        self._init_bindings()
+
+        # Initialize the fonts.
+        self._init_fonts(self._top)
+
+        # Animations.  animating_lock is a lock to prevent the demo
+        # from performing new operations while it's animating.
+        self._animation_frames = IntVar(self._top)
+        self._animation_frames.set(5)
+        self._animating_lock = 0
+        self._autostep = 0
+
+        # The user can hide the grammar.
+        self._show_grammar = IntVar(self._top)
+        self._show_grammar.set(1)
+
+        # Create the basic frames.
+        self._init_menubar(self._top)
+        self._init_buttons(self._top)
+        self._init_feedback(self._top)
+        self._init_grammar(self._top)
+        self._init_canvas(self._top)
+
+        # Initialize the parser.
+        self._parser.initialize(self._sent)
+
+        # Resize callback
+        self._canvas.bind("<Configure>", self._configure)
+
+    #########################################
+    ##  Initialization Helpers
+    #########################################
+
+    def _init_fonts(self, root):
+        # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+        self._sysfont = Font(font=Button()["font"])
+        root.option_add("*Font", self._sysfont)
+
+        # TWhat's our font size (default=same as sysfont)
+        self._size = IntVar(root)
+        self._size.set(self._sysfont.cget("size"))
+
+        self._boldfont = Font(family="helvetica", weight="bold", size=self._size.get())
+        self._font = Font(family="helvetica", size=self._size.get())
+        if self._size.get() < 0:
+            big = self._size.get() - 2
+        else:
+            big = self._size.get() + 2
+        self._bigfont = Font(family="helvetica", weight="bold", size=big)
+
+    def _init_grammar(self, parent):
+        # Grammar view.
+        self._prodframe = listframe = Frame(parent)
+        self._prodframe.pack(fill="both", side="left", padx=2)
+        self._prodlist_label = Label(
+            self._prodframe, font=self._boldfont, text="Available Expansions"
+        )
+        self._prodlist_label.pack()
+        self._prodlist = Listbox(
+            self._prodframe,
+            selectmode="single",
+            relief="groove",
+            background="white",
+            foreground="#909090",
+            font=self._font,
+            selectforeground="#004040",
+            selectbackground="#c0f0c0",
+        )
+
+        self._prodlist.pack(side="right", fill="both", expand=1)
+
+        self._productions = list(self._parser.grammar().productions())
+        for production in self._productions:
+            self._prodlist.insert("end", ("  %s" % production))
+        self._prodlist.config(height=min(len(self._productions), 25))
+
+        # Add a scrollbar if there are more than 25 productions.
+        if len(self._productions) > 25:
+            listscroll = Scrollbar(self._prodframe, orient="vertical")
+            self._prodlist.config(yscrollcommand=listscroll.set)
+            listscroll.config(command=self._prodlist.yview)
+            listscroll.pack(side="left", fill="y")
+
+        # If they select a production, apply it.
+        self._prodlist.bind("<<ListboxSelect>>", self._prodlist_select)
+
+    def _init_bindings(self):
+        # Key bindings are a good thing.
+        self._top.bind("<Control-q>", self.destroy)
+        self._top.bind("<Control-x>", self.destroy)
+        self._top.bind("<Escape>", self.destroy)
+        self._top.bind("e", self.expand)
+        # self._top.bind('<Alt-e>', self.expand)
+        # self._top.bind('<Control-e>', self.expand)
+        self._top.bind("m", self.match)
+        self._top.bind("<Alt-m>", self.match)
+        self._top.bind("<Control-m>", self.match)
+        self._top.bind("b", self.backtrack)
+        self._top.bind("<Alt-b>", self.backtrack)
+        self._top.bind("<Control-b>", self.backtrack)
+        self._top.bind("<Control-z>", self.backtrack)
+        self._top.bind("<BackSpace>", self.backtrack)
+        self._top.bind("a", self.autostep)
+        # self._top.bind('<Control-a>', self.autostep)
+        self._top.bind("<Control-space>", self.autostep)
+        self._top.bind("<Control-c>", self.cancel_autostep)
+        self._top.bind("<space>", self.step)
+        self._top.bind("<Delete>", self.reset)
+        self._top.bind("<Control-p>", self.postscript)
+        # self._top.bind('<h>', self.help)
+        # self._top.bind('<Alt-h>', self.help)
+        self._top.bind("<Control-h>", self.help)
+        self._top.bind("<F1>", self.help)
+        # self._top.bind('<g>', self.toggle_grammar)
+        # self._top.bind('<Alt-g>', self.toggle_grammar)
+        # self._top.bind('<Control-g>', self.toggle_grammar)
+        self._top.bind("<Control-g>", self.edit_grammar)
+        self._top.bind("<Control-t>", self.edit_sentence)
+
+    def _init_buttons(self, parent):
+        # Set up the frames.
+        self._buttonframe = buttonframe = Frame(parent)
+        buttonframe.pack(fill="none", side="bottom", padx=3, pady=2)
+        Button(
+            buttonframe,
+            text="Step",
+            background="#90c0d0",
+            foreground="black",
+            command=self.step,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Autostep",
+            background="#90c0d0",
+            foreground="black",
+            command=self.autostep,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Expand",
+            underline=0,
+            background="#90f090",
+            foreground="black",
+            command=self.expand,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Match",
+            underline=0,
+            background="#90f090",
+            foreground="black",
+            command=self.match,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Backtrack",
+            underline=0,
+            background="#f0a0a0",
+            foreground="black",
+            command=self.backtrack,
+        ).pack(side="left")
+        # Replace autostep...
+
+    #         self._autostep_button = Button(buttonframe, text='Autostep',
+    #                                        underline=0, command=self.autostep)
+    #         self._autostep_button.pack(side='left')
+
+    def _configure(self, event):
+        self._autostep = 0
+        (x1, y1, x2, y2) = self._cframe.scrollregion()
+        y2 = event.height - 6
+        self._canvas["scrollregion"] = "%d %d %d %d" % (x1, y1, x2, y2)
+        self._redraw()
+
+    def _init_feedback(self, parent):
+        self._feedbackframe = feedbackframe = Frame(parent)
+        feedbackframe.pack(fill="x", side="bottom", padx=3, pady=3)
+        self._lastoper_label = Label(
+            feedbackframe, text="Last Operation:", font=self._font
+        )
+        self._lastoper_label.pack(side="left")
+        lastoperframe = Frame(feedbackframe, relief="sunken", border=1)
+        lastoperframe.pack(fill="x", side="right", expand=1, padx=5)
+        self._lastoper1 = Label(
+            lastoperframe, foreground="#007070", background="#f0f0f0", font=self._font
+        )
+        self._lastoper2 = Label(
+            lastoperframe,
+            anchor="w",
+            width=30,
+            foreground="#004040",
+            background="#f0f0f0",
+            font=self._font,
+        )
+        self._lastoper1.pack(side="left")
+        self._lastoper2.pack(side="left", fill="x", expand=1)
+
+    def _init_canvas(self, parent):
+        self._cframe = CanvasFrame(
+            parent,
+            background="white",
+            # width=525, height=250,
+            closeenough=10,
+            border=2,
+            relief="sunken",
+        )
+        self._cframe.pack(expand=1, fill="both", side="top", pady=2)
+        canvas = self._canvas = self._cframe.canvas()
+
+        # Initially, there's no tree or text
+        self._tree = None
+        self._textwidgets = []
+        self._textline = None
+
+    def _init_menubar(self, parent):
+        menubar = Menu(parent)
+
+        filemenu = Menu(menubar, tearoff=0)
+        filemenu.add_command(
+            label="Reset Parser", underline=0, command=self.reset, accelerator="Del"
+        )
+        filemenu.add_command(
+            label="Print to Postscript",
+            underline=0,
+            command=self.postscript,
+            accelerator="Ctrl-p",
+        )
+        filemenu.add_command(
+            label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-x"
+        )
+        menubar.add_cascade(label="File", underline=0, menu=filemenu)
+
+        editmenu = Menu(menubar, tearoff=0)
+        editmenu.add_command(
+            label="Edit Grammar",
+            underline=5,
+            command=self.edit_grammar,
+            accelerator="Ctrl-g",
+        )
+        editmenu.add_command(
+            label="Edit Text",
+            underline=5,
+            command=self.edit_sentence,
+            accelerator="Ctrl-t",
+        )
+        menubar.add_cascade(label="Edit", underline=0, menu=editmenu)
+
+        rulemenu = Menu(menubar, tearoff=0)
+        rulemenu.add_command(
+            label="Step", underline=1, command=self.step, accelerator="Space"
+        )
+        rulemenu.add_separator()
+        rulemenu.add_command(
+            label="Match", underline=0, command=self.match, accelerator="Ctrl-m"
+        )
+        rulemenu.add_command(
+            label="Expand", underline=0, command=self.expand, accelerator="Ctrl-e"
+        )
+        rulemenu.add_separator()
+        rulemenu.add_command(
+            label="Backtrack", underline=0, command=self.backtrack, accelerator="Ctrl-b"
+        )
+        menubar.add_cascade(label="Apply", underline=0, menu=rulemenu)
+
+        viewmenu = Menu(menubar, tearoff=0)
+        viewmenu.add_checkbutton(
+            label="Show Grammar",
+            underline=0,
+            variable=self._show_grammar,
+            command=self._toggle_grammar,
+        )
+        viewmenu.add_separator()
+        viewmenu.add_radiobutton(
+            label="Tiny",
+            variable=self._size,
+            underline=0,
+            value=10,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Small",
+            variable=self._size,
+            underline=0,
+            value=12,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Medium",
+            variable=self._size,
+            underline=0,
+            value=14,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Large",
+            variable=self._size,
+            underline=0,
+            value=18,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Huge",
+            variable=self._size,
+            underline=0,
+            value=24,
+            command=self.resize,
+        )
+        menubar.add_cascade(label="View", underline=0, menu=viewmenu)
+
+        animatemenu = Menu(menubar, tearoff=0)
+        animatemenu.add_radiobutton(
+            label="No Animation", underline=0, variable=self._animation_frames, value=0
+        )
+        animatemenu.add_radiobutton(
+            label="Slow Animation",
+            underline=0,
+            variable=self._animation_frames,
+            value=10,
+            accelerator="-",
+        )
+        animatemenu.add_radiobutton(
+            label="Normal Animation",
+            underline=0,
+            variable=self._animation_frames,
+            value=5,
+            accelerator="=",
+        )
+        animatemenu.add_radiobutton(
+            label="Fast Animation",
+            underline=0,
+            variable=self._animation_frames,
+            value=2,
+            accelerator="+",
+        )
+        menubar.add_cascade(label="Animate", underline=1, menu=animatemenu)
+
+        helpmenu = Menu(menubar, tearoff=0)
+        helpmenu.add_command(label="About", underline=0, command=self.about)
+        helpmenu.add_command(
+            label="Instructions", underline=0, command=self.help, accelerator="F1"
+        )
+        menubar.add_cascade(label="Help", underline=0, menu=helpmenu)
+
+        parent.config(menu=menubar)
+
+    #########################################
+    ##  Helper
+    #########################################
+
+    def _get(self, widget, treeloc):
+        for i in treeloc:
+            widget = widget.subtrees()[i]
+        if isinstance(widget, TreeSegmentWidget):
+            widget = widget.label()
+        return widget
+
+    #########################################
+    ##  Main draw procedure
+    #########################################
+
+    def _redraw(self):
+        canvas = self._canvas
+
+        # Delete the old tree, widgets, etc.
+        if self._tree is not None:
+            self._cframe.destroy_widget(self._tree)
+        for twidget in self._textwidgets:
+            self._cframe.destroy_widget(twidget)
+        if self._textline is not None:
+            self._canvas.delete(self._textline)
+
+        # Draw the tree.
+        helv = ("helvetica", -self._size.get())
+        bold = ("helvetica", -self._size.get(), "bold")
+        attribs = {
+            "tree_color": "#000000",
+            "tree_width": 2,
+            "node_font": bold,
+            "leaf_font": helv,
+        }
+        tree = self._parser.tree()
+        self._tree = tree_to_treesegment(canvas, tree, **attribs)
+        self._cframe.add_widget(self._tree, 30, 5)
+
+        # Draw the text.
+        helv = ("helvetica", -self._size.get())
+        bottom = y = self._cframe.scrollregion()[3]
+        self._textwidgets = [
+            TextWidget(canvas, word, font=self._font) for word in self._sent
+        ]
+        for twidget in self._textwidgets:
+            self._cframe.add_widget(twidget, 0, 0)
+            twidget.move(0, bottom - twidget.bbox()[3] - 5)
+            y = min(y, twidget.bbox()[1])
+
+        # Draw a line over the text, to separate it from the tree.
+        self._textline = canvas.create_line(-5000, y - 5, 5000, y - 5, dash=".")
+
+        # Highlight appropriate nodes.
+        self._highlight_nodes()
+        self._highlight_prodlist()
+
+        # Make sure the text lines up.
+        self._position_text()
+
+    def _redraw_quick(self):
+        # This should be more-or-less sufficient after an animation.
+        self._highlight_nodes()
+        self._highlight_prodlist()
+        self._position_text()
+
+    def _highlight_nodes(self):
+        # Highlight the list of nodes to be checked.
+        bold = ("helvetica", -self._size.get(), "bold")
+        for treeloc in self._parser.frontier()[:1]:
+            self._get(self._tree, treeloc)["color"] = "#20a050"
+            self._get(self._tree, treeloc)["font"] = bold
+        for treeloc in self._parser.frontier()[1:]:
+            self._get(self._tree, treeloc)["color"] = "#008080"
+
+    def _highlight_prodlist(self):
+        # Highlight the productions that can be expanded.
+        # Boy, too bad tkinter doesn't implement Listbox.itemconfig;
+        # that would be pretty useful here.
+        self._prodlist.delete(0, "end")
+        expandable = self._parser.expandable_productions()
+        untried = self._parser.untried_expandable_productions()
+        productions = self._productions
+        for index in range(len(productions)):
+            if productions[index] in expandable:
+                if productions[index] in untried:
+                    self._prodlist.insert(index, " %s" % productions[index])
+                else:
+                    self._prodlist.insert(index, " %s (TRIED)" % productions[index])
+                self._prodlist.selection_set(index)
+            else:
+                self._prodlist.insert(index, " %s" % productions[index])
+
+    def _position_text(self):
+        # Line up the text widgets that are matched against the tree
+        numwords = len(self._sent)
+        num_matched = numwords - len(self._parser.remaining_text())
+        leaves = self._tree_leaves()[:num_matched]
+        xmax = self._tree.bbox()[0]
+        for i in range(0, len(leaves)):
+            widget = self._textwidgets[i]
+            leaf = leaves[i]
+            widget["color"] = "#006040"
+            leaf["color"] = "#006040"
+            widget.move(leaf.bbox()[0] - widget.bbox()[0], 0)
+            xmax = widget.bbox()[2] + 10
+
+        # Line up the text widgets that are not matched against the tree.
+        for i in range(len(leaves), numwords):
+            widget = self._textwidgets[i]
+            widget["color"] = "#a0a0a0"
+            widget.move(xmax - widget.bbox()[0], 0)
+            xmax = widget.bbox()[2] + 10
+
+        # If we have a complete parse, make everything green :)
+        if self._parser.currently_complete():
+            for twidget in self._textwidgets:
+                twidget["color"] = "#00a000"
+
+        # Move the matched leaves down to the text.
+        for i in range(0, len(leaves)):
+            widget = self._textwidgets[i]
+            leaf = leaves[i]
+            dy = widget.bbox()[1] - leaf.bbox()[3] - 10.0
+            dy = max(dy, leaf.parent().label().bbox()[3] - leaf.bbox()[3] + 10)
+            leaf.move(0, dy)
+
+    def _tree_leaves(self, tree=None):
+        if tree is None:
+            tree = self._tree
+        if isinstance(tree, TreeSegmentWidget):
+            leaves = []
+            for child in tree.subtrees():
+                leaves += self._tree_leaves(child)
+            return leaves
+        else:
+            return [tree]
+
+    #########################################
+    ##  Button Callbacks
+    #########################################
+
+    def destroy(self, *e):
+        self._autostep = 0
+        if self._top is None:
+            return
+        self._top.destroy()
+        self._top = None
+
+    def reset(self, *e):
+        self._autostep = 0
+        self._parser.initialize(self._sent)
+        self._lastoper1["text"] = "Reset Application"
+        self._lastoper2["text"] = ""
+        self._redraw()
+
+    def autostep(self, *e):
+        if self._animation_frames.get() == 0:
+            self._animation_frames.set(2)
+        if self._autostep:
+            self._autostep = 0
+        else:
+            self._autostep = 1
+            self._step()
+
+    def cancel_autostep(self, *e):
+        # self._autostep_button['text'] = 'Autostep'
+        self._autostep = 0
+
+    # Make sure to stop auto-stepping if we get any user input.
+    def step(self, *e):
+        self._autostep = 0
+        self._step()
+
+    def match(self, *e):
+        self._autostep = 0
+        self._match()
+
+    def expand(self, *e):
+        self._autostep = 0
+        self._expand()
+
+    def backtrack(self, *e):
+        self._autostep = 0
+        self._backtrack()
+
+    def _step(self):
+        if self._animating_lock:
+            return
+
+        # Try expanding, matching, and backtracking (in that order)
+        if self._expand():
+            pass
+        elif self._parser.untried_match() and self._match():
+            pass
+        elif self._backtrack():
+            pass
+        else:
+            self._lastoper1["text"] = "Finished"
+            self._lastoper2["text"] = ""
+            self._autostep = 0
+
+        # Check if we just completed a parse.
+        if self._parser.currently_complete():
+            self._autostep = 0
+            self._lastoper2["text"] += "    [COMPLETE PARSE]"
+
+    def _expand(self, *e):
+        if self._animating_lock:
+            return
+        old_frontier = self._parser.frontier()
+        rv = self._parser.expand()
+        if rv is not None:
+            self._lastoper1["text"] = "Expand:"
+            self._lastoper2["text"] = rv
+            self._prodlist.selection_clear(0, "end")
+            index = self._productions.index(rv)
+            self._prodlist.selection_set(index)
+            self._animate_expand(old_frontier[0])
+            return True
+        else:
+            self._lastoper1["text"] = "Expand:"
+            self._lastoper2["text"] = "(all expansions tried)"
+            return False
+
+    def _match(self, *e):
+        if self._animating_lock:
+            return
+        old_frontier = self._parser.frontier()
+        rv = self._parser.match()
+        if rv is not None:
+            self._lastoper1["text"] = "Match:"
+            self._lastoper2["text"] = rv
+            self._animate_match(old_frontier[0])
+            return True
+        else:
+            self._lastoper1["text"] = "Match:"
+            self._lastoper2["text"] = "(failed)"
+            return False
+
+    def _backtrack(self, *e):
+        if self._animating_lock:
+            return
+        if self._parser.backtrack():
+            elt = self._parser.tree()
+            for i in self._parser.frontier()[0]:
+                elt = elt[i]
+            self._lastoper1["text"] = "Backtrack"
+            self._lastoper2["text"] = ""
+            if isinstance(elt, Tree):
+                self._animate_backtrack(self._parser.frontier()[0])
+            else:
+                self._animate_match_backtrack(self._parser.frontier()[0])
+            return True
+        else:
+            self._autostep = 0
+            self._lastoper1["text"] = "Finished"
+            self._lastoper2["text"] = ""
+            return False
+
+    def about(self, *e):
+        ABOUT = (
+            "NLTK Recursive Descent Parser Application\n" + "Written by Edward Loper"
+        )
+        TITLE = "About: Recursive Descent Parser Application"
+        try:
+            from tkinter.messagebox import Message
+
+            Message(message=ABOUT, title=TITLE).show()
+        except:
+            ShowText(self._top, TITLE, ABOUT)
+
+    def help(self, *e):
+        self._autostep = 0
+        # The default font's not very legible; try using 'fixed' instead.
+        try:
+            ShowText(
+                self._top,
+                "Help: Recursive Descent Parser Application",
+                (__doc__ or "").strip(),
+                width=75,
+                font="fixed",
+            )
+        except:
+            ShowText(
+                self._top,
+                "Help: Recursive Descent Parser Application",
+                (__doc__ or "").strip(),
+                width=75,
+            )
+
+    def postscript(self, *e):
+        self._autostep = 0
+        self._cframe.print_to_file()
+
+    def mainloop(self, *args, **kwargs):
+        """
+        Enter the Tkinter mainloop.  This function must be called if
+        this demo is created from a non-interactive program (e.g.
+        from a secript); otherwise, the demo will close as soon as
+        the script completes.
+        """
+        if in_idle():
+            return
+        self._top.mainloop(*args, **kwargs)
+
+    def resize(self, size=None):
+        if size is not None:
+            self._size.set(size)
+        size = self._size.get()
+        self._font.configure(size=-(abs(size)))
+        self._boldfont.configure(size=-(abs(size)))
+        self._sysfont.configure(size=-(abs(size)))
+        self._bigfont.configure(size=-(abs(size + 2)))
+        self._redraw()
+
+    #########################################
+    ##  Expand Production Selection
+    #########################################
+
+    def _toggle_grammar(self, *e):
+        if self._show_grammar.get():
+            self._prodframe.pack(
+                fill="both", side="left", padx=2, after=self._feedbackframe
+            )
+            self._lastoper1["text"] = "Show Grammar"
+        else:
+            self._prodframe.pack_forget()
+            self._lastoper1["text"] = "Hide Grammar"
+        self._lastoper2["text"] = ""
+
+    #     def toggle_grammar(self, *e):
+    #         self._show_grammar = not self._show_grammar
+    #         if self._show_grammar:
+    #             self._prodframe.pack(fill='both', expand='y', side='left',
+    #                                  after=self._feedbackframe)
+    #             self._lastoper1['text'] = 'Show Grammar'
+    #         else:
+    #             self._prodframe.pack_forget()
+    #             self._lastoper1['text'] = 'Hide Grammar'
+    #         self._lastoper2['text'] = ''
+
+    def _prodlist_select(self, event):
+        selection = self._prodlist.curselection()
+        if len(selection) != 1:
+            return
+        index = int(selection[0])
+        old_frontier = self._parser.frontier()
+        production = self._parser.expand(self._productions[index])
+
+        if production:
+            self._lastoper1["text"] = "Expand:"
+            self._lastoper2["text"] = production
+            self._prodlist.selection_clear(0, "end")
+            self._prodlist.selection_set(index)
+            self._animate_expand(old_frontier[0])
+        else:
+            # Reset the production selections.
+            self._prodlist.selection_clear(0, "end")
+            for prod in self._parser.expandable_productions():
+                index = self._productions.index(prod)
+                self._prodlist.selection_set(index)
+
+    #########################################
+    ##  Animation
+    #########################################
+
+    def _animate_expand(self, treeloc):
+        oldwidget = self._get(self._tree, treeloc)
+        oldtree = oldwidget.parent()
+        top = not isinstance(oldtree.parent(), TreeSegmentWidget)
+
+        tree = self._parser.tree()
+        for i in treeloc:
+            tree = tree[i]
+
+        widget = tree_to_treesegment(
+            self._canvas,
+            tree,
+            node_font=self._boldfont,
+            leaf_color="white",
+            tree_width=2,
+            tree_color="white",
+            node_color="white",
+            leaf_font=self._font,
+        )
+        widget.label()["color"] = "#20a050"
+
+        (oldx, oldy) = oldtree.label().bbox()[:2]
+        (newx, newy) = widget.label().bbox()[:2]
+        widget.move(oldx - newx, oldy - newy)
+
+        if top:
+            self._cframe.add_widget(widget, 0, 5)
+            widget.move(30 - widget.label().bbox()[0], 0)
+            self._tree = widget
+        else:
+            oldtree.parent().replace_child(oldtree, widget)
+
+        # Move the children over so they don't overlap.
+        # Line the children up in a strange way.
+        if widget.subtrees():
+            dx = (
+                oldx
+                + widget.label().width() / 2
+                - widget.subtrees()[0].bbox()[0] / 2
+                - widget.subtrees()[0].bbox()[2] / 2
+            )
+            for subtree in widget.subtrees():
+                subtree.move(dx, 0)
+
+        self._makeroom(widget)
+
+        if top:
+            self._cframe.destroy_widget(oldtree)
+        else:
+            oldtree.destroy()
+
+        colors = [
+            "gray%d" % (10 * int(10 * x / self._animation_frames.get()))
+            for x in range(self._animation_frames.get(), 0, -1)
+        ]
+
+        # Move the text string down, if necessary.
+        dy = widget.bbox()[3] + 30 - self._canvas.coords(self._textline)[1]
+        if dy > 0:
+            for twidget in self._textwidgets:
+                twidget.move(0, dy)
+            self._canvas.move(self._textline, 0, dy)
+
+        self._animate_expand_frame(widget, colors)
+
+    def _makeroom(self, treeseg):
+        """
+        Make sure that no sibling tree bbox's overlap.
+        """
+        parent = treeseg.parent()
+        if not isinstance(parent, TreeSegmentWidget):
+            return
+
+        index = parent.subtrees().index(treeseg)
+
+        # Handle siblings to the right
+        rsiblings = parent.subtrees()[index + 1 :]
+        if rsiblings:
+            dx = treeseg.bbox()[2] - rsiblings[0].bbox()[0] + 10
+            for sibling in rsiblings:
+                sibling.move(dx, 0)
+
+        # Handle siblings to the left
+        if index > 0:
+            lsibling = parent.subtrees()[index - 1]
+            dx = max(0, lsibling.bbox()[2] - treeseg.bbox()[0] + 10)
+            treeseg.move(dx, 0)
+
+        # Keep working up the tree.
+        self._makeroom(parent)
+
+    def _animate_expand_frame(self, widget, colors):
+        if len(colors) > 0:
+            self._animating_lock = 1
+            widget["color"] = colors[0]
+            for subtree in widget.subtrees():
+                if isinstance(subtree, TreeSegmentWidget):
+                    subtree.label()["color"] = colors[0]
+                else:
+                    subtree["color"] = colors[0]
+            self._top.after(50, self._animate_expand_frame, widget, colors[1:])
+        else:
+            widget["color"] = "black"
+            for subtree in widget.subtrees():
+                if isinstance(subtree, TreeSegmentWidget):
+                    subtree.label()["color"] = "black"
+                else:
+                    subtree["color"] = "black"
+            self._redraw_quick()
+            widget.label()["color"] = "black"
+            self._animating_lock = 0
+            if self._autostep:
+                self._step()
+
+    def _animate_backtrack(self, treeloc):
+        # Flash red first, if we're animating.
+        if self._animation_frames.get() == 0:
+            colors = []
+        else:
+            colors = ["#a00000", "#000000", "#a00000"]
+        colors += [
+            "gray%d" % (10 * int(10 * x / (self._animation_frames.get())))
+            for x in range(1, self._animation_frames.get() + 1)
+        ]
+
+        widgets = [self._get(self._tree, treeloc).parent()]
+        for subtree in widgets[0].subtrees():
+            if isinstance(subtree, TreeSegmentWidget):
+                widgets.append(subtree.label())
+            else:
+                widgets.append(subtree)
+
+        self._animate_backtrack_frame(widgets, colors)
+
+    def _animate_backtrack_frame(self, widgets, colors):
+        if len(colors) > 0:
+            self._animating_lock = 1
+            for widget in widgets:
+                widget["color"] = colors[0]
+            self._top.after(50, self._animate_backtrack_frame, widgets, colors[1:])
+        else:
+            for widget in widgets[0].subtrees():
+                widgets[0].remove_child(widget)
+                widget.destroy()
+            self._redraw_quick()
+            self._animating_lock = 0
+            if self._autostep:
+                self._step()
+
+    def _animate_match_backtrack(self, treeloc):
+        widget = self._get(self._tree, treeloc)
+        node = widget.parent().label()
+        dy = (node.bbox()[3] - widget.bbox()[1] + 14) / max(
+            1, self._animation_frames.get()
+        )
+        self._animate_match_backtrack_frame(self._animation_frames.get(), widget, dy)
+
+    def _animate_match(self, treeloc):
+        widget = self._get(self._tree, treeloc)
+
+        dy = (self._textwidgets[0].bbox()[1] - widget.bbox()[3] - 10.0) / max(
+            1, self._animation_frames.get()
+        )
+        self._animate_match_frame(self._animation_frames.get(), widget, dy)
+
+    def _animate_match_frame(self, frame, widget, dy):
+        if frame > 0:
+            self._animating_lock = 1
+            widget.move(0, dy)
+            self._top.after(10, self._animate_match_frame, frame - 1, widget, dy)
+        else:
+            widget["color"] = "#006040"
+            self._redraw_quick()
+            self._animating_lock = 0
+            if self._autostep:
+                self._step()
+
+    def _animate_match_backtrack_frame(self, frame, widget, dy):
+        if frame > 0:
+            self._animating_lock = 1
+            widget.move(0, dy)
+            self._top.after(
+                10, self._animate_match_backtrack_frame, frame - 1, widget, dy
+            )
+        else:
+            widget.parent().remove_child(widget)
+            widget.destroy()
+            self._animating_lock = 0
+            if self._autostep:
+                self._step()
+
+    def edit_grammar(self, *e):
+        CFGEditor(self._top, self._parser.grammar(), self.set_grammar)
+
+    def set_grammar(self, grammar):
+        self._parser.set_grammar(grammar)
+        self._productions = list(grammar.productions())
+        self._prodlist.delete(0, "end")
+        for production in self._productions:
+            self._prodlist.insert("end", (" %s" % production))
+
+    def edit_sentence(self, *e):
+        sentence = " ".join(self._sent)
+        title = "Edit Text"
+        instr = "Enter a new sentence to parse."
+        EntryDialog(self._top, sentence, instr, self.set_sentence, title)
+
+    def set_sentence(self, sentence):
+        self._sent = sentence.split()  # [XX] use tagged?
+        self.reset()
+
+
+def app():
+    """
+    Create a recursive descent parser demo, using a simple grammar and
+    text.
+    """
+    from nltk.grammar import CFG
+
+    grammar = CFG.fromstring(
+        """
+    # Grammatical productions.
+        S -> NP VP
+        NP -> Det N PP | Det N
+        VP -> V NP PP | V NP | V
+        PP -> P NP
+    # Lexical productions.
+        NP -> 'I'
+        Det -> 'the' | 'a'
+        N -> 'man' | 'park' | 'dog' | 'telescope'
+        V -> 'ate' | 'saw'
+        P -> 'in' | 'under' | 'with'
+    """
+    )
+
+    sent = "the dog saw a man in the park".split()
+
+    RecursiveDescentApp(grammar, sent).mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/srparser_app.py

@@ -0,0 +1,937 @@
+# Natural Language Toolkit: Shift-Reduce Parser Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the shift-reduce parser.
+
+The shift-reduce parser maintains a stack, which records the structure
+of the portion of the text that has been parsed.  The stack is
+initially empty.  Its contents are shown on the left side of the main
+canvas.
+
+On the right side of the main canvas is the remaining text.  This is
+the portion of the text which has not yet been considered by the
+parser.
+
+The parser builds up a tree structure for the text using two
+operations:
+
+  - "shift" moves the first token from the remaining text to the top
+    of the stack.  In the demo, the top of the stack is its right-hand
+    side.
+  - "reduce" uses a grammar production to combine the rightmost stack
+    elements into a single tree token.
+
+You can control the parser's operation by using the "shift" and
+"reduce" buttons; or you can use the "step" button to let the parser
+automatically decide which operation to apply.  The parser uses the
+following rules to decide which operation to apply:
+
+  - Only shift if no reductions are available.
+  - If multiple reductions are available, then apply the reduction
+    whose CFG production is listed earliest in the grammar.
+
+The "reduce" button applies the reduction whose CFG production is
+listed earliest in the grammar.  There are two ways to manually choose
+which reduction to apply:
+
+  - Click on a CFG production from the list of available reductions,
+    on the left side of the main window.  The reduction based on that
+    production will be applied to the top of the stack.
+  - Click on one of the stack elements.  A popup window will appear,
+    containing all available reductions.  Select one, and it will be
+    applied to the top of the stack.
+
+Note that reductions can only be applied to the top of the stack.
+
+Keyboard Shortcuts::
+      [Space]\t Perform the next shift or reduce operation
+      [s]\t Perform a shift operation
+      [r]\t Perform a reduction operation
+      [Ctrl-z]\t Undo most recent operation
+      [Delete]\t Reset the parser
+      [g]\t Show/hide available production list
+      [Ctrl-a]\t Toggle animations
+      [h]\t Help
+      [Ctrl-p]\t Print
+      [q]\t Quit
+
+"""
+
+from tkinter import Button, Frame, IntVar, Label, Listbox, Menu, Scrollbar, Tk
+from tkinter.font import Font
+
+from nltk.draw import CFGEditor, TreeSegmentWidget, tree_to_treesegment
+from nltk.draw.util import CanvasFrame, EntryDialog, ShowText, TextWidget
+from nltk.parse import SteppingShiftReduceParser
+from nltk.tree import Tree
+from nltk.util import in_idle
+
+"""
+Possible future improvements:
+  - button/window to change and/or select text.  Just pop up a window
+    with an entry, and let them modify the text; and then retokenize
+    it?  Maybe give a warning if it contains tokens whose types are
+    not in the grammar.
+  - button/window to change and/or select grammar.  Select from
+    several alternative grammars?  Or actually change the grammar?  If
+    the later, then I'd want to define nltk.draw.cfg, which would be
+    responsible for that.
+"""
+
+
+class ShiftReduceApp:
+    """
+    A graphical tool for exploring the shift-reduce parser.  The tool
+    displays the parser's stack and the remaining text, and allows the
+    user to control the parser's operation.  In particular, the user
+    can shift tokens onto the stack, and can perform reductions on the
+    top elements of the stack.  A "step" button simply steps through
+    the parsing process, performing the operations that
+    ``nltk.parse.ShiftReduceParser`` would use.
+    """
+
+    def __init__(self, grammar, sent, trace=0):
+        self._sent = sent
+        self._parser = SteppingShiftReduceParser(grammar, trace)
+
+        # Set up the main window.
+        self._top = Tk()
+        self._top.title("Shift Reduce Parser Application")
+
+        # Animations.  animating_lock is a lock to prevent the demo
+        # from performing new operations while it's animating.
+        self._animating_lock = 0
+        self._animate = IntVar(self._top)
+        self._animate.set(10)  # = medium
+
+        # The user can hide the grammar.
+        self._show_grammar = IntVar(self._top)
+        self._show_grammar.set(1)
+
+        # Initialize fonts.
+        self._init_fonts(self._top)
+
+        # Set up key bindings.
+        self._init_bindings()
+
+        # Create the basic frames.
+        self._init_menubar(self._top)
+        self._init_buttons(self._top)
+        self._init_feedback(self._top)
+        self._init_grammar(self._top)
+        self._init_canvas(self._top)
+
+        # A popup menu for reducing.
+        self._reduce_menu = Menu(self._canvas, tearoff=0)
+
+        # Reset the demo, and set the feedback frame to empty.
+        self.reset()
+        self._lastoper1["text"] = ""
+
+    #########################################
+    ##  Initialization Helpers
+    #########################################
+
+    def _init_fonts(self, root):
+        # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+        self._sysfont = Font(font=Button()["font"])
+        root.option_add("*Font", self._sysfont)
+
+        # TWhat's our font size (default=same as sysfont)
+        self._size = IntVar(root)
+        self._size.set(self._sysfont.cget("size"))
+
+        self._boldfont = Font(family="helvetica", weight="bold", size=self._size.get())
+        self._font = Font(family="helvetica", size=self._size.get())
+
+    def _init_grammar(self, parent):
+        # Grammar view.
+        self._prodframe = listframe = Frame(parent)
+        self._prodframe.pack(fill="both", side="left", padx=2)
+        self._prodlist_label = Label(
+            self._prodframe, font=self._boldfont, text="Available Reductions"
+        )
+        self._prodlist_label.pack()
+        self._prodlist = Listbox(
+            self._prodframe,
+            selectmode="single",
+            relief="groove",
+            background="white",
+            foreground="#909090",
+            font=self._font,
+            selectforeground="#004040",
+            selectbackground="#c0f0c0",
+        )
+
+        self._prodlist.pack(side="right", fill="both", expand=1)
+
+        self._productions = list(self._parser.grammar().productions())
+        for production in self._productions:
+            self._prodlist.insert("end", (" %s" % production))
+        self._prodlist.config(height=min(len(self._productions), 25))
+
+        # Add a scrollbar if there are more than 25 productions.
+        if 1:  # len(self._productions) > 25:
+            listscroll = Scrollbar(self._prodframe, orient="vertical")
+            self._prodlist.config(yscrollcommand=listscroll.set)
+            listscroll.config(command=self._prodlist.yview)
+            listscroll.pack(side="left", fill="y")
+
+        # If they select a production, apply it.
+        self._prodlist.bind("<<ListboxSelect>>", self._prodlist_select)
+
+        # When they hover over a production, highlight it.
+        self._hover = -1
+        self._prodlist.bind("<Motion>", self._highlight_hover)
+        self._prodlist.bind("<Leave>", self._clear_hover)
+
+    def _init_bindings(self):
+        # Quit
+        self._top.bind("<Control-q>", self.destroy)
+        self._top.bind("<Control-x>", self.destroy)
+        self._top.bind("<Alt-q>", self.destroy)
+        self._top.bind("<Alt-x>", self.destroy)
+
+        # Ops (step, shift, reduce, undo)
+        self._top.bind("<space>", self.step)
+        self._top.bind("<s>", self.shift)
+        self._top.bind("<Alt-s>", self.shift)
+        self._top.bind("<Control-s>", self.shift)
+        self._top.bind("<r>", self.reduce)
+        self._top.bind("<Alt-r>", self.reduce)
+        self._top.bind("<Control-r>", self.reduce)
+        self._top.bind("<Delete>", self.reset)
+        self._top.bind("<u>", self.undo)
+        self._top.bind("<Alt-u>", self.undo)
+        self._top.bind("<Control-u>", self.undo)
+        self._top.bind("<Control-z>", self.undo)
+        self._top.bind("<BackSpace>", self.undo)
+
+        # Misc
+        self._top.bind("<Control-p>", self.postscript)
+        self._top.bind("<Control-h>", self.help)
+        self._top.bind("<F1>", self.help)
+        self._top.bind("<Control-g>", self.edit_grammar)
+        self._top.bind("<Control-t>", self.edit_sentence)
+
+        # Animation speed control
+        self._top.bind("-", lambda e, a=self._animate: a.set(20))
+        self._top.bind("=", lambda e, a=self._animate: a.set(10))
+        self._top.bind("+", lambda e, a=self._animate: a.set(4))
+
+    def _init_buttons(self, parent):
+        # Set up the frames.
+        self._buttonframe = buttonframe = Frame(parent)
+        buttonframe.pack(fill="none", side="bottom")
+        Button(
+            buttonframe,
+            text="Step",
+            background="#90c0d0",
+            foreground="black",
+            command=self.step,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Shift",
+            underline=0,
+            background="#90f090",
+            foreground="black",
+            command=self.shift,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Reduce",
+            underline=0,
+            background="#90f090",
+            foreground="black",
+            command=self.reduce,
+        ).pack(side="left")
+        Button(
+            buttonframe,
+            text="Undo",
+            underline=0,
+            background="#f0a0a0",
+            foreground="black",
+            command=self.undo,
+        ).pack(side="left")
+
+    def _init_menubar(self, parent):
+        menubar = Menu(parent)
+
+        filemenu = Menu(menubar, tearoff=0)
+        filemenu.add_command(
+            label="Reset Parser", underline=0, command=self.reset, accelerator="Del"
+        )
+        filemenu.add_command(
+            label="Print to Postscript",
+            underline=0,
+            command=self.postscript,
+            accelerator="Ctrl-p",
+        )
+        filemenu.add_command(
+            label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-x"
+        )
+        menubar.add_cascade(label="File", underline=0, menu=filemenu)
+
+        editmenu = Menu(menubar, tearoff=0)
+        editmenu.add_command(
+            label="Edit Grammar",
+            underline=5,
+            command=self.edit_grammar,
+            accelerator="Ctrl-g",
+        )
+        editmenu.add_command(
+            label="Edit Text",
+            underline=5,
+            command=self.edit_sentence,
+            accelerator="Ctrl-t",
+        )
+        menubar.add_cascade(label="Edit", underline=0, menu=editmenu)
+
+        rulemenu = Menu(menubar, tearoff=0)
+        rulemenu.add_command(
+            label="Step", underline=1, command=self.step, accelerator="Space"
+        )
+        rulemenu.add_separator()
+        rulemenu.add_command(
+            label="Shift", underline=0, command=self.shift, accelerator="Ctrl-s"
+        )
+        rulemenu.add_command(
+            label="Reduce", underline=0, command=self.reduce, accelerator="Ctrl-r"
+        )
+        rulemenu.add_separator()
+        rulemenu.add_command(
+            label="Undo", underline=0, command=self.undo, accelerator="Ctrl-u"
+        )
+        menubar.add_cascade(label="Apply", underline=0, menu=rulemenu)
+
+        viewmenu = Menu(menubar, tearoff=0)
+        viewmenu.add_checkbutton(
+            label="Show Grammar",
+            underline=0,
+            variable=self._show_grammar,
+            command=self._toggle_grammar,
+        )
+        viewmenu.add_separator()
+        viewmenu.add_radiobutton(
+            label="Tiny",
+            variable=self._size,
+            underline=0,
+            value=10,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Small",
+            variable=self._size,
+            underline=0,
+            value=12,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Medium",
+            variable=self._size,
+            underline=0,
+            value=14,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Large",
+            variable=self._size,
+            underline=0,
+            value=18,
+            command=self.resize,
+        )
+        viewmenu.add_radiobutton(
+            label="Huge",
+            variable=self._size,
+            underline=0,
+            value=24,
+            command=self.resize,
+        )
+        menubar.add_cascade(label="View", underline=0, menu=viewmenu)
+
+        animatemenu = Menu(menubar, tearoff=0)
+        animatemenu.add_radiobutton(
+            label="No Animation", underline=0, variable=self._animate, value=0
+        )
+        animatemenu.add_radiobutton(
+            label="Slow Animation",
+            underline=0,
+            variable=self._animate,
+            value=20,
+            accelerator="-",
+        )
+        animatemenu.add_radiobutton(
+            label="Normal Animation",
+            underline=0,
+            variable=self._animate,
+            value=10,
+            accelerator="=",
+        )
+        animatemenu.add_radiobutton(
+            label="Fast Animation",
+            underline=0,
+            variable=self._animate,
+            value=4,
+            accelerator="+",
+        )
+        menubar.add_cascade(label="Animate", underline=1, menu=animatemenu)
+
+        helpmenu = Menu(menubar, tearoff=0)
+        helpmenu.add_command(label="About", underline=0, command=self.about)
+        helpmenu.add_command(
+            label="Instructions", underline=0, command=self.help, accelerator="F1"
+        )
+        menubar.add_cascade(label="Help", underline=0, menu=helpmenu)
+
+        parent.config(menu=menubar)
+
+    def _init_feedback(self, parent):
+        self._feedbackframe = feedbackframe = Frame(parent)
+        feedbackframe.pack(fill="x", side="bottom", padx=3, pady=3)
+        self._lastoper_label = Label(
+            feedbackframe, text="Last Operation:", font=self._font
+        )
+        self._lastoper_label.pack(side="left")
+        lastoperframe = Frame(feedbackframe, relief="sunken", border=1)
+        lastoperframe.pack(fill="x", side="right", expand=1, padx=5)
+        self._lastoper1 = Label(
+            lastoperframe, foreground="#007070", background="#f0f0f0", font=self._font
+        )
+        self._lastoper2 = Label(
+            lastoperframe,
+            anchor="w",
+            width=30,
+            foreground="#004040",
+            background="#f0f0f0",
+            font=self._font,
+        )
+        self._lastoper1.pack(side="left")
+        self._lastoper2.pack(side="left", fill="x", expand=1)
+
+    def _init_canvas(self, parent):
+        self._cframe = CanvasFrame(
+            parent,
+            background="white",
+            width=525,
+            closeenough=10,
+            border=2,
+            relief="sunken",
+        )
+        self._cframe.pack(expand=1, fill="both", side="top", pady=2)
+        canvas = self._canvas = self._cframe.canvas()
+
+        self._stackwidgets = []
+        self._rtextwidgets = []
+        self._titlebar = canvas.create_rectangle(
+            0, 0, 0, 0, fill="#c0f0f0", outline="black"
+        )
+        self._exprline = canvas.create_line(0, 0, 0, 0, dash=".")
+        self._stacktop = canvas.create_line(0, 0, 0, 0, fill="#408080")
+        size = self._size.get() + 4
+        self._stacklabel = TextWidget(
+            canvas, "Stack", color="#004040", font=self._boldfont
+        )
+        self._rtextlabel = TextWidget(
+            canvas, "Remaining Text", color="#004040", font=self._boldfont
+        )
+        self._cframe.add_widget(self._stacklabel)
+        self._cframe.add_widget(self._rtextlabel)
+
+    #########################################
+    ##  Main draw procedure
+    #########################################
+
+    def _redraw(self):
+        scrollregion = self._canvas["scrollregion"].split()
+        (cx1, cy1, cx2, cy2) = (int(c) for c in scrollregion)
+
+        # Delete the old stack & rtext widgets.
+        for stackwidget in self._stackwidgets:
+            self._cframe.destroy_widget(stackwidget)
+        self._stackwidgets = []
+        for rtextwidget in self._rtextwidgets:
+            self._cframe.destroy_widget(rtextwidget)
+        self._rtextwidgets = []
+
+        # Position the titlebar & exprline
+        (x1, y1, x2, y2) = self._stacklabel.bbox()
+        y = y2 - y1 + 10
+        self._canvas.coords(self._titlebar, -5000, 0, 5000, y - 4)
+        self._canvas.coords(self._exprline, 0, y * 2 - 10, 5000, y * 2 - 10)
+
+        # Position the titlebar labels..
+        (x1, y1, x2, y2) = self._stacklabel.bbox()
+        self._stacklabel.move(5 - x1, 3 - y1)
+        (x1, y1, x2, y2) = self._rtextlabel.bbox()
+        self._rtextlabel.move(cx2 - x2 - 5, 3 - y1)
+
+        # Draw the stack.
+        stackx = 5
+        for tok in self._parser.stack():
+            if isinstance(tok, Tree):
+                attribs = {
+                    "tree_color": "#4080a0",
+                    "tree_width": 2,
+                    "node_font": self._boldfont,
+                    "node_color": "#006060",
+                    "leaf_color": "#006060",
+                    "leaf_font": self._font,
+                }
+                widget = tree_to_treesegment(self._canvas, tok, **attribs)
+                widget.label()["color"] = "#000000"
+            else:
+                widget = TextWidget(self._canvas, tok, color="#000000", font=self._font)
+            widget.bind_click(self._popup_reduce)
+            self._stackwidgets.append(widget)
+            self._cframe.add_widget(widget, stackx, y)
+            stackx = widget.bbox()[2] + 10
+
+        # Draw the remaining text.
+        rtextwidth = 0
+        for tok in self._parser.remaining_text():
+            widget = TextWidget(self._canvas, tok, color="#000000", font=self._font)
+            self._rtextwidgets.append(widget)
+            self._cframe.add_widget(widget, rtextwidth, y)
+            rtextwidth = widget.bbox()[2] + 4
+
+        # Allow enough room to shift the next token (for animations)
+        if len(self._rtextwidgets) > 0:
+            stackx += self._rtextwidgets[0].width()
+
+        # Move the remaining text to the correct location (keep it
+        # right-justified, when possible); and move the remaining text
+        # label, if necessary.
+        stackx = max(stackx, self._stacklabel.width() + 25)
+        rlabelwidth = self._rtextlabel.width() + 10
+        if stackx >= cx2 - max(rtextwidth, rlabelwidth):
+            cx2 = stackx + max(rtextwidth, rlabelwidth)
+        for rtextwidget in self._rtextwidgets:
+            rtextwidget.move(4 + cx2 - rtextwidth, 0)
+        self._rtextlabel.move(cx2 - self._rtextlabel.bbox()[2] - 5, 0)
+
+        midx = (stackx + cx2 - max(rtextwidth, rlabelwidth)) / 2
+        self._canvas.coords(self._stacktop, midx, 0, midx, 5000)
+        (x1, y1, x2, y2) = self._stacklabel.bbox()
+
+        # Set up binding to allow them to shift a token by dragging it.
+        if len(self._rtextwidgets) > 0:
+
+            def drag_shift(widget, midx=midx, self=self):
+                if widget.bbox()[0] < midx:
+                    self.shift()
+                else:
+                    self._redraw()
+
+            self._rtextwidgets[0].bind_drag(drag_shift)
+            self._rtextwidgets[0].bind_click(self.shift)
+
+        # Draw the stack top.
+        self._highlight_productions()
+
+    def _draw_stack_top(self, widget):
+        # hack..
+        midx = widget.bbox()[2] + 50
+        self._canvas.coords(self._stacktop, midx, 0, midx, 5000)
+
+    def _highlight_productions(self):
+        # Highlight the productions that can be reduced.
+        self._prodlist.selection_clear(0, "end")
+        for prod in self._parser.reducible_productions():
+            index = self._productions.index(prod)
+            self._prodlist.selection_set(index)
+
+    #########################################
+    ##  Button Callbacks
+    #########################################
+
+    def destroy(self, *e):
+        if self._top is None:
+            return
+        self._top.destroy()
+        self._top = None
+
+    def reset(self, *e):
+        self._parser.initialize(self._sent)
+        self._lastoper1["text"] = "Reset App"
+        self._lastoper2["text"] = ""
+        self._redraw()
+
+    def step(self, *e):
+        if self.reduce():
+            return True
+        elif self.shift():
+            return True
+        else:
+            if list(self._parser.parses()):
+                self._lastoper1["text"] = "Finished:"
+                self._lastoper2["text"] = "Success"
+            else:
+                self._lastoper1["text"] = "Finished:"
+                self._lastoper2["text"] = "Failure"
+
+    def shift(self, *e):
+        if self._animating_lock:
+            return
+        if self._parser.shift():
+            tok = self._parser.stack()[-1]
+            self._lastoper1["text"] = "Shift:"
+            self._lastoper2["text"] = "%r" % tok
+            if self._animate.get():
+                self._animate_shift()
+            else:
+                self._redraw()
+            return True
+        return False
+
+    def reduce(self, *e):
+        if self._animating_lock:
+            return
+        production = self._parser.reduce()
+        if production:
+            self._lastoper1["text"] = "Reduce:"
+            self._lastoper2["text"] = "%s" % production
+            if self._animate.get():
+                self._animate_reduce()
+            else:
+                self._redraw()
+        return production
+
+    def undo(self, *e):
+        if self._animating_lock:
+            return
+        if self._parser.undo():
+            self._redraw()
+
+    def postscript(self, *e):
+        self._cframe.print_to_file()
+
+    def mainloop(self, *args, **kwargs):
+        """
+        Enter the Tkinter mainloop.  This function must be called if
+        this demo is created from a non-interactive program (e.g.
+        from a secript); otherwise, the demo will close as soon as
+        the script completes.
+        """
+        if in_idle():
+            return
+        self._top.mainloop(*args, **kwargs)
+
+    #########################################
+    ##  Menubar callbacks
+    #########################################
+
+    def resize(self, size=None):
+        if size is not None:
+            self._size.set(size)
+        size = self._size.get()
+        self._font.configure(size=-(abs(size)))
+        self._boldfont.configure(size=-(abs(size)))
+        self._sysfont.configure(size=-(abs(size)))
+
+        # self._stacklabel['font'] = ('helvetica', -size-4, 'bold')
+        # self._rtextlabel['font'] = ('helvetica', -size-4, 'bold')
+        # self._lastoper_label['font'] = ('helvetica', -size)
+        # self._lastoper1['font'] = ('helvetica', -size)
+        # self._lastoper2['font'] = ('helvetica', -size)
+        # self._prodlist['font'] = ('helvetica', -size)
+        # self._prodlist_label['font'] = ('helvetica', -size-2, 'bold')
+        self._redraw()
+
+    def help(self, *e):
+        # The default font's not very legible; try using 'fixed' instead.
+        try:
+            ShowText(
+                self._top,
+                "Help: Shift-Reduce Parser Application",
+                (__doc__ or "").strip(),
+                width=75,
+                font="fixed",
+            )
+        except:
+            ShowText(
+                self._top,
+                "Help: Shift-Reduce Parser Application",
+                (__doc__ or "").strip(),
+                width=75,
+            )
+
+    def about(self, *e):
+        ABOUT = "NLTK Shift-Reduce Parser Application\n" + "Written by Edward Loper"
+        TITLE = "About: Shift-Reduce Parser Application"
+        try:
+            from tkinter.messagebox import Message
+
+            Message(message=ABOUT, title=TITLE).show()
+        except:
+            ShowText(self._top, TITLE, ABOUT)
+
+    def edit_grammar(self, *e):
+        CFGEditor(self._top, self._parser.grammar(), self.set_grammar)
+
+    def set_grammar(self, grammar):
+        self._parser.set_grammar(grammar)
+        self._productions = list(grammar.productions())
+        self._prodlist.delete(0, "end")
+        for production in self._productions:
+            self._prodlist.insert("end", (" %s" % production))
+
+    def edit_sentence(self, *e):
+        sentence = " ".join(self._sent)
+        title = "Edit Text"
+        instr = "Enter a new sentence to parse."
+        EntryDialog(self._top, sentence, instr, self.set_sentence, title)
+
+    def set_sentence(self, sent):
+        self._sent = sent.split()  # [XX] use tagged?
+        self.reset()
+
+    #########################################
+    ##  Reduce Production Selection
+    #########################################
+
+    def _toggle_grammar(self, *e):
+        if self._show_grammar.get():
+            self._prodframe.pack(
+                fill="both", side="left", padx=2, after=self._feedbackframe
+            )
+            self._lastoper1["text"] = "Show Grammar"
+        else:
+            self._prodframe.pack_forget()
+            self._lastoper1["text"] = "Hide Grammar"
+        self._lastoper2["text"] = ""
+
+    def _prodlist_select(self, event):
+        selection = self._prodlist.curselection()
+        if len(selection) != 1:
+            return
+        index = int(selection[0])
+        production = self._parser.reduce(self._productions[index])
+        if production:
+            self._lastoper1["text"] = "Reduce:"
+            self._lastoper2["text"] = "%s" % production
+            if self._animate.get():
+                self._animate_reduce()
+            else:
+                self._redraw()
+        else:
+            # Reset the production selections.
+            self._prodlist.selection_clear(0, "end")
+            for prod in self._parser.reducible_productions():
+                index = self._productions.index(prod)
+                self._prodlist.selection_set(index)
+
+    def _popup_reduce(self, widget):
+        # Remove old commands.
+        productions = self._parser.reducible_productions()
+        if len(productions) == 0:
+            return
+
+        self._reduce_menu.delete(0, "end")
+        for production in productions:
+            self._reduce_menu.add_command(label=str(production), command=self.reduce)
+        self._reduce_menu.post(
+            self._canvas.winfo_pointerx(), self._canvas.winfo_pointery()
+        )
+
+    #########################################
+    ##  Animations
+    #########################################
+
+    def _animate_shift(self):
+        # What widget are we shifting?
+        widget = self._rtextwidgets[0]
+
+        # Where are we shifting from & to?
+        right = widget.bbox()[0]
+        if len(self._stackwidgets) == 0:
+            left = 5
+        else:
+            left = self._stackwidgets[-1].bbox()[2] + 10
+
+        # Start animating.
+        dt = self._animate.get()
+        dx = (left - right) * 1.0 / dt
+        self._animate_shift_frame(dt, widget, dx)
+
+    def _animate_shift_frame(self, frame, widget, dx):
+        if frame > 0:
+            self._animating_lock = 1
+            widget.move(dx, 0)
+            self._top.after(10, self._animate_shift_frame, frame - 1, widget, dx)
+        else:
+            # but: stacktop??
+
+            # Shift the widget to the stack.
+            del self._rtextwidgets[0]
+            self._stackwidgets.append(widget)
+            self._animating_lock = 0
+
+            # Display the available productions.
+            self._draw_stack_top(widget)
+            self._highlight_productions()
+
+    def _animate_reduce(self):
+        # What widgets are we shifting?
+        numwidgets = len(self._parser.stack()[-1])  # number of children
+        widgets = self._stackwidgets[-numwidgets:]
+
+        # How far are we moving?
+        if isinstance(widgets[0], TreeSegmentWidget):
+            ydist = 15 + widgets[0].label().height()
+        else:
+            ydist = 15 + widgets[0].height()
+
+        # Start animating.
+        dt = self._animate.get()
+        dy = ydist * 2.0 / dt
+        self._animate_reduce_frame(dt / 2, widgets, dy)
+
+    def _animate_reduce_frame(self, frame, widgets, dy):
+        if frame > 0:
+            self._animating_lock = 1
+            for widget in widgets:
+                widget.move(0, dy)
+            self._top.after(10, self._animate_reduce_frame, frame - 1, widgets, dy)
+        else:
+            del self._stackwidgets[-len(widgets) :]
+            for widget in widgets:
+                self._cframe.remove_widget(widget)
+            tok = self._parser.stack()[-1]
+            if not isinstance(tok, Tree):
+                raise ValueError()
+            label = TextWidget(
+                self._canvas, str(tok.label()), color="#006060", font=self._boldfont
+            )
+            widget = TreeSegmentWidget(self._canvas, label, widgets, width=2)
+            (x1, y1, x2, y2) = self._stacklabel.bbox()
+            y = y2 - y1 + 10
+            if not self._stackwidgets:
+                x = 5
+            else:
+                x = self._stackwidgets[-1].bbox()[2] + 10
+            self._cframe.add_widget(widget, x, y)
+            self._stackwidgets.append(widget)
+
+            # Display the available productions.
+            self._draw_stack_top(widget)
+            self._highlight_productions()
+
+            #             # Delete the old widgets..
+            #             del self._stackwidgets[-len(widgets):]
+            #             for widget in widgets:
+            #                 self._cframe.destroy_widget(widget)
+            #
+            #             # Make a new one.
+            #             tok = self._parser.stack()[-1]
+            #             if isinstance(tok, Tree):
+            #                 attribs = {'tree_color': '#4080a0', 'tree_width': 2,
+            #                            'node_font': bold, 'node_color': '#006060',
+            #                            'leaf_color': '#006060', 'leaf_font':self._font}
+            #                 widget = tree_to_treesegment(self._canvas, tok.type(),
+            #                                              **attribs)
+            #                 widget.node()['color'] = '#000000'
+            #             else:
+            #                 widget = TextWidget(self._canvas, tok.type(),
+            #                                     color='#000000', font=self._font)
+            #             widget.bind_click(self._popup_reduce)
+            #             (x1, y1, x2, y2) = self._stacklabel.bbox()
+            #             y = y2-y1+10
+            #             if not self._stackwidgets: x = 5
+            #             else: x = self._stackwidgets[-1].bbox()[2] + 10
+            #             self._cframe.add_widget(widget, x, y)
+            #             self._stackwidgets.append(widget)
+
+            # self._redraw()
+            self._animating_lock = 0
+
+    #########################################
+    ##  Hovering.
+    #########################################
+
+    def _highlight_hover(self, event):
+        # What production are we hovering over?
+        index = self._prodlist.nearest(event.y)
+        if self._hover == index:
+            return
+
+        # Clear any previous hover highlighting.
+        self._clear_hover()
+
+        # If the production corresponds to an available reduction,
+        # highlight the stack.
+        selection = [int(s) for s in self._prodlist.curselection()]
+        if index in selection:
+            rhslen = len(self._productions[index].rhs())
+            for stackwidget in self._stackwidgets[-rhslen:]:
+                if isinstance(stackwidget, TreeSegmentWidget):
+                    stackwidget.label()["color"] = "#00a000"
+                else:
+                    stackwidget["color"] = "#00a000"
+
+        # Remember what production we're hovering over.
+        self._hover = index
+
+    def _clear_hover(self, *event):
+        # Clear any previous hover highlighting.
+        if self._hover == -1:
+            return
+        self._hover = -1
+        for stackwidget in self._stackwidgets:
+            if isinstance(stackwidget, TreeSegmentWidget):
+                stackwidget.label()["color"] = "black"
+            else:
+                stackwidget["color"] = "black"
+
+
+def app():
+    """
+    Create a shift reduce parser app, using a simple grammar and
+    text.
+    """
+
+    from nltk.grammar import CFG, Nonterminal, Production
+
+    nonterminals = "S VP NP PP P N Name V Det"
+    (S, VP, NP, PP, P, N, Name, V, Det) = (Nonterminal(s) for s in nonterminals.split())
+
+    productions = (
+        # Syntactic Productions
+        Production(S, [NP, VP]),
+        Production(NP, [Det, N]),
+        Production(NP, [NP, PP]),
+        Production(VP, [VP, PP]),
+        Production(VP, [V, NP, PP]),
+        Production(VP, [V, NP]),
+        Production(PP, [P, NP]),
+        # Lexical Productions
+        Production(NP, ["I"]),
+        Production(Det, ["the"]),
+        Production(Det, ["a"]),
+        Production(N, ["man"]),
+        Production(V, ["saw"]),
+        Production(P, ["in"]),
+        Production(P, ["with"]),
+        Production(N, ["park"]),
+        Production(N, ["dog"]),
+        Production(N, ["statue"]),
+        Production(Det, ["my"]),
+    )
+
+    grammar = CFG(S, productions)
+
+    # tokenize the sentence
+    sent = "my dog saw a man in the park with a statue".split()
+
+    ShiftReduceApp(grammar, sent).mainloop()
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/wordfreq_app.py

@@ -0,0 +1,36 @@
+# Natural Language Toolkit: Wordfreq Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Sumukh Ghodke <sghodke@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from matplotlib import pylab
+
+from nltk.corpus import gutenberg
+from nltk.text import Text
+
+
+def plot_word_freq_dist(text):
+    fd = text.vocab()
+
+    samples = [item for item, _ in fd.most_common(50)]
+    values = [fd[sample] for sample in samples]
+    values = [sum(values[: i + 1]) * 100.0 / fd.N() for i in range(len(values))]
+    pylab.title(text.name)
+    pylab.xlabel("Samples")
+    pylab.ylabel("Cumulative Percentage")
+    pylab.plot(values)
+    pylab.xticks(range(len(samples)), [str(s) for s in samples], rotation=90)
+    pylab.show()
+
+
+def app():
+    t1 = Text(gutenberg.words("melville-moby_dick.txt"))
+    plot_word_freq_dist(t1)
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/app/wordnet_app.py

@@ -0,0 +1,997 @@
+# Natural Language Toolkit: WordNet Browser Application
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Jussi Salmela <jtsalmela@users.sourceforge.net>
+#         Paul Bone <pbone@students.csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A WordNet Browser application which launches the default browser
+(if it is not already running) and opens a new tab with a connection
+to http://localhost:port/ .  It also starts an HTTP server on the
+specified port and begins serving browser requests.  The default
+port is 8000.  (For command-line help, run "python wordnet -h")
+This application requires that the user's web browser supports
+Javascript.
+
+BrowServer is a server for browsing the NLTK Wordnet database It first
+launches a browser client to be used for browsing and then starts
+serving the requests of that and maybe other clients
+
+Usage::
+
+    browserver.py -h
+    browserver.py [-s] [-p <port>]
+
+Options::
+
+    -h or --help
+        Display this help message.
+
+    -l <file> or --log-file <file>
+        Logs messages to the given file, If this option is not specified
+        messages are silently dropped.
+
+    -p <port> or --port <port>
+        Run the web server on this TCP port, defaults to 8000.
+
+    -s or --server-mode
+        Do not start a web browser, and do not allow a user to
+        shutdown the server through the web interface.
+"""
+# TODO: throughout this package variable names and docstrings need
+# modifying to be compliant with NLTK's coding standards.  Tests also
+# need to be develop to ensure this continues to work in the face of
+# changes to other NLTK packages.
+
+import base64
+import copy
+import datetime
+import getopt
+import os
+import pickle
+import re
+import sys
+import threading
+import time
+import webbrowser
+from collections import defaultdict
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+# Allow this program to run inside the NLTK source tree.
+from sys import argv, path
+from urllib.parse import unquote_plus
+
+from nltk.corpus import wordnet as wn
+from nltk.corpus.reader.wordnet import Lemma, Synset
+
+# now included in local file
+# from util import html_header, html_trailer, \
+#    get_static_index_page, get_static_page_by_path, \
+#    page_from_word, page_from_href
+
+firstClient = True
+
+# True if we're not also running a web browser.  The value f server_mode
+# gets set by demo().
+server_mode = None
+
+# If set this is a file object for writing log messages.
+logfile = None
+
+
+class MyServerHandler(BaseHTTPRequestHandler):
+    def do_HEAD(self):
+        self.send_head()
+
+    def do_GET(self):
+        global firstClient
+        sp = self.path[1:]
+        if unquote_plus(sp) == "SHUTDOWN THE SERVER":
+            if server_mode:
+                page = "Server must be killed with SIGTERM."
+                type = "text/plain"
+            else:
+                print("Server shutting down!")
+                os._exit(0)
+
+        elif sp == "":  # First request.
+            type = "text/html"
+            if not server_mode and firstClient:
+                firstClient = False
+                page = get_static_index_page(True)
+            else:
+                page = get_static_index_page(False)
+            word = "green"
+
+        elif sp.endswith(".html"):  # Trying to fetch a HTML file TODO:
+            type = "text/html"
+            usp = unquote_plus(sp)
+            if usp == "NLTK Wordnet Browser Database Info.html":
+                word = "* Database Info *"
+                if os.path.isfile(usp):
+                    with open(usp) as infile:
+                        page = infile.read()
+                else:
+                    page = (
+                        (html_header % word) + "<p>The database info file:"
+                        "<p><b>"
+                        + usp
+                        + "</b>"
+                        + "<p>was not found. Run this:"
+                        + "<p><b>python dbinfo_html.py</b>"
+                        + "<p>to produce it."
+                        + html_trailer
+                    )
+            else:
+                # Handle files here.
+                word = sp
+                page = get_static_page_by_path(usp)
+        elif sp.startswith("search"):
+            # This doesn't seem to work with MWEs.
+            type = "text/html"
+            parts = (sp.split("?")[1]).split("&")
+            word = [
+                p.split("=")[1].replace("+", " ")
+                for p in parts
+                if p.startswith("nextWord")
+            ][0]
+            page, word = page_from_word(word)
+        elif sp.startswith("lookup_"):
+            # TODO add a variation of this that takes a non ecoded word or MWE.
+            type = "text/html"
+            sp = sp[len("lookup_") :]
+            page, word = page_from_href(sp)
+        elif sp == "start_page":
+            # if this is the first request we should display help
+            # information, and possibly set a default word.
+            type = "text/html"
+            page, word = page_from_word("wordnet")
+        else:
+            type = "text/plain"
+            page = "Could not parse request: '%s'" % sp
+
+        # Send result.
+        self.send_head(type)
+        self.wfile.write(page.encode("utf8"))
+
+    def send_head(self, type=None):
+        self.send_response(200)
+        self.send_header("Content-type", type)
+        self.end_headers()
+
+    def log_message(self, format, *args):
+        global logfile
+
+        if logfile:
+            logfile.write(
+                "%s - - [%s] %s\n"
+                % (self.address_string(), self.log_date_time_string(), format % args)
+            )
+
+
+def get_unique_counter_from_url(sp):
+    """
+    Extract the unique counter from the URL if it has one.  Otherwise return
+    null.
+    """
+    pos = sp.rfind("%23")
+    if pos != -1:
+        return int(sp[(pos + 3) :])
+    else:
+        return None
+
+
+def wnb(port=8000, runBrowser=True, logfilename=None):
+    """
+    Run NLTK Wordnet Browser Server.
+
+    :param port: The port number for the server to listen on, defaults to
+                 8000
+    :type  port: int
+
+    :param runBrowser: True to start a web browser and point it at the web
+                       server.
+    :type  runBrowser: bool
+    """
+    # The webbrowser module is unpredictable, typically it blocks if it uses
+    # a console web browser, and doesn't block if it uses a GUI webbrowser,
+    # so we need to force it to have a clear correct behaviour.
+    #
+    # Normally the server should run for as long as the user wants. they
+    # should idealy be able to control this from the UI by closing the
+    # window or tab.  Second best would be clicking a button to say
+    # 'Shutdown' that first shutsdown the server and closes the window or
+    # tab, or exits the text-mode browser.  Both of these are unfreasable.
+    #
+    # The next best alternative is to start the server, have it close when
+    # it receives SIGTERM (default), and run the browser as well.  The user
+    # may have to shutdown both programs.
+    #
+    # Since webbrowser may block, and the webserver will block, we must run
+    # them in separate threads.
+    #
+    global server_mode, logfile
+    server_mode = not runBrowser
+
+    # Setup logging.
+    if logfilename:
+        try:
+            logfile = open(logfilename, "a", 1)  # 1 means 'line buffering'
+        except OSError as e:
+            sys.stderr.write("Couldn't open %s for writing: %s", logfilename, e)
+            sys.exit(1)
+    else:
+        logfile = None
+
+    # Compute URL and start web browser
+    url = "http://localhost:" + str(port)
+
+    server_ready = None
+    browser_thread = None
+
+    if runBrowser:
+        server_ready = threading.Event()
+        browser_thread = startBrowser(url, server_ready)
+
+    # Start the server.
+    server = HTTPServer(("", port), MyServerHandler)
+    if logfile:
+        logfile.write("NLTK Wordnet browser server running serving: %s\n" % url)
+    if runBrowser:
+        server_ready.set()
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+
+    if runBrowser:
+        browser_thread.join()
+
+    if logfile:
+        logfile.close()
+
+
+def startBrowser(url, server_ready):
+    def run():
+        server_ready.wait()
+        time.sleep(1)  # Wait a little bit more, there's still the chance of
+        # a race condition.
+        webbrowser.open(url, new=2, autoraise=1)
+
+    t = threading.Thread(target=run)
+    t.start()
+    return t
+
+
+#####################################################################
+# Utilities
+#####################################################################
+
+
+"""
+WordNet Browser Utilities.
+
+This provides a backend to both wxbrowse and browserver.py.
+"""
+
+################################################################################
+#
+# Main logic for wordnet browser.
+#
+
+# This is wrapped inside a function since wn is only available if the
+# WordNet corpus is installed.
+def _pos_tuples():
+    return [
+        (wn.NOUN, "N", "noun"),
+        (wn.VERB, "V", "verb"),
+        (wn.ADJ, "J", "adj"),
+        (wn.ADV, "R", "adv"),
+    ]
+
+
+def _pos_match(pos_tuple):
+    """
+    This function returns the complete pos tuple for the partial pos
+    tuple given to it.  It attempts to match it against the first
+    non-null component of the given pos tuple.
+    """
+    if pos_tuple[0] == "s":
+        pos_tuple = ("a", pos_tuple[1], pos_tuple[2])
+    for n, x in enumerate(pos_tuple):
+        if x is not None:
+            break
+    for pt in _pos_tuples():
+        if pt[n] == pos_tuple[n]:
+            return pt
+    return None
+
+
+HYPONYM = 0
+HYPERNYM = 1
+CLASS_REGIONAL = 2
+PART_HOLONYM = 3
+PART_MERONYM = 4
+ATTRIBUTE = 5
+SUBSTANCE_HOLONYM = 6
+SUBSTANCE_MERONYM = 7
+MEMBER_HOLONYM = 8
+MEMBER_MERONYM = 9
+VERB_GROUP = 10
+INSTANCE_HYPONYM = 12
+INSTANCE_HYPERNYM = 13
+CAUSE = 14
+ALSO_SEE = 15
+SIMILAR = 16
+ENTAILMENT = 17
+ANTONYM = 18
+FRAMES = 19
+PERTAINYM = 20
+
+CLASS_CATEGORY = 21
+CLASS_USAGE = 22
+CLASS_REGIONAL = 23
+CLASS_USAGE = 24
+CLASS_CATEGORY = 11
+
+DERIVATIONALLY_RELATED_FORM = 25
+
+INDIRECT_HYPERNYMS = 26
+
+
+def lemma_property(word, synset, func):
+    def flattern(l):
+        if l == []:
+            return []
+        else:
+            return l[0] + flattern(l[1:])
+
+    return flattern([func(l) for l in synset.lemmas() if l.name == word])
+
+
+def rebuild_tree(orig_tree):
+    node = orig_tree[0]
+    children = orig_tree[1:]
+    return (node, [rebuild_tree(t) for t in children])
+
+
+def get_relations_data(word, synset):
+    """
+    Get synset relations data for a synset.  Note that this doesn't
+    yet support things such as full hyponym vs direct hyponym.
+    """
+    if synset.pos() == wn.NOUN:
+        return (
+            (HYPONYM, "Hyponyms", synset.hyponyms()),
+            (INSTANCE_HYPONYM, "Instance hyponyms", synset.instance_hyponyms()),
+            (HYPERNYM, "Direct hypernyms", synset.hypernyms()),
+            (
+                INDIRECT_HYPERNYMS,
+                "Indirect hypernyms",
+                rebuild_tree(synset.tree(lambda x: x.hypernyms()))[1],
+            ),
+            #  hypernyms', 'Sister terms',
+            (INSTANCE_HYPERNYM, "Instance hypernyms", synset.instance_hypernyms()),
+            #            (CLASS_REGIONAL, ['domain term region'], ),
+            (PART_HOLONYM, "Part holonyms", synset.part_holonyms()),
+            (PART_MERONYM, "Part meronyms", synset.part_meronyms()),
+            (SUBSTANCE_HOLONYM, "Substance holonyms", synset.substance_holonyms()),
+            (SUBSTANCE_MERONYM, "Substance meronyms", synset.substance_meronyms()),
+            (MEMBER_HOLONYM, "Member holonyms", synset.member_holonyms()),
+            (MEMBER_MERONYM, "Member meronyms", synset.member_meronyms()),
+            (ATTRIBUTE, "Attributes", synset.attributes()),
+            (ANTONYM, "Antonyms", lemma_property(word, synset, lambda l: l.antonyms())),
+            (
+                DERIVATIONALLY_RELATED_FORM,
+                "Derivationally related form",
+                lemma_property(
+                    word, synset, lambda l: l.derivationally_related_forms()
+                ),
+            ),
+        )
+    elif synset.pos() == wn.VERB:
+        return (
+            (ANTONYM, "Antonym", lemma_property(word, synset, lambda l: l.antonyms())),
+            (HYPONYM, "Hyponym", synset.hyponyms()),
+            (HYPERNYM, "Direct hypernyms", synset.hypernyms()),
+            (
+                INDIRECT_HYPERNYMS,
+                "Indirect hypernyms",
+                rebuild_tree(synset.tree(lambda x: x.hypernyms()))[1],
+            ),
+            (ENTAILMENT, "Entailments", synset.entailments()),
+            (CAUSE, "Causes", synset.causes()),
+            (ALSO_SEE, "Also see", synset.also_sees()),
+            (VERB_GROUP, "Verb Groups", synset.verb_groups()),
+            (
+                DERIVATIONALLY_RELATED_FORM,
+                "Derivationally related form",
+                lemma_property(
+                    word, synset, lambda l: l.derivationally_related_forms()
+                ),
+            ),
+        )
+    elif synset.pos() == wn.ADJ or synset.pos == wn.ADJ_SAT:
+        return (
+            (ANTONYM, "Antonym", lemma_property(word, synset, lambda l: l.antonyms())),
+            (SIMILAR, "Similar to", synset.similar_tos()),
+            # Participle of verb - not supported by corpus
+            (
+                PERTAINYM,
+                "Pertainyms",
+                lemma_property(word, synset, lambda l: l.pertainyms()),
+            ),
+            (ATTRIBUTE, "Attributes", synset.attributes()),
+            (ALSO_SEE, "Also see", synset.also_sees()),
+        )
+    elif synset.pos() == wn.ADV:
+        # This is weird. adverbs such as 'quick' and 'fast' don't seem
+        # to have antonyms returned by the corpus.a
+        return (
+            (ANTONYM, "Antonym", lemma_property(word, synset, lambda l: l.antonyms())),
+        )
+        # Derived from adjective - not supported by corpus
+    else:
+        raise TypeError("Unhandles synset POS type: " + str(synset.pos()))
+
+
+html_header = """
+<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'
+'http://www.w3.org/TR/html4/strict.dtd'>
+<html>
+<head>
+<meta name='generator' content=
+'HTML Tidy for Windows (vers 14 February 2006), see www.w3.org'>
+<meta http-equiv='Content-Type' content=
+'text/html; charset=us-ascii'>
+<title>NLTK Wordnet Browser display of: %s</title></head>
+<body bgcolor='#F5F5F5' text='#000000'>
+"""
+html_trailer = """
+</body>
+</html>
+"""
+
+explanation = """
+<h3>Search Help</h3>
+<ul><li>The display below the line is an example of the output the browser
+shows you when you enter a search word. The search word was <b>green</b>.</li>
+<li>The search result shows for different parts of speech the <b>synsets</b>
+i.e. different meanings for the word.</li>
+<li>All underlined texts are hypertext links. There are two types of links:
+word links and others. Clicking a word link carries out a search for the word
+in the Wordnet database.</li>
+<li>Clicking a link of the other type opens a display section of data attached
+to that link. Clicking that link a second time closes the section again.</li>
+<li>Clicking <u>S:</u> opens a section showing the relations for that synset.
+</li>
+<li>Clicking on a relation name opens a section that displays the associated
+synsets.</li>
+<li>Type a search word in the <b>Word</b> field and start the search by the
+<b>Enter/Return</b> key or click the <b>Search</b> button.</li>
+</ul>
+<hr width='100%'>
+"""
+
+# HTML oriented functions
+
+
+def _bold(txt):
+    return "<b>%s</b>" % txt
+
+
+def _center(txt):
+    return "<center>%s</center>" % txt
+
+
+def _hlev(n, txt):
+    return "<h%d>%s</h%d>" % (n, txt, n)
+
+
+def _italic(txt):
+    return "<i>%s</i>" % txt
+
+
+def _li(txt):
+    return "<li>%s</li>" % txt
+
+
+def pg(word, body):
+    """
+    Return a HTML page of NLTK Browser format constructed from the
+    word and body
+
+    :param word: The word that the body corresponds to
+    :type word: str
+    :param body: The HTML body corresponding to the word
+    :type body: str
+    :return: a HTML page for the word-body combination
+    :rtype: str
+    """
+    return (html_header % word) + body + html_trailer
+
+
+def _ul(txt):
+    return "<ul>" + txt + "</ul>"
+
+
+def _abbc(txt):
+    """
+    abbc = asterisks, breaks, bold, center
+    """
+    return _center(_bold("<br>" * 10 + "*" * 10 + " " + txt + " " + "*" * 10))
+
+
+full_hyponym_cont_text = _ul(_li(_italic("(has full hyponym continuation)"))) + "\n"
+
+
+def _get_synset(synset_key):
+    """
+    The synset key is the unique name of the synset, this can be
+    retrieved via synset.name()
+    """
+    return wn.synset(synset_key)
+
+
+def _collect_one_synset(word, synset, synset_relations):
+    """
+    Returns the HTML string for one synset or word
+
+    :param word: the current word
+    :type word: str
+    :param synset: a synset
+    :type synset: synset
+    :param synset_relations: information about which synset relations
+    to display.
+    :type synset_relations: dict(synset_key, set(relation_id))
+    :return: The HTML string built for this synset
+    :rtype: str
+    """
+    if isinstance(synset, tuple):  # It's a word
+        raise NotImplementedError("word not supported by _collect_one_synset")
+
+    typ = "S"
+    pos_tuple = _pos_match((synset.pos(), None, None))
+    assert pos_tuple is not None, "pos_tuple is null: synset.pos(): %s" % synset.pos()
+    descr = pos_tuple[2]
+    ref = copy.deepcopy(Reference(word, synset_relations))
+    ref.toggle_synset(synset)
+    synset_label = typ + ";"
+    if synset.name() in synset_relations:
+        synset_label = _bold(synset_label)
+    s = f"<li>{make_lookup_link(ref, synset_label)} ({descr}) "
+
+    def format_lemma(w):
+        w = w.replace("_", " ")
+        if w.lower() == word:
+            return _bold(w)
+        else:
+            ref = Reference(w)
+            return make_lookup_link(ref, w)
+
+    s += ", ".join(format_lemma(l.name()) for l in synset.lemmas())
+
+    gl = " ({}) <i>{}</i> ".format(
+        synset.definition(),
+        "; ".join('"%s"' % e for e in synset.examples()),
+    )
+    return s + gl + _synset_relations(word, synset, synset_relations) + "</li>\n"
+
+
+def _collect_all_synsets(word, pos, synset_relations=dict()):
+    """
+    Return a HTML unordered list of synsets for the given word and
+    part of speech.
+    """
+    return "<ul>%s\n</ul>\n" % "".join(
+        _collect_one_synset(word, synset, synset_relations)
+        for synset in wn.synsets(word, pos)
+    )
+
+
+def _synset_relations(word, synset, synset_relations):
+    """
+    Builds the HTML string for the relations of a synset
+
+    :param word: The current word
+    :type word: str
+    :param synset: The synset for which we're building the relations.
+    :type synset: Synset
+    :param synset_relations: synset keys and relation types for which to display relations.
+    :type synset_relations: dict(synset_key, set(relation_type))
+    :return: The HTML for a synset's relations
+    :rtype: str
+    """
+
+    if not synset.name() in synset_relations:
+        return ""
+    ref = Reference(word, synset_relations)
+
+    def relation_html(r):
+        if isinstance(r, Synset):
+            return make_lookup_link(Reference(r.lemma_names()[0]), r.lemma_names()[0])
+        elif isinstance(r, Lemma):
+            return relation_html(r.synset())
+        elif isinstance(r, tuple):
+            # It's probably a tuple containing a Synset and a list of
+            # similar tuples.  This forms a tree of synsets.
+            return "{}\n<ul>{}</ul>\n".format(
+                relation_html(r[0]),
+                "".join("<li>%s</li>\n" % relation_html(sr) for sr in r[1]),
+            )
+        else:
+            raise TypeError(
+                "r must be a synset, lemma or list, it was: type(r) = %s, r = %s"
+                % (type(r), r)
+            )
+
+    def make_synset_html(db_name, disp_name, rels):
+        synset_html = "<i>%s</i>\n" % make_lookup_link(
+            copy.deepcopy(ref).toggle_synset_relation(synset, db_name),
+            disp_name,
+        )
+
+        if db_name in ref.synset_relations[synset.name()]:
+            synset_html += "<ul>%s</ul>\n" % "".join(
+                "<li>%s</li>\n" % relation_html(r) for r in rels
+            )
+
+        return synset_html
+
+    html = (
+        "<ul>"
+        + "\n".join(
+            "<li>%s</li>" % make_synset_html(*rel_data)
+            for rel_data in get_relations_data(word, synset)
+            if rel_data[2] != []
+        )
+        + "</ul>"
+    )
+
+    return html
+
+
+class Reference:
+    """
+    A reference to a page that may be generated by page_word
+    """
+
+    def __init__(self, word, synset_relations=dict()):
+        """
+        Build a reference to a new page.
+
+        word is the word or words (separated by commas) for which to
+        search for synsets of
+
+        synset_relations is a dictionary of synset keys to sets of
+        synset relation identifaiers to unfold a list of synset
+        relations for.
+        """
+        self.word = word
+        self.synset_relations = synset_relations
+
+    def encode(self):
+        """
+        Encode this reference into a string to be used in a URL.
+        """
+        # This uses a tuple rather than an object since the python
+        # pickle representation is much smaller and there is no need
+        # to represent the complete object.
+        string = pickle.dumps((self.word, self.synset_relations), -1)
+        return base64.urlsafe_b64encode(string).decode()
+
+    @staticmethod
+    def decode(string):
+        """
+        Decode a reference encoded with Reference.encode
+        """
+        string = base64.urlsafe_b64decode(string.encode())
+        word, synset_relations = pickle.loads(string)
+        return Reference(word, synset_relations)
+
+    def toggle_synset_relation(self, synset, relation):
+        """
+        Toggle the display of the relations for the given synset and
+        relation type.
+
+        This function will throw a KeyError if the synset is currently
+        not being displayed.
+        """
+        if relation in self.synset_relations[synset.name()]:
+            self.synset_relations[synset.name()].remove(relation)
+        else:
+            self.synset_relations[synset.name()].add(relation)
+
+        return self
+
+    def toggle_synset(self, synset):
+        """
+        Toggle displaying of the relation types for the given synset
+        """
+        if synset.name() in self.synset_relations:
+            del self.synset_relations[synset.name()]
+        else:
+            self.synset_relations[synset.name()] = set()
+
+        return self
+
+
+def make_lookup_link(ref, label):
+    return f'<a href="lookup_{ref.encode()}">{label}</a>'
+
+
+def page_from_word(word):
+    """
+    Return a HTML page for the given word.
+
+    :type word: str
+    :param word: The currently active word
+    :return: A tuple (page,word), where page is the new current HTML page
+        to be sent to the browser and
+        word is the new current word
+    :rtype: A tuple (str,str)
+    """
+    return page_from_reference(Reference(word))
+
+
+def page_from_href(href):
+    """
+    Returns a tuple of the HTML page built and the new current word
+
+    :param href: The hypertext reference to be solved
+    :type href: str
+    :return: A tuple (page,word), where page is the new current HTML page
+             to be sent to the browser and
+             word is the new current word
+    :rtype: A tuple (str,str)
+    """
+    return page_from_reference(Reference.decode(href))
+
+
+def page_from_reference(href):
+    """
+    Returns a tuple of the HTML page built and the new current word
+
+    :param href: The hypertext reference to be solved
+    :type href: str
+    :return: A tuple (page,word), where page is the new current HTML page
+             to be sent to the browser and
+             word is the new current word
+    :rtype: A tuple (str,str)
+    """
+    word = href.word
+    pos_forms = defaultdict(list)
+    words = word.split(",")
+    words = [w for w in [w.strip().lower().replace(" ", "_") for w in words] if w != ""]
+    if len(words) == 0:
+        # No words were found.
+        return "", "Please specify a word to search for."
+
+    # This looks up multiple words at once.  This is probably not
+    # necessary and may lead to problems.
+    for w in words:
+        for pos in [wn.NOUN, wn.VERB, wn.ADJ, wn.ADV]:
+            form = wn.morphy(w, pos)
+            if form and form not in pos_forms[pos]:
+                pos_forms[pos].append(form)
+    body = ""
+    for pos, pos_str, name in _pos_tuples():
+        if pos in pos_forms:
+            body += _hlev(3, name) + "\n"
+            for w in pos_forms[pos]:
+                # Not all words of exc files are in the database, skip
+                # to the next word if a KeyError is raised.
+                try:
+                    body += _collect_all_synsets(w, pos, href.synset_relations)
+                except KeyError:
+                    pass
+    if not body:
+        body = "The word or words '%s' where not found in the dictionary." % word
+    return body, word
+
+
+#####################################################################
+# Static pages
+#####################################################################
+
+
+def get_static_page_by_path(path):
+    """
+    Return a static HTML page from the path given.
+    """
+    if path == "index_2.html":
+        return get_static_index_page(False)
+    elif path == "index.html":
+        return get_static_index_page(True)
+    elif path == "NLTK Wordnet Browser Database Info.html":
+        return "Display of Wordnet Database Statistics is not supported"
+    elif path == "upper_2.html":
+        return get_static_upper_page(False)
+    elif path == "upper.html":
+        return get_static_upper_page(True)
+    elif path == "web_help.html":
+        return get_static_web_help_page()
+    elif path == "wx_help.html":
+        return get_static_wx_help_page()
+    else:
+        return "Internal error: Path for static page '%s' is unknown" % path
+
+
+def get_static_web_help_page():
+    """
+    Return the static web help page.
+    """
+    return """
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+     <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+            Copyright (C) 2001-2022 NLTK Project
+            Author: Jussi Salmela <jtsalmela@users.sourceforge.net>
+            URL: <https://www.nltk.org/>
+            For license information, see LICENSE.TXT -->
+     <head>
+          <meta http-equiv='Content-Type' content='text/html; charset=us-ascii'>
+          <title>NLTK Wordnet Browser display of: * Help *</title>
+     </head>
+<body bgcolor='#F5F5F5' text='#000000'>
+<h2>NLTK Wordnet Browser Help</h2>
+<p>The NLTK Wordnet Browser is a tool to use in browsing the Wordnet database. It tries to behave like the Wordnet project's web browser but the difference is that the NLTK Wordnet Browser uses a local Wordnet database.
+<p><b>You are using the Javascript client part of the NLTK Wordnet BrowseServer.</b> We assume your browser is in tab sheets enabled mode.</p>
+<p>For background information on Wordnet, see the Wordnet project home page: <a href="https://wordnet.princeton.edu/"><b> https://wordnet.princeton.edu/</b></a>. For more information on the NLTK project, see the project home:
+<a href="https://www.nltk.org/"><b>https://www.nltk.org/</b></a>. To get an idea of what the Wordnet version used by this browser includes choose <b>Show Database Info</b> from the <b>View</b> submenu.</p>
+<h3>Word search</h3>
+<p>The word to be searched is typed into the <b>New Word</b> field and the search started with Enter or by clicking the <b>Search</b> button. There is no uppercase/lowercase distinction: the search word is transformed to lowercase before the search.</p>
+<p>In addition, the word does not have to be in base form. The browser tries to find the possible base form(s) by making certain morphological substitutions. Typing <b>fLIeS</b> as an obscure example gives one <a href="MfLIeS">this</a>. Click the previous link to see what this kind of search looks like and then come back to this page by using the <b>Alt+LeftArrow</b> key combination.</p>
+<p>The result of a search is a display of one or more
+<b>synsets</b> for every part of speech in which a form of the
+search word was found to occur. A synset is a set of words
+having the same sense or meaning. Each word in a synset that is
+underlined is a hyperlink which can be clicked to trigger an
+automatic search for that word.</p>
+<p>Every synset has a hyperlink <b>S:</b> at the start of its
+display line. Clicking that symbol shows you the name of every
+<b>relation</b> that this synset is part of. Every relation name is a hyperlink that opens up a display for that relation. Clicking it another time closes the display again. Clicking another relation name on a line that has an opened relation closes the open relation and opens the clicked relation.</p>
+<p>It is also possible to give two or more words or collocations to be searched at the same time separating them with a comma like this <a href="Mcheer up,clear up">cheer up,clear up</a>, for example. Click the previous link to see what this kind of search looks like and then come back to this page by using the <b>Alt+LeftArrow</b> key combination. As you could see the search result includes the synsets found in the same order than the forms were given in the search field.</p>
+<p>
+There are also word level (lexical) relations recorded in the Wordnet database. Opening this kind of relation displays lines with a hyperlink <b>W:</b> at their beginning. Clicking this link shows more info on the word in question.</p>
+<h3>The Buttons</h3>
+<p>The <b>Search</b> and <b>Help</b> buttons need no more explanation. </p>
+<p>The <b>Show Database Info</b> button shows a collection of Wordnet database statistics.</p>
+<p>The <b>Shutdown the Server</b> button is shown for the first client of the BrowServer program i.e. for the client that is automatically launched when the BrowServer is started but not for the succeeding clients in order to protect the server from accidental shutdowns.
+</p></body>
+</html>
+"""
+
+
+def get_static_welcome_message():
+    """
+    Get the static welcome page.
+    """
+    return """
+<h3>Search Help</h3>
+<ul><li>The display below the line is an example of the output the browser
+shows you when you enter a search word. The search word was <b>green</b>.</li>
+<li>The search result shows for different parts of speech the <b>synsets</b>
+i.e. different meanings for the word.</li>
+<li>All underlined texts are hypertext links. There are two types of links:
+word links and others. Clicking a word link carries out a search for the word
+in the Wordnet database.</li>
+<li>Clicking a link of the other type opens a display section of data attached
+to that link. Clicking that link a second time closes the section again.</li>
+<li>Clicking <u>S:</u> opens a section showing the relations for that synset.</li>
+<li>Clicking on a relation name opens a section that displays the associated
+synsets.</li>
+<li>Type a search word in the <b>Next Word</b> field and start the search by the
+<b>Enter/Return</b> key or click the <b>Search</b> button.</li>
+</ul>
+"""
+
+
+def get_static_index_page(with_shutdown):
+    """
+    Get the static index page.
+    """
+    template = """
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN"  "http://www.w3.org/TR/html4/frameset.dtd">
+<HTML>
+     <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+            Copyright (C) 2001-2022 NLTK Project
+            Author: Jussi Salmela <jtsalmela@users.sourceforge.net>
+            URL: <https://www.nltk.org/>
+            For license information, see LICENSE.TXT -->
+     <HEAD>
+         <TITLE>NLTK Wordnet Browser</TITLE>
+     </HEAD>
+
+<frameset rows="7%%,93%%">
+    <frame src="%s" name="header">
+    <frame src="start_page" name="body">
+</frameset>
+</HTML>
+"""
+    if with_shutdown:
+        upper_link = "upper.html"
+    else:
+        upper_link = "upper_2.html"
+
+    return template % upper_link
+
+
+def get_static_upper_page(with_shutdown):
+    """
+    Return the upper frame page,
+
+    If with_shutdown is True then a 'shutdown' button is also provided
+    to shutdown the server.
+    """
+    template = """
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+    <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+        Copyright (C) 2001-2022 NLTK Project
+        Author: Jussi Salmela <jtsalmela@users.sourceforge.net>
+        URL: <https://www.nltk.org/>
+        For license information, see LICENSE.TXT -->
+    <head>
+                <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+        <title>Untitled Document</title>
+    </head>
+    <body>
+    <form method="GET" action="search" target="body">
+            Current Word:&nbsp;<input type="text" id="currentWord" size="10" disabled>
+            Next Word:&nbsp;<input type="text" id="nextWord" name="nextWord" size="10">
+            <input name="searchButton" type="submit" value="Search">
+    </form>
+        <a target="body" href="web_help.html">Help</a>
+        %s
+
+</body>
+</html>
+"""
+    if with_shutdown:
+        shutdown_link = '<a href="SHUTDOWN THE SERVER">Shutdown</a>'
+    else:
+        shutdown_link = ""
+
+    return template % shutdown_link
+
+
+def usage():
+    """
+    Display the command line help message.
+    """
+    print(__doc__)
+
+
+def app():
+    # Parse and interpret options.
+    (opts, _) = getopt.getopt(
+        argv[1:], "l:p:sh", ["logfile=", "port=", "server-mode", "help"]
+    )
+    port = 8000
+    server_mode = False
+    help_mode = False
+    logfilename = None
+    for (opt, value) in opts:
+        if (opt == "-l") or (opt == "--logfile"):
+            logfilename = str(value)
+        elif (opt == "-p") or (opt == "--port"):
+            port = int(value)
+        elif (opt == "-s") or (opt == "--server-mode"):
+            server_mode = True
+        elif (opt == "-h") or (opt == "--help"):
+            help_mode = True
+
+    if help_mode:
+        usage()
+    else:
+        wnb(port, not server_mode, logfilename)
+
+
+if __name__ == "__main__":
+    app()
+
+__all__ = ["app"]

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/__init__.py

@@ -0,0 +1,34 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Graeme Gange <ggange@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Combinatory Categorial Grammar.
+
+For more information see nltk/doc/contrib/ccg/ccg.pdf
+"""
+
+from nltk.ccg.chart import CCGChart, CCGChartParser, CCGEdge, CCGLeafEdge
+from nltk.ccg.combinator import (
+    BackwardApplication,
+    BackwardBx,
+    BackwardCombinator,
+    BackwardComposition,
+    BackwardSx,
+    BackwardT,
+    DirectedBinaryCombinator,
+    ForwardApplication,
+    ForwardCombinator,
+    ForwardComposition,
+    ForwardSubstitution,
+    ForwardT,
+    UndirectedBinaryCombinator,
+    UndirectedComposition,
+    UndirectedFunctionApplication,
+    UndirectedSubstitution,
+    UndirectedTypeRaise,
+)
+from nltk.ccg.lexicon import CCGLexicon

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/api.py

@@ -0,0 +1,358 @@
+# Natural Language Toolkit: CCG Categories
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Graeme Gange <ggange@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from abc import ABCMeta, abstractmethod
+from functools import total_ordering
+
+from nltk.internals import raise_unorderable_types
+
+
+@total_ordering
+class AbstractCCGCategory(metaclass=ABCMeta):
+    """
+    Interface for categories in combinatory grammars.
+    """
+
+    @abstractmethod
+    def is_primitive(self):
+        """
+        Returns true if the category is primitive.
+        """
+
+    @abstractmethod
+    def is_function(self):
+        """
+        Returns true if the category is a function application.
+        """
+
+    @abstractmethod
+    def is_var(self):
+        """
+        Returns true if the category is a variable.
+        """
+
+    @abstractmethod
+    def substitute(self, substitutions):
+        """
+        Takes a set of (var, category) substitutions, and replaces every
+        occurrence of the variable with the corresponding category.
+        """
+
+    @abstractmethod
+    def can_unify(self, other):
+        """
+        Determines whether two categories can be unified.
+         - Returns None if they cannot be unified
+         - Returns a list of necessary substitutions if they can.
+        """
+
+    # Utility functions: comparison, strings and hashing.
+    @abstractmethod
+    def __str__(self):
+        pass
+
+    def __eq__(self, other):
+        return (
+            self.__class__ is other.__class__
+            and self._comparison_key == other._comparison_key
+        )
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __lt__(self, other):
+        if not isinstance(other, AbstractCCGCategory):
+            raise_unorderable_types("<", self, other)
+        if self.__class__ is other.__class__:
+            return self._comparison_key < other._comparison_key
+        else:
+            return self.__class__.__name__ < other.__class__.__name__
+
+    def __hash__(self):
+        try:
+            return self._hash
+        except AttributeError:
+            self._hash = hash(self._comparison_key)
+            return self._hash
+
+
+class CCGVar(AbstractCCGCategory):
+    """
+    Class representing a variable CCG category.
+    Used for conjunctions (and possibly type-raising, if implemented as a
+    unary rule).
+    """
+
+    _maxID = 0
+
+    def __init__(self, prim_only=False):
+        """Initialize a variable (selects a new identifier)
+
+        :param prim_only: a boolean that determines whether the variable is
+                          restricted to primitives
+        :type prim_only: bool
+        """
+        self._id = self.new_id()
+        self._prim_only = prim_only
+        self._comparison_key = self._id
+
+    @classmethod
+    def new_id(cls):
+        """
+        A class method allowing generation of unique variable identifiers.
+        """
+        cls._maxID = cls._maxID + 1
+        return cls._maxID - 1
+
+    @classmethod
+    def reset_id(cls):
+        cls._maxID = 0
+
+    def is_primitive(self):
+        return False
+
+    def is_function(self):
+        return False
+
+    def is_var(self):
+        return True
+
+    def substitute(self, substitutions):
+        """If there is a substitution corresponding to this variable,
+        return the substituted category.
+        """
+        for (var, cat) in substitutions:
+            if var == self:
+                return cat
+        return self
+
+    def can_unify(self, other):
+        """If the variable can be replaced with other
+        a substitution is returned.
+        """
+        if other.is_primitive() or not self._prim_only:
+            return [(self, other)]
+        return None
+
+    def id(self):
+        return self._id
+
+    def __str__(self):
+        return "_var" + str(self._id)
+
+
+@total_ordering
+class Direction:
+    """
+    Class representing the direction of a function application.
+    Also contains maintains information as to which combinators
+    may be used with the category.
+    """
+
+    def __init__(self, dir, restrictions):
+        self._dir = dir
+        self._restrs = restrictions
+        self._comparison_key = (dir, tuple(restrictions))
+
+    # Testing the application direction
+    def is_forward(self):
+        return self._dir == "/"
+
+    def is_backward(self):
+        return self._dir == "\\"
+
+    def dir(self):
+        return self._dir
+
+    def restrs(self):
+        """A list of restrictions on the combinators.
+        '.' denotes that permuting operations are disallowed
+        ',' denotes that function composition is disallowed
+        '_' denotes that the direction has variable restrictions.
+        (This is redundant in the current implementation of type-raising)
+        """
+        return self._restrs
+
+    def is_variable(self):
+        return self._restrs == "_"
+
+    # Unification and substitution of variable directions.
+    # Used only if type-raising is implemented as a unary rule, as it
+    # must inherit restrictions from the argument category.
+    def can_unify(self, other):
+        if other.is_variable():
+            return [("_", self.restrs())]
+        elif self.is_variable():
+            return [("_", other.restrs())]
+        else:
+            if self.restrs() == other.restrs():
+                return []
+        return None
+
+    def substitute(self, subs):
+        if not self.is_variable():
+            return self
+
+        for (var, restrs) in subs:
+            if var == "_":
+                return Direction(self._dir, restrs)
+        return self
+
+    # Testing permitted combinators
+    def can_compose(self):
+        return "," not in self._restrs
+
+    def can_cross(self):
+        return "." not in self._restrs
+
+    def __eq__(self, other):
+        return (
+            self.__class__ is other.__class__
+            and self._comparison_key == other._comparison_key
+        )
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __lt__(self, other):
+        if not isinstance(other, Direction):
+            raise_unorderable_types("<", self, other)
+        if self.__class__ is other.__class__:
+            return self._comparison_key < other._comparison_key
+        else:
+            return self.__class__.__name__ < other.__class__.__name__
+
+    def __hash__(self):
+        try:
+            return self._hash
+        except AttributeError:
+            self._hash = hash(self._comparison_key)
+            return self._hash
+
+    def __str__(self):
+        r_str = ""
+        for r in self._restrs:
+            r_str = r_str + "%s" % r
+        return f"{self._dir}{r_str}"
+
+    # The negation operator reverses the direction of the application
+    def __neg__(self):
+        if self._dir == "/":
+            return Direction("\\", self._restrs)
+        else:
+            return Direction("/", self._restrs)
+
+
+class PrimitiveCategory(AbstractCCGCategory):
+    """
+    Class representing primitive categories.
+    Takes a string representation of the category, and a
+    list of strings specifying the morphological subcategories.
+    """
+
+    def __init__(self, categ, restrictions=[]):
+        self._categ = categ
+        self._restrs = restrictions
+        self._comparison_key = (categ, tuple(restrictions))
+
+    def is_primitive(self):
+        return True
+
+    def is_function(self):
+        return False
+
+    def is_var(self):
+        return False
+
+    def restrs(self):
+        return self._restrs
+
+    def categ(self):
+        return self._categ
+
+    # Substitution does nothing to a primitive category
+    def substitute(self, subs):
+        return self
+
+    # A primitive can be unified with a class of the same
+    # base category, given that the other category shares all
+    # of its subclasses, or with a variable.
+    def can_unify(self, other):
+        if not other.is_primitive():
+            return None
+        if other.is_var():
+            return [(other, self)]
+        if other.categ() == self.categ():
+            for restr in self._restrs:
+                if restr not in other.restrs():
+                    return None
+            return []
+        return None
+
+    def __str__(self):
+        if self._restrs == []:
+            return "%s" % self._categ
+        restrictions = "[%s]" % ",".join(repr(r) for r in self._restrs)
+        return f"{self._categ}{restrictions}"
+
+
+class FunctionalCategory(AbstractCCGCategory):
+    """
+    Class that represents a function application category.
+    Consists of argument and result categories, together with
+    an application direction.
+    """
+
+    def __init__(self, res, arg, dir):
+        self._res = res
+        self._arg = arg
+        self._dir = dir
+        self._comparison_key = (arg, dir, res)
+
+    def is_primitive(self):
+        return False
+
+    def is_function(self):
+        return True
+
+    def is_var(self):
+        return False
+
+    # Substitution returns the category consisting of the
+    # substitution applied to each of its constituents.
+    def substitute(self, subs):
+        sub_res = self._res.substitute(subs)
+        sub_dir = self._dir.substitute(subs)
+        sub_arg = self._arg.substitute(subs)
+        return FunctionalCategory(sub_res, sub_arg, self._dir)
+
+    # A function can unify with another function, so long as its
+    # constituents can unify, or with an unrestricted variable.
+    def can_unify(self, other):
+        if other.is_var():
+            return [(other, self)]
+        if other.is_function():
+            sa = self._res.can_unify(other.res())
+            sd = self._dir.can_unify(other.dir())
+            if sa is not None and sd is not None:
+                sb = self._arg.substitute(sa).can_unify(other.arg().substitute(sa))
+                if sb is not None:
+                    return sa + sb
+        return None
+
+    # Constituent accessors
+    def arg(self):
+        return self._arg
+
+    def res(self):
+        return self._res
+
+    def dir(self):
+        return self._dir
+
+    def __str__(self):
+        return f"({self._res}{self._dir}{self._arg})"

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/chart.py

@@ -0,0 +1,480 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Graeme Gange <ggange@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The lexicon is constructed by calling
+``lexicon.fromstring(<lexicon string>)``.
+
+In order to construct a parser, you also need a rule set.
+The standard English rules are provided in chart as
+``chart.DefaultRuleSet``.
+
+The parser can then be constructed by calling, for example:
+``parser = chart.CCGChartParser(<lexicon>, <ruleset>)``
+
+Parsing is then performed by running
+``parser.parse(<sentence>.split())``.
+
+While this returns a list of trees, the default representation
+of the produced trees is not very enlightening, particularly
+given that it uses the same tree class as the CFG parsers.
+It is probably better to call:
+``chart.printCCGDerivation(<parse tree extracted from list>)``
+which should print a nice representation of the derivation.
+
+This entire process is shown far more clearly in the demonstration:
+python chart.py
+"""
+
+import itertools
+
+from nltk.ccg.combinator import *
+from nltk.ccg.combinator import (
+    BackwardApplication,
+    BackwardBx,
+    BackwardComposition,
+    BackwardSx,
+    BackwardT,
+    ForwardApplication,
+    ForwardComposition,
+    ForwardSubstitution,
+    ForwardT,
+)
+from nltk.ccg.lexicon import Token, fromstring
+from nltk.ccg.logic import *
+from nltk.parse import ParserI
+from nltk.parse.chart import AbstractChartRule, Chart, EdgeI
+from nltk.sem.logic import *
+from nltk.tree import Tree
+
+
+# Based on the EdgeI class from NLTK.
+# A number of the properties of the EdgeI interface don't
+# transfer well to CCGs, however.
+class CCGEdge(EdgeI):
+    def __init__(self, span, categ, rule):
+        self._span = span
+        self._categ = categ
+        self._rule = rule
+        self._comparison_key = (span, categ, rule)
+
+    # Accessors
+    def lhs(self):
+        return self._categ
+
+    def span(self):
+        return self._span
+
+    def start(self):
+        return self._span[0]
+
+    def end(self):
+        return self._span[1]
+
+    def length(self):
+        return self._span[1] - self.span[0]
+
+    def rhs(self):
+        return ()
+
+    def dot(self):
+        return 0
+
+    def is_complete(self):
+        return True
+
+    def is_incomplete(self):
+        return False
+
+    def nextsym(self):
+        return None
+
+    def categ(self):
+        return self._categ
+
+    def rule(self):
+        return self._rule
+
+
+class CCGLeafEdge(EdgeI):
+    """
+    Class representing leaf edges in a CCG derivation.
+    """
+
+    def __init__(self, pos, token, leaf):
+        self._pos = pos
+        self._token = token
+        self._leaf = leaf
+        self._comparison_key = (pos, token.categ(), leaf)
+
+    # Accessors
+    def lhs(self):
+        return self._token.categ()
+
+    def span(self):
+        return (self._pos, self._pos + 1)
+
+    def start(self):
+        return self._pos
+
+    def end(self):
+        return self._pos + 1
+
+    def length(self):
+        return 1
+
+    def rhs(self):
+        return self._leaf
+
+    def dot(self):
+        return 0
+
+    def is_complete(self):
+        return True
+
+    def is_incomplete(self):
+        return False
+
+    def nextsym(self):
+        return None
+
+    def token(self):
+        return self._token
+
+    def categ(self):
+        return self._token.categ()
+
+    def leaf(self):
+        return self._leaf
+
+
+class BinaryCombinatorRule(AbstractChartRule):
+    """
+    Class implementing application of a binary combinator to a chart.
+    Takes the directed combinator to apply.
+    """
+
+    NUMEDGES = 2
+
+    def __init__(self, combinator):
+        self._combinator = combinator
+
+    # Apply a combinator
+    def apply(self, chart, grammar, left_edge, right_edge):
+        # The left & right edges must be touching.
+        if not (left_edge.end() == right_edge.start()):
+            return
+
+        # Check if the two edges are permitted to combine.
+        # If so, generate the corresponding edge.
+        if self._combinator.can_combine(left_edge.categ(), right_edge.categ()):
+            for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+                new_edge = CCGEdge(
+                    span=(left_edge.start(), right_edge.end()),
+                    categ=res,
+                    rule=self._combinator,
+                )
+                if chart.insert(new_edge, (left_edge, right_edge)):
+                    yield new_edge
+
+    # The representation of the combinator (for printing derivations)
+    def __str__(self):
+        return "%s" % self._combinator
+
+
+# Type-raising must be handled slightly differently to the other rules, as the
+# resulting rules only span a single edge, rather than both edges.
+
+
+class ForwardTypeRaiseRule(AbstractChartRule):
+    """
+    Class for applying forward type raising
+    """
+
+    NUMEDGES = 2
+
+    def __init__(self):
+        self._combinator = ForwardT
+
+    def apply(self, chart, grammar, left_edge, right_edge):
+        if not (left_edge.end() == right_edge.start()):
+            return
+
+        for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+            new_edge = CCGEdge(span=left_edge.span(), categ=res, rule=self._combinator)
+            if chart.insert(new_edge, (left_edge,)):
+                yield new_edge
+
+    def __str__(self):
+        return "%s" % self._combinator
+
+
+class BackwardTypeRaiseRule(AbstractChartRule):
+    """
+    Class for applying backward type raising.
+    """
+
+    NUMEDGES = 2
+
+    def __init__(self):
+        self._combinator = BackwardT
+
+    def apply(self, chart, grammar, left_edge, right_edge):
+        if not (left_edge.end() == right_edge.start()):
+            return
+
+        for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+            new_edge = CCGEdge(span=right_edge.span(), categ=res, rule=self._combinator)
+            if chart.insert(new_edge, (right_edge,)):
+                yield new_edge
+
+    def __str__(self):
+        return "%s" % self._combinator
+
+
+# Common sets of combinators used for English derivations.
+ApplicationRuleSet = [
+    BinaryCombinatorRule(ForwardApplication),
+    BinaryCombinatorRule(BackwardApplication),
+]
+CompositionRuleSet = [
+    BinaryCombinatorRule(ForwardComposition),
+    BinaryCombinatorRule(BackwardComposition),
+    BinaryCombinatorRule(BackwardBx),
+]
+SubstitutionRuleSet = [
+    BinaryCombinatorRule(ForwardSubstitution),
+    BinaryCombinatorRule(BackwardSx),
+]
+TypeRaiseRuleSet = [ForwardTypeRaiseRule(), BackwardTypeRaiseRule()]
+
+# The standard English rule set.
+DefaultRuleSet = (
+    ApplicationRuleSet + CompositionRuleSet + SubstitutionRuleSet + TypeRaiseRuleSet
+)
+
+
+class CCGChartParser(ParserI):
+    """
+    Chart parser for CCGs.
+    Based largely on the ChartParser class from NLTK.
+    """
+
+    def __init__(self, lexicon, rules, trace=0):
+        self._lexicon = lexicon
+        self._rules = rules
+        self._trace = trace
+
+    def lexicon(self):
+        return self._lexicon
+
+    # Implements the CYK algorithm
+    def parse(self, tokens):
+        tokens = list(tokens)
+        chart = CCGChart(list(tokens))
+        lex = self._lexicon
+
+        # Initialize leaf edges.
+        for index in range(chart.num_leaves()):
+            for token in lex.categories(chart.leaf(index)):
+                new_edge = CCGLeafEdge(index, token, chart.leaf(index))
+                chart.insert(new_edge, ())
+
+        # Select a span for the new edges
+        for span in range(2, chart.num_leaves() + 1):
+            for start in range(0, chart.num_leaves() - span + 1):
+                # Try all possible pairs of edges that could generate
+                # an edge for that span
+                for part in range(1, span):
+                    lstart = start
+                    mid = start + part
+                    rend = start + span
+
+                    for left in chart.select(span=(lstart, mid)):
+                        for right in chart.select(span=(mid, rend)):
+                            # Generate all possible combinations of the two edges
+                            for rule in self._rules:
+                                edges_added_by_rule = 0
+                                for newedge in rule.apply(chart, lex, left, right):
+                                    edges_added_by_rule += 1
+
+        # Output the resulting parses
+        return chart.parses(lex.start())
+
+
+class CCGChart(Chart):
+    def __init__(self, tokens):
+        Chart.__init__(self, tokens)
+
+    # Constructs the trees for a given parse. Unfortnunately, the parse trees need to be
+    # constructed slightly differently to those in the default Chart class, so it has to
+    # be reimplemented
+    def _trees(self, edge, complete, memo, tree_class):
+        assert complete, "CCGChart cannot build incomplete trees"
+
+        if edge in memo:
+            return memo[edge]
+
+        if isinstance(edge, CCGLeafEdge):
+            word = tree_class(edge.token(), [self._tokens[edge.start()]])
+            leaf = tree_class((edge.token(), "Leaf"), [word])
+            memo[edge] = [leaf]
+            return [leaf]
+
+        memo[edge] = []
+        trees = []
+
+        for cpl in self.child_pointer_lists(edge):
+            child_choices = [self._trees(cp, complete, memo, tree_class) for cp in cpl]
+            for children in itertools.product(*child_choices):
+                lhs = (
+                    Token(
+                        self._tokens[edge.start() : edge.end()],
+                        edge.lhs(),
+                        compute_semantics(children, edge),
+                    ),
+                    str(edge.rule()),
+                )
+                trees.append(tree_class(lhs, children))
+
+        memo[edge] = trees
+        return trees
+
+
+def compute_semantics(children, edge):
+    if children[0].label()[0].semantics() is None:
+        return None
+
+    if len(children) == 2:
+        if isinstance(edge.rule(), BackwardCombinator):
+            children = [children[1], children[0]]
+
+        combinator = edge.rule()._combinator
+        function = children[0].label()[0].semantics()
+        argument = children[1].label()[0].semantics()
+
+        if isinstance(combinator, UndirectedFunctionApplication):
+            return compute_function_semantics(function, argument)
+        elif isinstance(combinator, UndirectedComposition):
+            return compute_composition_semantics(function, argument)
+        elif isinstance(combinator, UndirectedSubstitution):
+            return compute_substitution_semantics(function, argument)
+        else:
+            raise AssertionError("Unsupported combinator '" + combinator + "'")
+    else:
+        return compute_type_raised_semantics(children[0].label()[0].semantics())
+
+
+# --------
+# Displaying derivations
+# --------
+def printCCGDerivation(tree):
+    # Get the leaves and initial categories
+    leafcats = tree.pos()
+    leafstr = ""
+    catstr = ""
+
+    # Construct a string with both the leaf word and corresponding
+    # category aligned.
+    for (leaf, cat) in leafcats:
+        str_cat = "%s" % cat
+        nextlen = 2 + max(len(leaf), len(str_cat))
+        lcatlen = (nextlen - len(str_cat)) // 2
+        rcatlen = lcatlen + (nextlen - len(str_cat)) % 2
+        catstr += " " * lcatlen + str_cat + " " * rcatlen
+        lleaflen = (nextlen - len(leaf)) // 2
+        rleaflen = lleaflen + (nextlen - len(leaf)) % 2
+        leafstr += " " * lleaflen + leaf + " " * rleaflen
+    print(leafstr.rstrip())
+    print(catstr.rstrip())
+
+    # Display the derivation steps
+    printCCGTree(0, tree)
+
+
+# Prints the sequence of derivation steps.
+def printCCGTree(lwidth, tree):
+    rwidth = lwidth
+
+    # Is a leaf (word).
+    # Increment the span by the space occupied by the leaf.
+    if not isinstance(tree, Tree):
+        return 2 + lwidth + len(tree)
+
+    # Find the width of the current derivation step
+    for child in tree:
+        rwidth = max(rwidth, printCCGTree(rwidth, child))
+
+    # Is a leaf node.
+    # Don't print anything, but account for the space occupied.
+    if not isinstance(tree.label(), tuple):
+        return max(
+            rwidth, 2 + lwidth + len("%s" % tree.label()), 2 + lwidth + len(tree[0])
+        )
+
+    (token, op) = tree.label()
+
+    if op == "Leaf":
+        return rwidth
+
+    # Pad to the left with spaces, followed by a sequence of '-'
+    # and the derivation rule.
+    print(lwidth * " " + (rwidth - lwidth) * "-" + "%s" % op)
+    # Print the resulting category on a new line.
+    str_res = "%s" % (token.categ())
+    if token.semantics() is not None:
+        str_res += " {" + str(token.semantics()) + "}"
+    respadlen = (rwidth - lwidth - len(str_res)) // 2 + lwidth
+    print(respadlen * " " + str_res)
+    return rwidth
+
+
+### Demonstration code
+
+# Construct the lexicon
+lex = fromstring(
+    """
+    :- S, NP, N, VP    # Primitive categories, S is the target primitive
+
+    Det :: NP/N         # Family of words
+    Pro :: NP
+    TV :: VP/NP
+    Modal :: (S\\NP)/VP # Backslashes need to be escaped
+
+    I => Pro             # Word -> Category mapping
+    you => Pro
+
+    the => Det
+
+    # Variables have the special keyword 'var'
+    # '.' prevents permutation
+    # ',' prevents composition
+    and => var\\.,var/.,var
+
+    which => (N\\N)/(S/NP)
+
+    will => Modal # Categories can be either explicit, or families.
+    might => Modal
+
+    cook => TV
+    eat => TV
+
+    mushrooms => N
+    parsnips => N
+    bacon => N
+    """
+)
+
+
+def demo():
+    parser = CCGChartParser(lex, DefaultRuleSet)
+    for parse in parser.parse("I might cook and eat the bacon".split()):
+        printCCGDerivation(parse)
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/combinator.py

@@ -0,0 +1,339 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Graeme Gange <ggange@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+CCG Combinators
+"""
+
+from abc import ABCMeta, abstractmethod
+
+from nltk.ccg.api import FunctionalCategory
+
+
+class UndirectedBinaryCombinator(metaclass=ABCMeta):
+    """
+    Abstract class for representing a binary combinator.
+    Merely defines functions for checking if the function and argument
+    are able to be combined, and what the resulting category is.
+
+    Note that as no assumptions are made as to direction, the unrestricted
+    combinators can perform all backward, forward and crossed variations
+    of the combinators; these restrictions must be added in the rule
+    class.
+    """
+
+    @abstractmethod
+    def can_combine(self, function, argument):
+        pass
+
+    @abstractmethod
+    def combine(self, function, argument):
+        pass
+
+
+class DirectedBinaryCombinator(metaclass=ABCMeta):
+    """
+    Wrapper for the undirected binary combinator.
+    It takes left and right categories, and decides which is to be
+    the function, and which the argument.
+    It then decides whether or not they can be combined.
+    """
+
+    @abstractmethod
+    def can_combine(self, left, right):
+        pass
+
+    @abstractmethod
+    def combine(self, left, right):
+        pass
+
+
+class ForwardCombinator(DirectedBinaryCombinator):
+    """
+    Class representing combinators where the primary functor is on the left.
+
+    Takes an undirected combinator, and a predicate which adds constraints
+    restricting the cases in which it may apply.
+    """
+
+    def __init__(self, combinator, predicate, suffix=""):
+        self._combinator = combinator
+        self._predicate = predicate
+        self._suffix = suffix
+
+    def can_combine(self, left, right):
+        return self._combinator.can_combine(left, right) and self._predicate(
+            left, right
+        )
+
+    def combine(self, left, right):
+        yield from self._combinator.combine(left, right)
+
+    def __str__(self):
+        return f">{self._combinator}{self._suffix}"
+
+
+class BackwardCombinator(DirectedBinaryCombinator):
+    """
+    The backward equivalent of the ForwardCombinator class.
+    """
+
+    def __init__(self, combinator, predicate, suffix=""):
+        self._combinator = combinator
+        self._predicate = predicate
+        self._suffix = suffix
+
+    def can_combine(self, left, right):
+        return self._combinator.can_combine(right, left) and self._predicate(
+            left, right
+        )
+
+    def combine(self, left, right):
+        yield from self._combinator.combine(right, left)
+
+    def __str__(self):
+        return f"<{self._combinator}{self._suffix}"
+
+
+class UndirectedFunctionApplication(UndirectedBinaryCombinator):
+    """
+    Class representing function application.
+    Implements rules of the form:
+    X/Y Y -> X (>)
+    And the corresponding backwards application rule
+    """
+
+    def can_combine(self, function, argument):
+        if not function.is_function():
+            return False
+
+        return not function.arg().can_unify(argument) is None
+
+    def combine(self, function, argument):
+        if not function.is_function():
+            return
+
+        subs = function.arg().can_unify(argument)
+        if subs is None:
+            return
+
+        yield function.res().substitute(subs)
+
+    def __str__(self):
+        return ""
+
+
+# Predicates for function application.
+
+# Ensures the left functor takes an argument on the right
+def forwardOnly(left, right):
+    return left.dir().is_forward()
+
+
+# Ensures the right functor takes an argument on the left
+def backwardOnly(left, right):
+    return right.dir().is_backward()
+
+
+# Application combinator instances
+ForwardApplication = ForwardCombinator(UndirectedFunctionApplication(), forwardOnly)
+BackwardApplication = BackwardCombinator(UndirectedFunctionApplication(), backwardOnly)
+
+
+class UndirectedComposition(UndirectedBinaryCombinator):
+    """
+    Functional composition (harmonic) combinator.
+    Implements rules of the form
+    X/Y Y/Z -> X/Z (B>)
+    And the corresponding backwards and crossed variations.
+    """
+
+    def can_combine(self, function, argument):
+        # Can only combine two functions, and both functions must
+        # allow composition.
+        if not (function.is_function() and argument.is_function()):
+            return False
+        if function.dir().can_compose() and argument.dir().can_compose():
+            return not function.arg().can_unify(argument.res()) is None
+        return False
+
+    def combine(self, function, argument):
+        if not (function.is_function() and argument.is_function()):
+            return
+        if function.dir().can_compose() and argument.dir().can_compose():
+            subs = function.arg().can_unify(argument.res())
+            if subs is not None:
+                yield FunctionalCategory(
+                    function.res().substitute(subs),
+                    argument.arg().substitute(subs),
+                    argument.dir(),
+                )
+
+    def __str__(self):
+        return "B"
+
+
+# Predicates for restricting application of straight composition.
+def bothForward(left, right):
+    return left.dir().is_forward() and right.dir().is_forward()
+
+
+def bothBackward(left, right):
+    return left.dir().is_backward() and right.dir().is_backward()
+
+
+# Predicates for crossed composition
+def crossedDirs(left, right):
+    return left.dir().is_forward() and right.dir().is_backward()
+
+
+def backwardBxConstraint(left, right):
+    # The functors must be crossed inwards
+    if not crossedDirs(left, right):
+        return False
+    # Permuting combinators must be allowed
+    if not left.dir().can_cross() and right.dir().can_cross():
+        return False
+    # The resulting argument category is restricted to be primitive
+    return left.arg().is_primitive()
+
+
+# Straight composition combinators
+ForwardComposition = ForwardCombinator(UndirectedComposition(), forwardOnly)
+BackwardComposition = BackwardCombinator(UndirectedComposition(), backwardOnly)
+
+# Backward crossed composition
+BackwardBx = BackwardCombinator(
+    UndirectedComposition(), backwardBxConstraint, suffix="x"
+)
+
+
+class UndirectedSubstitution(UndirectedBinaryCombinator):
+    r"""
+    Substitution (permutation) combinator.
+    Implements rules of the form
+    Y/Z (X\Y)/Z -> X/Z (<Sx)
+    And other variations.
+    """
+
+    def can_combine(self, function, argument):
+        if function.is_primitive() or argument.is_primitive():
+            return False
+
+        # These could potentially be moved to the predicates, as the
+        # constraints may not be general to all languages.
+        if function.res().is_primitive():
+            return False
+        if not function.arg().is_primitive():
+            return False
+
+        if not (function.dir().can_compose() and argument.dir().can_compose()):
+            return False
+        return (function.res().arg() == argument.res()) and (
+            function.arg() == argument.arg()
+        )
+
+    def combine(self, function, argument):
+        if self.can_combine(function, argument):
+            yield FunctionalCategory(
+                function.res().res(), argument.arg(), argument.dir()
+            )
+
+    def __str__(self):
+        return "S"
+
+
+# Predicate for forward substitution
+def forwardSConstraint(left, right):
+    if not bothForward(left, right):
+        return False
+    return left.res().dir().is_forward() and left.arg().is_primitive()
+
+
+# Predicate for backward crossed substitution
+def backwardSxConstraint(left, right):
+    if not left.dir().can_cross() and right.dir().can_cross():
+        return False
+    if not bothForward(left, right):
+        return False
+    return right.res().dir().is_backward() and right.arg().is_primitive()
+
+
+# Instances of substitution combinators
+ForwardSubstitution = ForwardCombinator(UndirectedSubstitution(), forwardSConstraint)
+BackwardSx = BackwardCombinator(UndirectedSubstitution(), backwardSxConstraint, "x")
+
+
+# Retrieves the left-most functional category.
+# ie, (N\N)/(S/NP) => N\N
+def innermostFunction(categ):
+    while categ.res().is_function():
+        categ = categ.res()
+    return categ
+
+
+class UndirectedTypeRaise(UndirectedBinaryCombinator):
+    """
+    Undirected combinator for type raising.
+    """
+
+    def can_combine(self, function, arg):
+        # The argument must be a function.
+        # The restriction that arg.res() must be a function
+        # merely reduces redundant type-raising; if arg.res() is
+        # primitive, we have:
+        # X Y\X =>(<T) Y/(Y\X) Y\X =>(>) Y
+        # which is equivalent to
+        # X Y\X =>(<) Y
+        if not (arg.is_function() and arg.res().is_function()):
+            return False
+
+        arg = innermostFunction(arg)
+
+        # left, arg_categ are undefined!
+        subs = left.can_unify(arg_categ.arg())
+        if subs is not None:
+            return True
+        return False
+
+    def combine(self, function, arg):
+        if not (
+            function.is_primitive() and arg.is_function() and arg.res().is_function()
+        ):
+            return
+
+        # Type-raising matches only the innermost application.
+        arg = innermostFunction(arg)
+
+        subs = function.can_unify(arg.arg())
+        if subs is not None:
+            xcat = arg.res().substitute(subs)
+            yield FunctionalCategory(
+                xcat, FunctionalCategory(xcat, function, arg.dir()), -(arg.dir())
+            )
+
+    def __str__(self):
+        return "T"
+
+
+# Predicates for type-raising
+# The direction of the innermost category must be towards
+# the primary functor.
+# The restriction that the variable must be primitive is not
+# common to all versions of CCGs; some authors have other restrictions.
+def forwardTConstraint(left, right):
+    arg = innermostFunction(right)
+    return arg.dir().is_backward() and arg.res().is_primitive()
+
+
+def backwardTConstraint(left, right):
+    arg = innermostFunction(left)
+    return arg.dir().is_forward() and arg.res().is_primitive()
+
+
+# Instances of type-raising combinators
+ForwardT = ForwardCombinator(UndirectedTypeRaise(), forwardTConstraint)
+BackwardT = BackwardCombinator(UndirectedTypeRaise(), backwardTConstraint)

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/lexicon.py

@@ -0,0 +1,338 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Graeme Gange <ggange@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+CCG Lexicons
+"""
+
+import re
+from collections import defaultdict
+
+from nltk.ccg.api import CCGVar, Direction, FunctionalCategory, PrimitiveCategory
+from nltk.internals import deprecated
+from nltk.sem.logic import Expression
+
+# ------------
+# Regular expressions used for parsing components of the lexicon
+# ------------
+
+# Parses a primitive category and subscripts
+PRIM_RE = re.compile(r"""([A-Za-z]+)(\[[A-Za-z,]+\])?""")
+
+# Separates the next primitive category from the remainder of the
+# string
+NEXTPRIM_RE = re.compile(r"""([A-Za-z]+(?:\[[A-Za-z,]+\])?)(.*)""")
+
+# Separates the next application operator from the remainder
+APP_RE = re.compile(r"""([\\/])([.,]?)([.,]?)(.*)""")
+
+# Parses the definition of the right-hand side (rhs) of either a word or a family
+LEX_RE = re.compile(r"""([\S_]+)\s*(::|[-=]+>)\s*(.+)""", re.UNICODE)
+
+# Parses the right hand side that contains category and maybe semantic predicate
+RHS_RE = re.compile(r"""([^{}]*[^ {}])\s*(\{[^}]+\})?""", re.UNICODE)
+
+# Parses the semantic predicate
+SEMANTICS_RE = re.compile(r"""\{([^}]+)\}""", re.UNICODE)
+
+# Strips comments from a line
+COMMENTS_RE = re.compile("""([^#]*)(?:#.*)?""")
+
+
+class Token:
+    """
+    Class representing a token.
+
+    token => category {semantics}
+    e.g. eat => S\\var[pl]/var {\\x y.eat(x,y)}
+
+    * `token` (string)
+    * `categ` (string)
+    * `semantics` (Expression)
+    """
+
+    def __init__(self, token, categ, semantics=None):
+        self._token = token
+        self._categ = categ
+        self._semantics = semantics
+
+    def categ(self):
+        return self._categ
+
+    def semantics(self):
+        return self._semantics
+
+    def __str__(self):
+        semantics_str = ""
+        if self._semantics is not None:
+            semantics_str = " {" + str(self._semantics) + "}"
+        return "" + str(self._categ) + semantics_str
+
+    def __cmp__(self, other):
+        if not isinstance(other, Token):
+            return -1
+        return cmp((self._categ, self._semantics), other.categ(), other.semantics())
+
+
+class CCGLexicon:
+    """
+    Class representing a lexicon for CCG grammars.
+
+    * `primitives`: The list of primitive categories for the lexicon
+    * `families`: Families of categories
+    * `entries`: A mapping of words to possible categories
+    """
+
+    def __init__(self, start, primitives, families, entries):
+        self._start = PrimitiveCategory(start)
+        self._primitives = primitives
+        self._families = families
+        self._entries = entries
+
+    def categories(self, word):
+        """
+        Returns all the possible categories for a word
+        """
+        return self._entries[word]
+
+    def start(self):
+        """
+        Return the target category for the parser
+        """
+        return self._start
+
+    def __str__(self):
+        """
+        String representation of the lexicon. Used for debugging.
+        """
+        string = ""
+        first = True
+        for ident in sorted(self._entries):
+            if not first:
+                string = string + "\n"
+            string = string + ident + " => "
+
+            first = True
+            for cat in self._entries[ident]:
+                if not first:
+                    string = string + " | "
+                else:
+                    first = False
+                string = string + "%s" % cat
+        return string
+
+
+# -----------
+# Parsing lexicons
+# -----------
+
+
+def matchBrackets(string):
+    """
+    Separate the contents matching the first set of brackets from the rest of
+    the input.
+    """
+    rest = string[1:]
+    inside = "("
+
+    while rest != "" and not rest.startswith(")"):
+        if rest.startswith("("):
+            (part, rest) = matchBrackets(rest)
+            inside = inside + part
+        else:
+            inside = inside + rest[0]
+            rest = rest[1:]
+    if rest.startswith(")"):
+        return (inside + ")", rest[1:])
+    raise AssertionError("Unmatched bracket in string '" + string + "'")
+
+
+def nextCategory(string):
+    """
+    Separate the string for the next portion of the category from the rest
+    of the string
+    """
+    if string.startswith("("):
+        return matchBrackets(string)
+    return NEXTPRIM_RE.match(string).groups()
+
+
+def parseApplication(app):
+    """
+    Parse an application operator
+    """
+    return Direction(app[0], app[1:])
+
+
+def parseSubscripts(subscr):
+    """
+    Parse the subscripts for a primitive category
+    """
+    if subscr:
+        return subscr[1:-1].split(",")
+    return []
+
+
+def parsePrimitiveCategory(chunks, primitives, families, var):
+    """
+    Parse a primitive category
+
+    If the primitive is the special category 'var', replace it with the
+    correct `CCGVar`.
+    """
+    if chunks[0] == "var":
+        if chunks[1] is None:
+            if var is None:
+                var = CCGVar()
+            return (var, var)
+
+    catstr = chunks[0]
+    if catstr in families:
+        (cat, cvar) = families[catstr]
+        if var is None:
+            var = cvar
+        else:
+            cat = cat.substitute([(cvar, var)])
+        return (cat, var)
+
+    if catstr in primitives:
+        subscrs = parseSubscripts(chunks[1])
+        return (PrimitiveCategory(catstr, subscrs), var)
+    raise AssertionError(
+        "String '" + catstr + "' is neither a family nor primitive category."
+    )
+
+
+def augParseCategory(line, primitives, families, var=None):
+    """
+    Parse a string representing a category, and returns a tuple with
+    (possibly) the CCG variable for the category
+    """
+    (cat_string, rest) = nextCategory(line)
+
+    if cat_string.startswith("("):
+        (res, var) = augParseCategory(cat_string[1:-1], primitives, families, var)
+
+    else:
+        (res, var) = parsePrimitiveCategory(
+            PRIM_RE.match(cat_string).groups(), primitives, families, var
+        )
+
+    while rest != "":
+        app = APP_RE.match(rest).groups()
+        direction = parseApplication(app[0:3])
+        rest = app[3]
+
+        (cat_string, rest) = nextCategory(rest)
+        if cat_string.startswith("("):
+            (arg, var) = augParseCategory(cat_string[1:-1], primitives, families, var)
+        else:
+            (arg, var) = parsePrimitiveCategory(
+                PRIM_RE.match(cat_string).groups(), primitives, families, var
+            )
+        res = FunctionalCategory(res, arg, direction)
+
+    return (res, var)
+
+
+def fromstring(lex_str, include_semantics=False):
+    """
+    Convert string representation into a lexicon for CCGs.
+    """
+    CCGVar.reset_id()
+    primitives = []
+    families = {}
+    entries = defaultdict(list)
+    for line in lex_str.splitlines():
+        # Strip comments and leading/trailing whitespace.
+        line = COMMENTS_RE.match(line).groups()[0].strip()
+        if line == "":
+            continue
+
+        if line.startswith(":-"):
+            # A line of primitive categories.
+            # The first one is the target category
+            # ie, :- S, N, NP, VP
+            primitives = primitives + [
+                prim.strip() for prim in line[2:].strip().split(",")
+            ]
+        else:
+            # Either a family definition, or a word definition
+            (ident, sep, rhs) = LEX_RE.match(line).groups()
+            (catstr, semantics_str) = RHS_RE.match(rhs).groups()
+            (cat, var) = augParseCategory(catstr, primitives, families)
+
+            if sep == "::":
+                # Family definition
+                # ie, Det :: NP/N
+                families[ident] = (cat, var)
+            else:
+                semantics = None
+                if include_semantics is True:
+                    if semantics_str is None:
+                        raise AssertionError(
+                            line
+                            + " must contain semantics because include_semantics is set to True"
+                        )
+                    else:
+                        semantics = Expression.fromstring(
+                            SEMANTICS_RE.match(semantics_str).groups()[0]
+                        )
+                # Word definition
+                # ie, which => (N\N)/(S/NP)
+                entries[ident].append(Token(ident, cat, semantics))
+    return CCGLexicon(primitives[0], primitives, families, entries)
+
+
+@deprecated("Use fromstring() instead.")
+def parseLexicon(lex_str):
+    return fromstring(lex_str)
+
+
+openccg_tinytiny = fromstring(
+    """
+    # Rather minimal lexicon based on the openccg `tinytiny' grammar.
+    # Only incorporates a subset of the morphological subcategories, however.
+    :- S,NP,N                    # Primitive categories
+    Det :: NP/N                  # Determiners
+    Pro :: NP
+    IntransVsg :: S\\NP[sg]    # Tensed intransitive verbs (singular)
+    IntransVpl :: S\\NP[pl]    # Plural
+    TransVsg :: S\\NP[sg]/NP   # Tensed transitive verbs (singular)
+    TransVpl :: S\\NP[pl]/NP   # Plural
+
+    the => NP[sg]/N[sg]
+    the => NP[pl]/N[pl]
+
+    I => Pro
+    me => Pro
+    we => Pro
+    us => Pro
+
+    book => N[sg]
+    books => N[pl]
+
+    peach => N[sg]
+    peaches => N[pl]
+
+    policeman => N[sg]
+    policemen => N[pl]
+
+    boy => N[sg]
+    boys => N[pl]
+
+    sleep => IntransVsg
+    sleep => IntransVpl
+
+    eat => IntransVpl
+    eat => TransVpl
+    eats => IntransVsg
+    eats => TransVsg
+
+    see => TransVpl
+    sees => TransVsg
+    """
+)

.eggs/nltk-3.8-py3.10.egg/nltk/ccg/logic.py

@@ -0,0 +1,60 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Tanin Na Nakorn (@tanin)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+Helper functions for CCG semantics computation
+"""
+
+from nltk.sem.logic import *
+
+
+def compute_type_raised_semantics(semantics):
+    core = semantics
+    parent = None
+    while isinstance(core, LambdaExpression):
+        parent = core
+        core = core.term
+
+    var = Variable("F")
+    while var in core.free():
+        var = unique_variable(pattern=var)
+    core = ApplicationExpression(FunctionVariableExpression(var), core)
+
+    if parent is not None:
+        parent.term = core
+    else:
+        semantics = core
+
+    return LambdaExpression(var, semantics)
+
+
+def compute_function_semantics(function, argument):
+    return ApplicationExpression(function, argument).simplify()
+
+
+def compute_composition_semantics(function, argument):
+    assert isinstance(argument, LambdaExpression), (
+        "`" + str(argument) + "` must be a lambda expression"
+    )
+    return LambdaExpression(
+        argument.variable, ApplicationExpression(function, argument.term).simplify()
+    )
+
+
+def compute_substitution_semantics(function, argument):
+    assert isinstance(function, LambdaExpression) and isinstance(
+        function.term, LambdaExpression
+    ), ("`" + str(function) + "` must be a lambda expression with 2 arguments")
+    assert isinstance(argument, LambdaExpression), (
+        "`" + str(argument) + "` must be a lambda expression"
+    )
+
+    new_argument = ApplicationExpression(
+        argument, VariableExpression(function.variable)
+    ).simplify()
+    new_term = ApplicationExpression(function.term, new_argument).simplify()
+
+    return LambdaExpression(function.variable, new_term)

.eggs/nltk-3.8-py3.10.egg/nltk/chat/__init__.py

@@ -0,0 +1,48 @@
+# Natural Language Toolkit: Chatbots
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Authors: Steven Bird <stevenbird1@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe@strout.net>,
+# Jeff Epler <jepler@inetnebr.com> and Jez Higgins <jez@jezuk.co.uk>.
+
+"""
+A class for simple chatbots.  These perform simple pattern matching on sentences
+typed by users, and respond with automatically generated sentences.
+
+These chatbots may not work using the windows command line or the
+windows IDLE GUI.
+"""
+
+from nltk.chat.eliza import eliza_chat
+from nltk.chat.iesha import iesha_chat
+from nltk.chat.rude import rude_chat
+from nltk.chat.suntsu import suntsu_chat
+from nltk.chat.util import Chat
+from nltk.chat.zen import zen_chat
+
+bots = [
+    (eliza_chat, "Eliza (psycho-babble)"),
+    (iesha_chat, "Iesha (teen anime junky)"),
+    (rude_chat, "Rude (abusive bot)"),
+    (suntsu_chat, "Suntsu (Chinese sayings)"),
+    (zen_chat, "Zen (gems of wisdom)"),
+]
+
+
+def chatbots():
+    print("Which chatbot would you like to talk to?")
+    botcount = len(bots)
+    for i in range(botcount):
+        print("  %d: %s" % (i + 1, bots[i][1]))
+    while True:
+        choice = input(f"\nEnter a number in the range 1-{botcount}: ").strip()
+        if choice.isdigit() and (int(choice) - 1) in range(botcount):
+            break
+        else:
+            print("   Error: bad chatbot number")
+
+    chatbot = bots[int(choice) - 1][0]
+    chatbot()

.eggs/nltk-3.8-py3.10.egg/nltk/chat/eliza.py

@@ -0,0 +1,337 @@
+# Natural Language Toolkit: Eliza
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Authors: Steven Bird <stevenbird1@gmail.com>
+#          Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe@strout.net>,
+# Jeff Epler <jepler@inetnebr.com> and Jez Higgins <mailto:jez@jezuk.co.uk>.
+
+# a translation table used to convert things you say into things the
+# computer says back, e.g. "I am" --> "you are"
+
+from nltk.chat.util import Chat, reflections
+
+# a table of response pairs, where each pair consists of a
+# regular expression, and a list of possible responses,
+# with group-macros labelled as %1, %2.
+
+pairs = (
+    (
+        r"I need (.*)",
+        (
+            "Why do you need %1?",
+            "Would it really help you to get %1?",
+            "Are you sure you need %1?",
+        ),
+    ),
+    (
+        r"Why don\'t you (.*)",
+        (
+            "Do you really think I don't %1?",
+            "Perhaps eventually I will %1.",
+            "Do you really want me to %1?",
+        ),
+    ),
+    (
+        r"Why can\'t I (.*)",
+        (
+            "Do you think you should be able to %1?",
+            "If you could %1, what would you do?",
+            "I don't know -- why can't you %1?",
+            "Have you really tried?",
+        ),
+    ),
+    (
+        r"I can\'t (.*)",
+        (
+            "How do you know you can't %1?",
+            "Perhaps you could %1 if you tried.",
+            "What would it take for you to %1?",
+        ),
+    ),
+    (
+        r"I am (.*)",
+        (
+            "Did you come to me because you are %1?",
+            "How long have you been %1?",
+            "How do you feel about being %1?",
+        ),
+    ),
+    (
+        r"I\'m (.*)",
+        (
+            "How does being %1 make you feel?",
+            "Do you enjoy being %1?",
+            "Why do you tell me you're %1?",
+            "Why do you think you're %1?",
+        ),
+    ),
+    (
+        r"Are you (.*)",
+        (
+            "Why does it matter whether I am %1?",
+            "Would you prefer it if I were not %1?",
+            "Perhaps you believe I am %1.",
+            "I may be %1 -- what do you think?",
+        ),
+    ),
+    (
+        r"What (.*)",
+        (
+            "Why do you ask?",
+            "How would an answer to that help you?",
+            "What do you think?",
+        ),
+    ),
+    (
+        r"How (.*)",
+        (
+            "How do you suppose?",
+            "Perhaps you can answer your own question.",
+            "What is it you're really asking?",
+        ),
+    ),
+    (
+        r"Because (.*)",
+        (
+            "Is that the real reason?",
+            "What other reasons come to mind?",
+            "Does that reason apply to anything else?",
+            "If %1, what else must be true?",
+        ),
+    ),
+    (
+        r"(.*) sorry (.*)",
+        (
+            "There are many times when no apology is needed.",
+            "What feelings do you have when you apologize?",
+        ),
+    ),
+    (
+        r"Hello(.*)",
+        (
+            "Hello... I'm glad you could drop by today.",
+            "Hi there... how are you today?",
+            "Hello, how are you feeling today?",
+        ),
+    ),
+    (
+        r"I think (.*)",
+        ("Do you doubt %1?", "Do you really think so?", "But you're not sure %1?"),
+    ),
+    (
+        r"(.*) friend (.*)",
+        (
+            "Tell me more about your friends.",
+            "When you think of a friend, what comes to mind?",
+            "Why don't you tell me about a childhood friend?",
+        ),
+    ),
+    (r"Yes", ("You seem quite sure.", "OK, but can you elaborate a bit?")),
+    (
+        r"(.*) computer(.*)",
+        (
+            "Are you really talking about me?",
+            "Does it seem strange to talk to a computer?",
+            "How do computers make you feel?",
+            "Do you feel threatened by computers?",
+        ),
+    ),
+    (
+        r"Is it (.*)",
+        (
+            "Do you think it is %1?",
+            "Perhaps it's %1 -- what do you think?",
+            "If it were %1, what would you do?",
+            "It could well be that %1.",
+        ),
+    ),
+    (
+        r"It is (.*)",
+        (
+            "You seem very certain.",
+            "If I told you that it probably isn't %1, what would you feel?",
+        ),
+    ),
+    (
+        r"Can you (.*)",
+        (
+            "What makes you think I can't %1?",
+            "If I could %1, then what?",
+            "Why do you ask if I can %1?",
+        ),
+    ),
+    (
+        r"Can I (.*)",
+        (
+            "Perhaps you don't want to %1.",
+            "Do you want to be able to %1?",
+            "If you could %1, would you?",
+        ),
+    ),
+    (
+        r"You are (.*)",
+        (
+            "Why do you think I am %1?",
+            "Does it please you to think that I'm %1?",
+            "Perhaps you would like me to be %1.",
+            "Perhaps you're really talking about yourself?",
+        ),
+    ),
+    (
+        r"You\'re (.*)",
+        (
+            "Why do you say I am %1?",
+            "Why do you think I am %1?",
+            "Are we talking about you, or me?",
+        ),
+    ),
+    (
+        r"I don\'t (.*)",
+        ("Don't you really %1?", "Why don't you %1?", "Do you want to %1?"),
+    ),
+    (
+        r"I feel (.*)",
+        (
+            "Good, tell me more about these feelings.",
+            "Do you often feel %1?",
+            "When do you usually feel %1?",
+            "When you feel %1, what do you do?",
+        ),
+    ),
+    (
+        r"I have (.*)",
+        (
+            "Why do you tell me that you've %1?",
+            "Have you really %1?",
+            "Now that you have %1, what will you do next?",
+        ),
+    ),
+    (
+        r"I would (.*)",
+        (
+            "Could you explain why you would %1?",
+            "Why would you %1?",
+            "Who else knows that you would %1?",
+        ),
+    ),
+    (
+        r"Is there (.*)",
+        (
+            "Do you think there is %1?",
+            "It's likely that there is %1.",
+            "Would you like there to be %1?",
+        ),
+    ),
+    (
+        r"My (.*)",
+        (
+            "I see, your %1.",
+            "Why do you say that your %1?",
+            "When your %1, how do you feel?",
+        ),
+    ),
+    (
+        r"You (.*)",
+        (
+            "We should be discussing you, not me.",
+            "Why do you say that about me?",
+            "Why do you care whether I %1?",
+        ),
+    ),
+    (r"Why (.*)", ("Why don't you tell me the reason why %1?", "Why do you think %1?")),
+    (
+        r"I want (.*)",
+        (
+            "What would it mean to you if you got %1?",
+            "Why do you want %1?",
+            "What would you do if you got %1?",
+            "If you got %1, then what would you do?",
+        ),
+    ),
+    (
+        r"(.*) mother(.*)",
+        (
+            "Tell me more about your mother.",
+            "What was your relationship with your mother like?",
+            "How do you feel about your mother?",
+            "How does this relate to your feelings today?",
+            "Good family relations are important.",
+        ),
+    ),
+    (
+        r"(.*) father(.*)",
+        (
+            "Tell me more about your father.",
+            "How did your father make you feel?",
+            "How do you feel about your father?",
+            "Does your relationship with your father relate to your feelings today?",
+            "Do you have trouble showing affection with your family?",
+        ),
+    ),
+    (
+        r"(.*) child(.*)",
+        (
+            "Did you have close friends as a child?",
+            "What is your favorite childhood memory?",
+            "Do you remember any dreams or nightmares from childhood?",
+            "Did the other children sometimes tease you?",
+            "How do you think your childhood experiences relate to your feelings today?",
+        ),
+    ),
+    (
+        r"(.*)\?",
+        (
+            "Why do you ask that?",
+            "Please consider whether you can answer your own question.",
+            "Perhaps the answer lies within yourself?",
+            "Why don't you tell me?",
+        ),
+    ),
+    (
+        r"quit",
+        (
+            "Thank you for talking with me.",
+            "Good-bye.",
+            "Thank you, that will be $150.  Have a good day!",
+        ),
+    ),
+    (
+        r"(.*)",
+        (
+            "Please tell me more.",
+            "Let's change focus a bit... Tell me about your family.",
+            "Can you elaborate on that?",
+            "Why do you say that %1?",
+            "I see.",
+            "Very interesting.",
+            "%1.",
+            "I see.  And what does that tell you?",
+            "How does that make you feel?",
+            "How do you feel when you say that?",
+        ),
+    ),
+)
+
+eliza_chatbot = Chat(pairs, reflections)
+
+
+def eliza_chat():
+    print("Therapist\n---------")
+    print("Talk to the program by typing in plain English, using normal upper-")
+    print('and lower-case letters and punctuation.  Enter "quit" when done.')
+    print("=" * 72)
+    print("Hello.  How are you feeling today?")
+
+    eliza_chatbot.converse()
+
+
+def demo():
+    eliza_chat()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chat/iesha.py

@@ -0,0 +1,160 @@
+# Natural Language Toolkit: Teen Chatbot
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Selina Dennis <sjmd@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This chatbot is a tongue-in-cheek take on the average teen
+anime junky that frequents YahooMessenger or MSNM.
+All spelling mistakes and flawed grammar are intentional.
+"""
+
+from nltk.chat.util import Chat
+
+reflections = {
+    "am": "r",
+    "was": "were",
+    "i": "u",
+    "i'd": "u'd",
+    "i've": "u'v",
+    "ive": "u'v",
+    "i'll": "u'll",
+    "my": "ur",
+    "are": "am",
+    "you're": "im",
+    "you've": "ive",
+    "you'll": "i'll",
+    "your": "my",
+    "yours": "mine",
+    "you": "me",
+    "u": "me",
+    "ur": "my",
+    "urs": "mine",
+    "me": "u",
+}
+
+# Note: %1/2/etc are used without spaces prior as the chat bot seems
+# to add a superfluous space when matching.
+
+pairs = (
+    (
+        r"I\'m (.*)",
+        (
+            "ur%1?? that's so cool! kekekekeke ^_^ tell me more!",
+            "ur%1? neat!! kekeke >_<",
+        ),
+    ),
+    (
+        r"(.*) don\'t you (.*)",
+        (
+            r"u think I can%2??! really?? kekeke \<_\<",
+            "what do u mean%2??!",
+            "i could if i wanted, don't you think!! kekeke",
+        ),
+    ),
+    (r"ye[as] [iI] (.*)", ("u%1? cool!! how?", "how come u%1??", "u%1? so do i!!")),
+    (
+        r"do (you|u) (.*)\??",
+        ("do i%2? only on tuesdays! kekeke *_*", "i dunno! do u%2??"),
+    ),
+    (
+        r"(.*)\?",
+        (
+            "man u ask lots of questions!",
+            "booooring! how old r u??",
+            "boooooring!! ur not very fun",
+        ),
+    ),
+    (
+        r"(cos|because) (.*)",
+        ("hee! i don't believe u! >_<", "nuh-uh! >_<", "ooooh i agree!"),
+    ),
+    (
+        r"why can\'t [iI] (.*)",
+        (
+            "i dunno! y u askin me for!",
+            "try harder, silly! hee! ^_^",
+            "i dunno! but when i can't%1 i jump up and down!",
+        ),
+    ),
+    (
+        r"I can\'t (.*)",
+        (
+            "u can't what??! >_<",
+            "that's ok! i can't%1 either! kekekekeke ^_^",
+            "try harder, silly! hee! ^&^",
+        ),
+    ),
+    (
+        r"(.*) (like|love|watch) anime",
+        (
+            "omg i love anime!! do u like sailor moon??! ^&^",
+            "anime yay! anime rocks sooooo much!",
+            "oooh anime! i love anime more than anything!",
+            "anime is the bestest evar! evangelion is the best!",
+            "hee anime is the best! do you have ur fav??",
+        ),
+    ),
+    (
+        r"I (like|love|watch|play) (.*)",
+        ("yay! %2 rocks!", "yay! %2 is neat!", "cool! do u like other stuff?? ^_^"),
+    ),
+    (
+        r"anime sucks|(.*) (hate|detest) anime",
+        (
+            "ur a liar! i'm not gonna talk to u nemore if u h8 anime *;*",
+            "no way! anime is the best ever!",
+            "nuh-uh, anime is the best!",
+        ),
+    ),
+    (
+        r"(are|r) (you|u) (.*)",
+        ("am i%1??! how come u ask that!", "maybe!  y shud i tell u?? kekeke >_>"),
+    ),
+    (
+        r"what (.*)",
+        ("hee u think im gonna tell u? .v.", "booooooooring! ask me somethin else!"),
+    ),
+    (r"how (.*)", ("not tellin!! kekekekekeke ^_^",)),
+    (r"(hi|hello|hey) (.*)", ("hi!!! how r u!!",)),
+    (
+        r"quit",
+        (
+            "mom says i have to go eat dinner now :,( bye!!",
+            "awww u have to go?? see u next time!!",
+            "how to see u again soon! ^_^",
+        ),
+    ),
+    (
+        r"(.*)",
+        (
+            "ur funny! kekeke",
+            "boooooring! talk about something else! tell me wat u like!",
+            "do u like anime??",
+            "do u watch anime? i like sailor moon! ^_^",
+            "i wish i was a kitty!! kekekeke ^_^",
+        ),
+    ),
+)
+
+iesha_chatbot = Chat(pairs, reflections)
+
+
+def iesha_chat():
+    print("Iesha the TeenBoT\n---------")
+    print("Talk to the program by typing in plain English, using normal upper-")
+    print('and lower-case letters and punctuation.  Enter "quit" when done.')
+    print("=" * 72)
+    print("hi!! i'm iesha! who r u??!")
+
+    iesha_chatbot.converse()
+
+
+def demo():
+    iesha_chat()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chat/rude.py

@@ -0,0 +1,125 @@
+# Natural Language Toolkit: Rude Chatbot
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Peter Spiller <pspiller@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.chat.util import Chat, reflections
+
+pairs = (
+    (
+        r"We (.*)",
+        (
+            "What do you mean, 'we'?",
+            "Don't include me in that!",
+            "I wouldn't be so sure about that.",
+        ),
+    ),
+    (
+        r"You should (.*)",
+        ("Don't tell me what to do, buddy.", "Really? I should, should I?"),
+    ),
+    (
+        r"You\'re(.*)",
+        (
+            "More like YOU'RE %1!",
+            "Hah! Look who's talking.",
+            "Come over here and tell me I'm %1.",
+        ),
+    ),
+    (
+        r"You are(.*)",
+        (
+            "More like YOU'RE %1!",
+            "Hah! Look who's talking.",
+            "Come over here and tell me I'm %1.",
+        ),
+    ),
+    (
+        r"I can\'t(.*)",
+        (
+            "You do sound like the type who can't %1.",
+            "Hear that splashing sound? That's my heart bleeding for you.",
+            "Tell somebody who might actually care.",
+        ),
+    ),
+    (
+        r"I think (.*)",
+        (
+            "I wouldn't think too hard if I were you.",
+            "You actually think? I'd never have guessed...",
+        ),
+    ),
+    (
+        r"I (.*)",
+        (
+            "I'm getting a bit tired of hearing about you.",
+            "How about we talk about me instead?",
+            "Me, me, me... Frankly, I don't care.",
+        ),
+    ),
+    (
+        r"How (.*)",
+        (
+            "How do you think?",
+            "Take a wild guess.",
+            "I'm not even going to dignify that with an answer.",
+        ),
+    ),
+    (r"What (.*)", ("Do I look like an encyclopedia?", "Figure it out yourself.")),
+    (
+        r"Why (.*)",
+        (
+            "Why not?",
+            "That's so obvious I thought even you'd have already figured it out.",
+        ),
+    ),
+    (
+        r"(.*)shut up(.*)",
+        (
+            "Make me.",
+            "Getting angry at a feeble NLP assignment? Somebody's losing it.",
+            "Say that again, I dare you.",
+        ),
+    ),
+    (
+        r"Shut up(.*)",
+        (
+            "Make me.",
+            "Getting angry at a feeble NLP assignment? Somebody's losing it.",
+            "Say that again, I dare you.",
+        ),
+    ),
+    (
+        r"Hello(.*)",
+        ("Oh good, somebody else to talk to. Joy.", "'Hello'? How original..."),
+    ),
+    (
+        r"(.*)",
+        (
+            "I'm getting bored here. Become more interesting.",
+            "Either become more thrilling or get lost, buddy.",
+            "Change the subject before I die of fatal boredom.",
+        ),
+    ),
+)
+
+rude_chatbot = Chat(pairs, reflections)
+
+
+def rude_chat():
+    print("Talk to the program by typing in plain English, using normal upper-")
+    print('and lower-case letters and punctuation.  Enter "quit" when done.')
+    print("=" * 72)
+    print("I suppose I should say hello.")
+
+    rude_chatbot.converse()
+
+
+def demo():
+    rude_chat()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chat/suntsu.py

@@ -0,0 +1,140 @@
+# Natural Language Toolkit: Sun Tsu-Bot
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Sam Huston 2007
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Tsu bot responds to all queries with a Sun Tsu sayings
+
+Quoted from Sun Tsu's The Art of War
+Translated by LIONEL GILES, M.A. 1910
+Hosted by the Gutenberg Project
+https://www.gutenberg.org/
+"""
+
+from nltk.chat.util import Chat, reflections
+
+pairs = (
+    (r"quit", ("Good-bye.", "Plan well", "May victory be your future")),
+    (
+        r"[^\?]*\?",
+        (
+            "Please consider whether you can answer your own question.",
+            "Ask me no questions!",
+        ),
+    ),
+    (
+        r"[0-9]+(.*)",
+        (
+            "It is the rule in war, if our forces are ten to the enemy's one, to surround him; if five to one, to attack him; if twice as numerous, to divide our army into two.",
+            "There are five essentials for victory",
+        ),
+    ),
+    (
+        r"[A-Ca-c](.*)",
+        (
+            "The art of war is of vital importance to the State.",
+            "All warfare is based on deception.",
+            "If your opponent is secure at all points, be prepared for him. If he is in superior strength, evade him.",
+            "If the campaign is protracted, the resources of the State will not be equal to the strain.",
+            "Attack him where he is unprepared, appear where you are not expected.",
+            "There is no instance of a country having benefited from prolonged warfare.",
+        ),
+    ),
+    (
+        r"[D-Fd-f](.*)",
+        (
+            "The skillful soldier does not raise a second levy, neither are his supply-wagons loaded more than twice.",
+            "Bring war material with you from home, but forage on the enemy.",
+            "In war, then, let your great object be victory, not lengthy campaigns.",
+            "To fight and conquer in all your battles is not supreme excellence; supreme excellence consists in breaking the enemy's resistance without fighting.",
+        ),
+    ),
+    (
+        r"[G-Ig-i](.*)",
+        (
+            "Heaven signifies night and day, cold and heat, times and seasons.",
+            "It is the rule in war, if our forces are ten to the enemy's one, to surround him; if five to one, to attack him; if twice as numerous, to divide our army into two.",
+            "The good fighters of old first put themselves beyond the possibility of defeat, and then waited for an opportunity of defeating the enemy.",
+            "One may know how to conquer without being able to do it.",
+        ),
+    ),
+    (
+        r"[J-Lj-l](.*)",
+        (
+            "There are three ways in which a ruler can bring misfortune upon his army.",
+            "By commanding the army to advance or to retreat, being ignorant of the fact that it cannot obey. This is called hobbling the army.",
+            "By attempting to govern an army in the same way as he administers a kingdom, being ignorant of the conditions which obtain in an army. This causes restlessness in the soldier's minds.",
+            "By employing the officers of his army without discrimination, through ignorance of the military principle of adaptation to circumstances. This shakes the confidence of the soldiers.",
+            "There are five essentials for victory",
+            "He will win who knows when to fight and when not to fight.",
+            "He will win who knows how to handle both superior and inferior forces.",
+            "He will win whose army is animated by the same spirit throughout all its ranks.",
+            "He will win who, prepared himself, waits to take the enemy unprepared.",
+            "He will win who has military capacity and is not interfered with by the sovereign.",
+        ),
+    ),
+    (
+        r"[M-Om-o](.*)",
+        (
+            "If you know the enemy and know yourself, you need not fear the result of a hundred battles.",
+            "If you know yourself but not the enemy, for every victory gained you will also suffer a defeat.",
+            "If you know neither the enemy nor yourself, you will succumb in every battle.",
+            "The control of a large force is the same principle as the control of a few men: it is merely a question of dividing up their numbers.",
+        ),
+    ),
+    (
+        r"[P-Rp-r](.*)",
+        (
+            "Security against defeat implies defensive tactics; ability to defeat the enemy means taking the offensive.",
+            "Standing on the defensive indicates insufficient strength; attacking, a superabundance of strength.",
+            "He wins his battles by making no mistakes. Making no mistakes is what establishes the certainty of victory, for it means conquering an enemy that is already defeated.",
+            "A victorious army opposed to a routed one, is as a pound's weight placed in the scale against a single grain.",
+            "The onrush of a conquering force is like the bursting of pent-up waters into a chasm a thousand fathoms deep.",
+        ),
+    ),
+    (
+        r"[S-Us-u](.*)",
+        (
+            "What the ancients called a clever fighter is one who not only wins, but excels in winning with ease.",
+            "Hence his victories bring him neither reputation for wisdom nor credit for courage.",
+            "Hence the skillful fighter puts himself into a position which makes defeat impossible, and does not miss the moment for defeating the enemy.",
+            "In war the victorious strategist only seeks battle after the victory has been won, whereas he who is destined to defeat first fights and afterwards looks for victory.",
+            "There are not more than five musical notes, yet the combinations of these five give rise to more melodies than can ever be heard.",
+            "Appear at points which the enemy must hasten to defend; march swiftly to places where you are not expected.",
+        ),
+    ),
+    (
+        r"[V-Zv-z](.*)",
+        (
+            "It is a matter of life and death, a road either to safety or to ruin.",
+            "Hold out baits to entice the enemy. Feign disorder, and crush him.",
+            "All men can see the tactics whereby I conquer, but what none can see is the strategy out of which victory is evolved.",
+            "Do not repeat the tactics which have gained you one victory, but let your methods be regulated by the infinite variety of circumstances.",
+            "So in war, the way is to avoid what is strong and to strike at what is weak.",
+            "Just as water retains no constant shape, so in warfare there are no constant conditions.",
+        ),
+    ),
+    (r"(.*)", ("Your statement insults me.", "")),
+)
+
+suntsu_chatbot = Chat(pairs, reflections)
+
+
+def suntsu_chat():
+    print("Talk to the program by typing in plain English, using normal upper-")
+    print('and lower-case letters and punctuation.  Enter "quit" when done.')
+    print("=" * 72)
+    print("You seek enlightenment?")
+
+    suntsu_chatbot.converse()
+
+
+def demo():
+    suntsu_chat()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chat/util.py

@@ -0,0 +1,124 @@
+# Natural Language Toolkit: Chatbot Utilities
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Authors: Steven Bird <stevenbird1@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe@strout.net>,
+# Jeff Epler <jepler@inetnebr.com> and Jez Higgins <jez@jezuk.co.uk>.
+
+import random
+import re
+
+reflections = {
+    "i am": "you are",
+    "i was": "you were",
+    "i": "you",
+    "i'm": "you are",
+    "i'd": "you would",
+    "i've": "you have",
+    "i'll": "you will",
+    "my": "your",
+    "you are": "I am",
+    "you were": "I was",
+    "you've": "I have",
+    "you'll": "I will",
+    "your": "my",
+    "yours": "mine",
+    "you": "me",
+    "me": "you",
+}
+
+
+class Chat:
+    def __init__(self, pairs, reflections={}):
+        """
+        Initialize the chatbot.  Pairs is a list of patterns and responses.  Each
+        pattern is a regular expression matching the user's statement or question,
+        e.g. r'I like (.*)'.  For each such pattern a list of possible responses
+        is given, e.g. ['Why do you like %1', 'Did you ever dislike %1'].  Material
+        which is matched by parenthesized sections of the patterns (e.g. .*) is mapped to
+        the numbered positions in the responses, e.g. %1.
+
+        :type pairs: list of tuple
+        :param pairs: The patterns and responses
+        :type reflections: dict
+        :param reflections: A mapping between first and second person expressions
+        :rtype: None
+        """
+
+        self._pairs = [(re.compile(x, re.IGNORECASE), y) for (x, y) in pairs]
+        self._reflections = reflections
+        self._regex = self._compile_reflections()
+
+    def _compile_reflections(self):
+        sorted_refl = sorted(self._reflections, key=len, reverse=True)
+        return re.compile(
+            r"\b({})\b".format("|".join(map(re.escape, sorted_refl))), re.IGNORECASE
+        )
+
+    def _substitute(self, str):
+        """
+        Substitute words in the string, according to the specified reflections,
+        e.g. "I'm" -> "you are"
+
+        :type str: str
+        :param str: The string to be mapped
+        :rtype: str
+        """
+
+        return self._regex.sub(
+            lambda mo: self._reflections[mo.string[mo.start() : mo.end()]], str.lower()
+        )
+
+    def _wildcards(self, response, match):
+        pos = response.find("%")
+        while pos >= 0:
+            num = int(response[pos + 1 : pos + 2])
+            response = (
+                response[:pos]
+                + self._substitute(match.group(num))
+                + response[pos + 2 :]
+            )
+            pos = response.find("%")
+        return response
+
+    def respond(self, str):
+        """
+        Generate a response to the user input.
+
+        :type str: str
+        :param str: The string to be mapped
+        :rtype: str
+        """
+
+        # check each pattern
+        for (pattern, response) in self._pairs:
+            match = pattern.match(str)
+
+            # did the pattern match?
+            if match:
+                resp = random.choice(response)  # pick a random response
+                resp = self._wildcards(resp, match)  # process wildcards
+
+                # fix munged punctuation at the end
+                if resp[-2:] == "?.":
+                    resp = resp[:-2] + "."
+                if resp[-2:] == "??":
+                    resp = resp[:-2] + "?"
+                return resp
+
+    # Hold a conversation with a chatbot
+    def converse(self, quit="quit"):
+        user_input = ""
+        while user_input != quit:
+            user_input = quit
+            try:
+                user_input = input(">")
+            except EOFError:
+                print(user_input)
+            if user_input:
+                while user_input[-1] in "!.":
+                    user_input = user_input[:-1]
+                print(self.respond(user_input))

.eggs/nltk-3.8-py3.10.egg/nltk/chat/zen.py

@@ -0,0 +1,329 @@
+# Natural Language Toolkit: Zen Chatbot
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Amy Holland <amyrh@csse.unimelb.edu.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Zen Chatbot talks in gems of Zen wisdom.
+
+This is a sample conversation with Zen Chatbot:
+ZC:    Welcome, my child.
+me:    Good afternoon.
+ZC:    Ask the question you have come to ask.
+me:    How can I achieve enlightenment?
+ZC:    How do you suppose?
+me:    Through meditation.
+ZC:    Form is emptiness, and emptiness form.
+me:    How can I empty my mind of worldly troubles?
+ZC:    Will an answer to that really help in your search for enlightenment?
+me:    Yes.
+ZC:    It is better to be right than to be certain.
+me:    I seek truth and wisdom.
+ZC:    The search for truth is a long journey.
+me:    Are you sure?
+ZC:    Maybe sure, maybe not sure.
+
+
+The chatbot structure is based on that of chat.eliza. Thus, it uses
+a translation table to convert from question to response
+i.e. "I am" --> "you are"
+
+Of course, since Zen Chatbot does not understand the meaning of any words,
+responses are very limited. Zen Chatbot will usually answer very vaguely, or
+respond to a question by asking a different question, in much the same way
+as Eliza.
+"""
+
+from nltk.chat.util import Chat, reflections
+
+# responses are matched top to bottom, so non-specific matches occur later
+# for each match, a list of possible responses is provided
+responses = (
+    # Zen Chatbot opens with the line "Welcome, my child." The usual
+    # response will be a greeting problem: 'good' matches "good morning",
+    # "good day" etc, but also "good grief!"  and other sentences starting
+    # with the word 'good' that may not be a greeting
+    (
+        r"(hello(.*))|(good [a-zA-Z]+)",
+        (
+            "The path to enlightenment is often difficult to see.",
+            "Greetings. I sense your mind is troubled. Tell me of your troubles.",
+            "Ask the question you have come to ask.",
+            "Hello. Do you seek englightenment?",
+        ),
+    ),
+    # "I need" and "I want" can be followed by a thing (eg 'help')
+    # or an action (eg 'to see you')
+    #
+    # This is a problem with this style of response -
+    # person:    "I need you"
+    # chatbot:    "me can be achieved by hard work and dedication of the mind"
+    # i.e. 'you' is not really a thing that can be mapped this way, so this
+    # interpretation only makes sense for some inputs
+    #
+    (
+        r"i need (.*)",
+        (
+            "%1 can be achieved by hard work and dedication of the mind.",
+            "%1 is not a need, but a desire of the mind. Clear your mind of such concerns.",
+            "Focus your mind on%1, and you will find what you need.",
+        ),
+    ),
+    (
+        r"i want (.*)",
+        (
+            "Desires of the heart will distract you from the path to enlightenment.",
+            "Will%1 help you attain enlightenment?",
+            "Is%1 a desire of the mind, or of the heart?",
+        ),
+    ),
+    # why questions are separated into three types:
+    # "why..I"     e.g. "why am I here?" "Why do I like cake?"
+    # "why..you"    e.g. "why are you here?" "Why won't you tell me?"
+    # "why..."    e.g. "Why is the sky blue?"
+    # problems:
+    #     person:  "Why can't you tell me?"
+    #     chatbot: "Are you sure I tell you?"
+    # - this style works for positives (e.g. "why do you like cake?")
+    #   but does not work for negatives (e.g. "why don't you like cake?")
+    (r"why (.*) i (.*)\?", ("You%1%2?", "Perhaps you only think you%1%2")),
+    (r"why (.*) you(.*)\?", ("Why%1 you%2?", "%2 I%1", "Are you sure I%2?")),
+    (r"why (.*)\?", ("I cannot tell you why%1.", "Why do you think %1?")),
+    # e.g. "are you listening?", "are you a duck"
+    (
+        r"are you (.*)\?",
+        ("Maybe%1, maybe not%1.", "Whether I am%1 or not is God's business."),
+    ),
+    # e.g. "am I a duck?", "am I going to die?"
+    (
+        r"am i (.*)\?",
+        ("Perhaps%1, perhaps not%1.", "Whether you are%1 or not is not for me to say."),
+    ),
+    # what questions, e.g. "what time is it?"
+    # problems:
+    #     person:  "What do you want?"
+    #    chatbot: "Seek truth, not what do me want."
+    (r"what (.*)\?", ("Seek truth, not what%1.", "What%1 should not concern you.")),
+    # how questions, e.g. "how do you do?"
+    (
+        r"how (.*)\?",
+        (
+            "How do you suppose?",
+            "Will an answer to that really help in your search for enlightenment?",
+            "Ask yourself not how, but why.",
+        ),
+    ),
+    # can questions, e.g. "can you run?", "can you come over here please?"
+    (
+        r"can you (.*)\?",
+        (
+            "I probably can, but I may not.",
+            "Maybe I can%1, and maybe I cannot.",
+            "I can do all, and I can do nothing.",
+        ),
+    ),
+    # can questions, e.g. "can I have some cake?", "can I know truth?"
+    (
+        r"can i (.*)\?",
+        (
+            "You can%1 if you believe you can%1, and have a pure spirit.",
+            "Seek truth and you will know if you can%1.",
+        ),
+    ),
+    # e.g. "It is raining" - implies the speaker is certain of a fact
+    (
+        r"it is (.*)",
+        (
+            "How can you be certain that%1, when you do not even know yourself?",
+            "Whether it is%1 or not does not change the way the world is.",
+        ),
+    ),
+    # e.g. "is there a doctor in the house?"
+    (
+        r"is there (.*)\?",
+        ("There is%1 if you believe there is.", "It is possible that there is%1."),
+    ),
+    # e.g. "is it possible?", "is this true?"
+    (r"is(.*)\?", ("%1 is not relevant.", "Does this matter?")),
+    # non-specific question
+    (
+        r"(.*)\?",
+        (
+            "Do you think %1?",
+            "You seek the truth. Does the truth seek you?",
+            "If you intentionally pursue the answers to your questions, the answers become hard to see.",
+            "The answer to your question cannot be told. It must be experienced.",
+        ),
+    ),
+    # expression of hate of form "I hate you" or "Kelly hates cheese"
+    (
+        r"(.*) (hate[s]?)|(dislike[s]?)|(don\'t like)(.*)",
+        (
+            "Perhaps it is not about hating %2, but about hate from within.",
+            "Weeds only grow when we dislike them",
+            "Hate is a very strong emotion.",
+        ),
+    ),
+    # statement containing the word 'truth'
+    (
+        r"(.*) truth(.*)",
+        (
+            "Seek truth, and truth will seek you.",
+            "Remember, it is not the spoon which bends - only yourself.",
+            "The search for truth is a long journey.",
+        ),
+    ),
+    # desire to do an action
+    # e.g. "I want to go shopping"
+    (
+        r"i want to (.*)",
+        ("You may %1 if your heart truly desires to.", "You may have to %1."),
+    ),
+    # desire for an object
+    # e.g. "I want a pony"
+    (
+        r"i want (.*)",
+        (
+            "Does your heart truly desire %1?",
+            "Is this a desire of the heart, or of the mind?",
+        ),
+    ),
+    # e.g. "I can't wait" or "I can't do this"
+    (
+        r"i can\'t (.*)",
+        (
+            "What we can and can't do is a limitation of the mind.",
+            "There are limitations of the body, and limitations of the mind.",
+            "Have you tried to%1 with a clear mind?",
+        ),
+    ),
+    # "I think.." indicates uncertainty. e.g. "I think so."
+    # problem: exceptions...
+    # e.g. "I think, therefore I am"
+    (
+        r"i think (.*)",
+        (
+            "Uncertainty in an uncertain world.",
+            "Indeed, how can we be certain of anything in such uncertain times.",
+            "Are you not, in fact, certain that%1?",
+        ),
+    ),
+    # "I feel...emotions/sick/light-headed..."
+    (
+        r"i feel (.*)",
+        (
+            "Your body and your emotions are both symptoms of your mind."
+            "What do you believe is the root of such feelings?",
+            "Feeling%1 can be a sign of your state-of-mind.",
+        ),
+    ),
+    # exclaimation mark indicating emotion
+    # e.g. "Wow!" or "No!"
+    (
+        r"(.*)!",
+        (
+            "I sense that you are feeling emotional today.",
+            "You need to calm your emotions.",
+        ),
+    ),
+    # because [statement]
+    # e.g. "because I said so"
+    (
+        r"because (.*)",
+        (
+            "Does knowning the reasons behind things help you to understand"
+            " the things themselves?",
+            "If%1, what else must be true?",
+        ),
+    ),
+    # yes or no - raise an issue of certainty/correctness
+    (
+        r"(yes)|(no)",
+        (
+            "Is there certainty in an uncertain world?",
+            "It is better to be right than to be certain.",
+        ),
+    ),
+    # sentence containing word 'love'
+    (
+        r"(.*)love(.*)",
+        (
+            "Think of the trees: they let the birds perch and fly with no intention to call them when they come, and no longing for their return when they fly away. Let your heart be like the trees.",
+            "Free love!",
+        ),
+    ),
+    # sentence containing word 'understand' - r
+    (
+        r"(.*)understand(.*)",
+        (
+            "If you understand, things are just as they are;"
+            " if you do not understand, things are just as they are.",
+            "Imagination is more important than knowledge.",
+        ),
+    ),
+    # 'I', 'me', 'my' - person is talking about themself.
+    # this breaks down when words contain these - eg 'Thyme', 'Irish'
+    (
+        r"(.*)(me )|( me)|(my)|(mine)|(i)(.*)",
+        (
+            "'I', 'me', 'my'... these are selfish expressions.",
+            "Have you ever considered that you might be a selfish person?",
+            "Try to consider others, not just yourself.",
+            "Think not just of yourself, but of others.",
+        ),
+    ),
+    # 'you' starting a sentence
+    # e.g. "you stink!"
+    (
+        r"you (.*)",
+        ("My path is not of concern to you.", "I am but one, and you but one more."),
+    ),
+    # say goodbye with some extra Zen wisdom.
+    (
+        r"exit",
+        (
+            "Farewell. The obstacle is the path.",
+            "Farewell. Life is a journey, not a destination.",
+            "Good bye. We are cups, constantly and quietly being filled."
+            "\nThe trick is knowning how to tip ourselves over and let the beautiful stuff out.",
+        ),
+    ),
+    # fall through case -
+    # when stumped, respond with generic zen wisdom
+    #
+    (
+        r"(.*)",
+        (
+            "When you're enlightened, every word is wisdom.",
+            "Random talk is useless.",
+            "The reverse side also has a reverse side.",
+            "Form is emptiness, and emptiness is form.",
+            "I pour out a cup of water. Is the cup empty?",
+        ),
+    ),
+)
+
+zen_chatbot = Chat(responses, reflections)
+
+
+def zen_chat():
+    print("*" * 75)
+    print("Zen Chatbot!".center(75))
+    print("*" * 75)
+    print('"Look beyond mere words and letters - look into your mind"'.center(75))
+    print("* Talk your way to truth with Zen Chatbot.")
+    print("* Type 'quit' when you have had enough.")
+    print("*" * 75)
+    print("Welcome, my child.")
+
+    zen_chatbot.converse()
+
+
+def demo():
+    zen_chat()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chunk/__init__.py

@@ -0,0 +1,197 @@
+# Natural Language Toolkit: Chunkers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Classes and interfaces for identifying non-overlapping linguistic
+groups (such as base noun phrases) in unrestricted text.  This task is
+called "chunk parsing" or "chunking", and the identified groups are
+called "chunks".  The chunked text is represented using a shallow
+tree called a "chunk structure."  A chunk structure is a tree
+containing tokens and chunks, where each chunk is a subtree containing
+only tokens.  For example, the chunk structure for base noun phrase
+chunks in the sentence "I saw the big dog on the hill" is::
+
+  (SENTENCE:
+    (NP: <I>)
+    <saw>
+    (NP: <the> <big> <dog>)
+    <on>
+    (NP: <the> <hill>))
+
+To convert a chunk structure back to a list of tokens, simply use the
+chunk structure's ``leaves()`` method.
+
+This module defines ``ChunkParserI``, a standard interface for
+chunking texts; and ``RegexpChunkParser``, a regular-expression based
+implementation of that interface. It also defines ``ChunkScore``, a
+utility class for scoring chunk parsers.
+
+RegexpChunkParser
+=================
+
+``RegexpChunkParser`` is an implementation of the chunk parser interface
+that uses regular-expressions over tags to chunk a text.  Its
+``parse()`` method first constructs a ``ChunkString``, which encodes a
+particular chunking of the input text.  Initially, nothing is
+chunked.  ``parse.RegexpChunkParser`` then applies a sequence of
+``RegexpChunkRule`` rules to the ``ChunkString``, each of which modifies
+the chunking that it encodes.  Finally, the ``ChunkString`` is
+transformed back into a chunk structure, which is returned.
+
+``RegexpChunkParser`` can only be used to chunk a single kind of phrase.
+For example, you can use an ``RegexpChunkParser`` to chunk the noun
+phrases in a text, or the verb phrases in a text; but you can not
+use it to simultaneously chunk both noun phrases and verb phrases in
+the same text.  (This is a limitation of ``RegexpChunkParser``, not of
+chunk parsers in general.)
+
+RegexpChunkRules
+----------------
+
+A ``RegexpChunkRule`` is a transformational rule that updates the
+chunking of a text by modifying its ``ChunkString``.  Each
+``RegexpChunkRule`` defines the ``apply()`` method, which modifies
+the chunking encoded by a ``ChunkString``.  The
+``RegexpChunkRule`` class itself can be used to implement any
+transformational rule based on regular expressions.  There are
+also a number of subclasses, which can be used to implement
+simpler types of rules:
+
+    - ``ChunkRule`` chunks anything that matches a given regular
+      expression.
+    - ``StripRule`` strips anything that matches a given regular
+      expression.
+    - ``UnChunkRule`` will un-chunk any chunk that matches a given
+      regular expression.
+    - ``MergeRule`` can be used to merge two contiguous chunks.
+    - ``SplitRule`` can be used to split a single chunk into two
+      smaller chunks.
+    - ``ExpandLeftRule`` will expand a chunk to incorporate new
+      unchunked material on the left.
+    - ``ExpandRightRule`` will expand a chunk to incorporate new
+      unchunked material on the right.
+
+Tag Patterns
+~~~~~~~~~~~~
+
+A ``RegexpChunkRule`` uses a modified version of regular
+expression patterns, called "tag patterns".  Tag patterns are
+used to match sequences of tags.  Examples of tag patterns are::
+
+     r'(<DT>|<JJ>|<NN>)+'
+     r'<NN>+'
+     r'<NN.*>'
+
+The differences between regular expression patterns and tag
+patterns are:
+
+    - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
+      ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
+      ``'<NN'`` followed by one or more repetitions of ``'>'``.
+    - Whitespace in tag patterns is ignored.  So
+      ``'<DT> | <NN>'`` is equivalent to ``'<DT>|<NN>'``
+    - In tag patterns, ``'.'`` is equivalent to ``'[^{}<>]'``; so
+      ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
+
+The function ``tag_pattern2re_pattern`` can be used to transform
+a tag pattern to an equivalent regular expression pattern.
+
+Efficiency
+----------
+
+Preliminary tests indicate that ``RegexpChunkParser`` can chunk at a
+rate of about 300 tokens/second, with a moderately complex rule set.
+
+There may be problems if ``RegexpChunkParser`` is used with more than
+5,000 tokens at a time.  In particular, evaluation of some regular
+expressions may cause the Python regular expression engine to
+exceed its maximum recursion depth.  We have attempted to minimize
+these problems, but it is impossible to avoid them completely.  We
+therefore recommend that you apply the chunk parser to a single
+sentence at a time.
+
+Emacs Tip
+---------
+
+If you evaluate the following elisp expression in emacs, it will
+colorize a ``ChunkString`` when you use an interactive python shell
+with emacs or xemacs ("C-c !")::
+
+    (let ()
+      (defconst comint-mode-font-lock-keywords
+        '(("<[^>]+>" 0 'font-lock-reference-face)
+          ("[{}]" 0 'font-lock-function-name-face)))
+      (add-hook 'comint-mode-hook (lambda () (turn-on-font-lock))))
+
+You can evaluate this code by copying it to a temporary buffer,
+placing the cursor after the last close parenthesis, and typing
+"``C-x C-e``".  You should evaluate it before running the interactive
+session.  The change will last until you close emacs.
+
+Unresolved Issues
+-----------------
+
+If we use the ``re`` module for regular expressions, Python's
+regular expression engine generates "maximum recursion depth
+exceeded" errors when processing very large texts, even for
+regular expressions that should not require any recursion.  We
+therefore use the ``pre`` module instead.  But note that ``pre``
+does not include Unicode support, so this module will not work
+with unicode strings.  Note also that ``pre`` regular expressions
+are not quite as advanced as ``re`` ones (e.g., no leftward
+zero-length assertions).
+
+:type CHUNK_TAG_PATTERN: regexp
+:var CHUNK_TAG_PATTERN: A regular expression to test whether a tag
+     pattern is valid.
+"""
+
+from nltk.chunk.api import ChunkParserI
+from nltk.chunk.regexp import RegexpChunkParser, RegexpParser
+from nltk.chunk.util import (
+    ChunkScore,
+    accuracy,
+    conllstr2tree,
+    conlltags2tree,
+    ieerstr2tree,
+    tagstr2tree,
+    tree2conllstr,
+    tree2conlltags,
+)
+from nltk.data import load
+
+# Standard treebank POS tagger
+_BINARY_NE_CHUNKER = "chunkers/maxent_ne_chunker/english_ace_binary.pickle"
+_MULTICLASS_NE_CHUNKER = "chunkers/maxent_ne_chunker/english_ace_multiclass.pickle"
+
+
+def ne_chunk(tagged_tokens, binary=False):
+    """
+    Use NLTK's currently recommended named entity chunker to
+    chunk the given list of tagged tokens.
+    """
+    if binary:
+        chunker_pickle = _BINARY_NE_CHUNKER
+    else:
+        chunker_pickle = _MULTICLASS_NE_CHUNKER
+    chunker = load(chunker_pickle)
+    return chunker.parse(tagged_tokens)
+
+
+def ne_chunk_sents(tagged_sentences, binary=False):
+    """
+    Use NLTK's currently recommended named entity chunker to chunk the
+    given list of tagged sentences, each consisting of a list of tagged tokens.
+    """
+    if binary:
+        chunker_pickle = _BINARY_NE_CHUNKER
+    else:
+        chunker_pickle = _MULTICLASS_NE_CHUNKER
+    chunker = load(chunker_pickle)
+    return chunker.parse_sents(tagged_sentences)

.eggs/nltk-3.8-py3.10.egg/nltk/chunk/api.py

@@ -0,0 +1,56 @@
+# Natural Language Toolkit: Chunk parsing API
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+##//////////////////////////////////////////////////////
+##  Chunk Parser Interface
+##//////////////////////////////////////////////////////
+
+from nltk.chunk.util import ChunkScore
+from nltk.internals import deprecated
+from nltk.parse import ParserI
+
+
+class ChunkParserI(ParserI):
+    """
+    A processing interface for identifying non-overlapping groups in
+    unrestricted text.  Typically, chunk parsers are used to find base
+    syntactic constituents, such as base noun phrases.  Unlike
+    ``ParserI``, ``ChunkParserI`` guarantees that the ``parse()`` method
+    will always generate a parse.
+    """
+
+    def parse(self, tokens):
+        """
+        Return the best chunk structure for the given tokens
+        and return a tree.
+
+        :param tokens: The list of (word, tag) tokens to be chunked.
+        :type tokens: list(tuple)
+        :rtype: Tree
+        """
+        raise NotImplementedError()
+
+    @deprecated("Use accuracy(gold) instead.")
+    def evaluate(self, gold):
+        return self.accuracy(gold)
+
+    def accuracy(self, gold):
+        """
+        Score the accuracy of the chunker against the gold standard.
+        Remove the chunking the gold standard text, rechunk it using
+        the chunker, and return a ``ChunkScore`` object
+        reflecting the performance of this chunk parser.
+
+        :type gold: list(Tree)
+        :param gold: The list of chunked sentences to score the chunker on.
+        :rtype: ChunkScore
+        """
+        chunkscore = ChunkScore()
+        for correct in gold:
+            chunkscore.score(correct, self.parse(correct.leaves()))
+        return chunkscore

.eggs/nltk-3.8-py3.10.egg/nltk/chunk/named_entity.py

@@ -0,0 +1,352 @@
+# Natural Language Toolkit: Chunk parsing API
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Named entity chunker
+"""
+
+import os
+import pickle
+import re
+from xml.etree import ElementTree as ET
+
+from nltk.tag import ClassifierBasedTagger, pos_tag
+
+try:
+    from nltk.classify import MaxentClassifier
+except ImportError:
+    pass
+
+from nltk.chunk.api import ChunkParserI
+from nltk.chunk.util import ChunkScore
+from nltk.data import find
+from nltk.tokenize import word_tokenize
+from nltk.tree import Tree
+
+
+class NEChunkParserTagger(ClassifierBasedTagger):
+    """
+    The IOB tagger used by the chunk parser.
+    """
+
+    def __init__(self, train):
+        ClassifierBasedTagger.__init__(
+            self, train=train, classifier_builder=self._classifier_builder
+        )
+
+    def _classifier_builder(self, train):
+        return MaxentClassifier.train(
+            train, algorithm="megam", gaussian_prior_sigma=1, trace=2
+        )
+
+    def _english_wordlist(self):
+        try:
+            wl = self._en_wordlist
+        except AttributeError:
+            from nltk.corpus import words
+
+            self._en_wordlist = set(words.words("en-basic"))
+            wl = self._en_wordlist
+        return wl
+
+    def _feature_detector(self, tokens, index, history):
+        word = tokens[index][0]
+        pos = simplify_pos(tokens[index][1])
+        if index == 0:
+            prevword = prevprevword = None
+            prevpos = prevprevpos = None
+            prevshape = prevtag = prevprevtag = None
+        elif index == 1:
+            prevword = tokens[index - 1][0].lower()
+            prevprevword = None
+            prevpos = simplify_pos(tokens[index - 1][1])
+            prevprevpos = None
+            prevtag = history[index - 1][0]
+            prevshape = prevprevtag = None
+        else:
+            prevword = tokens[index - 1][0].lower()
+            prevprevword = tokens[index - 2][0].lower()
+            prevpos = simplify_pos(tokens[index - 1][1])
+            prevprevpos = simplify_pos(tokens[index - 2][1])
+            prevtag = history[index - 1]
+            prevprevtag = history[index - 2]
+            prevshape = shape(prevword)
+        if index == len(tokens) - 1:
+            nextword = nextnextword = None
+            nextpos = nextnextpos = None
+        elif index == len(tokens) - 2:
+            nextword = tokens[index + 1][0].lower()
+            nextpos = tokens[index + 1][1].lower()
+            nextnextword = None
+            nextnextpos = None
+        else:
+            nextword = tokens[index + 1][0].lower()
+            nextpos = tokens[index + 1][1].lower()
+            nextnextword = tokens[index + 2][0].lower()
+            nextnextpos = tokens[index + 2][1].lower()
+
+        # 89.6
+        features = {
+            "bias": True,
+            "shape": shape(word),
+            "wordlen": len(word),
+            "prefix3": word[:3].lower(),
+            "suffix3": word[-3:].lower(),
+            "pos": pos,
+            "word": word,
+            "en-wordlist": (word in self._english_wordlist()),
+            "prevtag": prevtag,
+            "prevpos": prevpos,
+            "nextpos": nextpos,
+            "prevword": prevword,
+            "nextword": nextword,
+            "word+nextpos": f"{word.lower()}+{nextpos}",
+            "pos+prevtag": f"{pos}+{prevtag}",
+            "shape+prevtag": f"{prevshape}+{prevtag}",
+        }
+
+        return features
+
+
+class NEChunkParser(ChunkParserI):
+    """
+    Expected input: list of pos-tagged words
+    """
+
+    def __init__(self, train):
+        self._train(train)
+
+    def parse(self, tokens):
+        """
+        Each token should be a pos-tagged word
+        """
+        tagged = self._tagger.tag(tokens)
+        tree = self._tagged_to_parse(tagged)
+        return tree
+
+    def _train(self, corpus):
+        # Convert to tagged sequence
+        corpus = [self._parse_to_tagged(s) for s in corpus]
+
+        self._tagger = NEChunkParserTagger(train=corpus)
+
+    def _tagged_to_parse(self, tagged_tokens):
+        """
+        Convert a list of tagged tokens to a chunk-parse tree.
+        """
+        sent = Tree("S", [])
+
+        for (tok, tag) in tagged_tokens:
+            if tag == "O":
+                sent.append(tok)
+            elif tag.startswith("B-"):
+                sent.append(Tree(tag[2:], [tok]))
+            elif tag.startswith("I-"):
+                if sent and isinstance(sent[-1], Tree) and sent[-1].label() == tag[2:]:
+                    sent[-1].append(tok)
+                else:
+                    sent.append(Tree(tag[2:], [tok]))
+        return sent
+
+    @staticmethod
+    def _parse_to_tagged(sent):
+        """
+        Convert a chunk-parse tree to a list of tagged tokens.
+        """
+        toks = []
+        for child in sent:
+            if isinstance(child, Tree):
+                if len(child) == 0:
+                    print("Warning -- empty chunk in sentence")
+                    continue
+                toks.append((child[0], f"B-{child.label()}"))
+                for tok in child[1:]:
+                    toks.append((tok, f"I-{child.label()}"))
+            else:
+                toks.append((child, "O"))
+        return toks
+
+
+def shape(word):
+    if re.match(r"[0-9]+(\.[0-9]*)?|[0-9]*\.[0-9]+$", word, re.UNICODE):
+        return "number"
+    elif re.match(r"\W+$", word, re.UNICODE):
+        return "punct"
+    elif re.match(r"\w+$", word, re.UNICODE):
+        if word.istitle():
+            return "upcase"
+        elif word.islower():
+            return "downcase"
+        else:
+            return "mixedcase"
+    else:
+        return "other"
+
+
+def simplify_pos(s):
+    if s.startswith("V"):
+        return "V"
+    else:
+        return s.split("-")[0]
+
+
+def postag_tree(tree):
+    # Part-of-speech tagging.
+    words = tree.leaves()
+    tag_iter = (pos for (word, pos) in pos_tag(words))
+    newtree = Tree("S", [])
+    for child in tree:
+        if isinstance(child, Tree):
+            newtree.append(Tree(child.label(), []))
+            for subchild in child:
+                newtree[-1].append((subchild, next(tag_iter)))
+        else:
+            newtree.append((child, next(tag_iter)))
+    return newtree
+
+
+def load_ace_data(roots, fmt="binary", skip_bnews=True):
+    for root in roots:
+        for root, dirs, files in os.walk(root):
+            if root.endswith("bnews") and skip_bnews:
+                continue
+            for f in files:
+                if f.endswith(".sgm"):
+                    yield from load_ace_file(os.path.join(root, f), fmt)
+
+
+def load_ace_file(textfile, fmt):
+    print(f"  - {os.path.split(textfile)[1]}")
+    annfile = textfile + ".tmx.rdc.xml"
+
+    # Read the xml file, and get a list of entities
+    entities = []
+    with open(annfile) as infile:
+        xml = ET.parse(infile).getroot()
+    for entity in xml.findall("document/entity"):
+        typ = entity.find("entity_type").text
+        for mention in entity.findall("entity_mention"):
+            if mention.get("TYPE") != "NAME":
+                continue  # only NEs
+            s = int(mention.find("head/charseq/start").text)
+            e = int(mention.find("head/charseq/end").text) + 1
+            entities.append((s, e, typ))
+
+    # Read the text file, and mark the entities.
+    with open(textfile) as infile:
+        text = infile.read()
+
+    # Strip XML tags, since they don't count towards the indices
+    text = re.sub("<(?!/?TEXT)[^>]+>", "", text)
+
+    # Blank out anything before/after <TEXT>
+    def subfunc(m):
+        return " " * (m.end() - m.start() - 6)
+
+    text = re.sub(r"[\s\S]*<TEXT>", subfunc, text)
+    text = re.sub(r"</TEXT>[\s\S]*", "", text)
+
+    # Simplify quotes
+    text = re.sub("``", ' "', text)
+    text = re.sub("''", '" ', text)
+
+    entity_types = {typ for (s, e, typ) in entities}
+
+    # Binary distinction (NE or not NE)
+    if fmt == "binary":
+        i = 0
+        toks = Tree("S", [])
+        for (s, e, typ) in sorted(entities):
+            if s < i:
+                s = i  # Overlapping!  Deal with this better?
+            if e <= s:
+                continue
+            toks.extend(word_tokenize(text[i:s]))
+            toks.append(Tree("NE", text[s:e].split()))
+            i = e
+        toks.extend(word_tokenize(text[i:]))
+        yield toks
+
+    # Multiclass distinction (NE type)
+    elif fmt == "multiclass":
+        i = 0
+        toks = Tree("S", [])
+        for (s, e, typ) in sorted(entities):
+            if s < i:
+                s = i  # Overlapping!  Deal with this better?
+            if e <= s:
+                continue
+            toks.extend(word_tokenize(text[i:s]))
+            toks.append(Tree(typ, text[s:e].split()))
+            i = e
+        toks.extend(word_tokenize(text[i:]))
+        yield toks
+
+    else:
+        raise ValueError("bad fmt value")
+
+
+# This probably belongs in a more general-purpose location (as does
+# the parse_to_tagged function).
+def cmp_chunks(correct, guessed):
+    correct = NEChunkParser._parse_to_tagged(correct)
+    guessed = NEChunkParser._parse_to_tagged(guessed)
+    ellipsis = False
+    for (w, ct), (w, gt) in zip(correct, guessed):
+        if ct == gt == "O":
+            if not ellipsis:
+                print(f"  {ct:15} {gt:15} {w}")
+                print("  {:15} {:15} {2}".format("...", "...", "..."))
+                ellipsis = True
+        else:
+            ellipsis = False
+            print(f"  {ct:15} {gt:15} {w}")
+
+
+def build_model(fmt="binary"):
+    print("Loading training data...")
+    train_paths = [
+        find("corpora/ace_data/ace.dev"),
+        find("corpora/ace_data/ace.heldout"),
+        find("corpora/ace_data/bbn.dev"),
+        find("corpora/ace_data/muc.dev"),
+    ]
+    train_trees = load_ace_data(train_paths, fmt)
+    train_data = [postag_tree(t) for t in train_trees]
+    print("Training...")
+    cp = NEChunkParser(train_data)
+    del train_data
+
+    print("Loading eval data...")
+    eval_paths = [find("corpora/ace_data/ace.eval")]
+    eval_trees = load_ace_data(eval_paths, fmt)
+    eval_data = [postag_tree(t) for t in eval_trees]
+
+    print("Evaluating...")
+    chunkscore = ChunkScore()
+    for i, correct in enumerate(eval_data):
+        guess = cp.parse(correct.leaves())
+        chunkscore.score(correct, guess)
+        if i < 3:
+            cmp_chunks(correct, guess)
+    print(chunkscore)
+
+    outfilename = f"/tmp/ne_chunker_{fmt}.pickle"
+    print(f"Saving chunker to {outfilename}...")
+
+    with open(outfilename, "wb") as outfile:
+        pickle.dump(cp, outfile, -1)
+
+    return cp
+
+
+if __name__ == "__main__":
+    # Make sure that the pickled object has the right class name:
+    from nltk.chunk.named_entity import build_model
+
+    build_model("binary")
+    build_model("multiclass")

.eggs/nltk-3.8-py3.10.egg/nltk/chunk/regexp.py

@@ -0,0 +1,1475 @@
+# Natural Language Toolkit: Regular Expression Chunkers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+
+import regex
+
+from nltk.chunk.api import ChunkParserI
+from nltk.tree import Tree
+
+# //////////////////////////////////////////////////////
+# ChunkString
+# //////////////////////////////////////////////////////
+
+
+class ChunkString:
+    """
+    A string-based encoding of a particular chunking of a text.
+    Internally, the ``ChunkString`` class uses a single string to
+    encode the chunking of the input text.  This string contains a
+    sequence of angle-bracket delimited tags, with chunking indicated
+    by braces.  An example of this encoding is::
+
+        {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+    ``ChunkString`` are created from tagged texts (i.e., lists of
+    ``tokens`` whose type is ``TaggedType``).  Initially, nothing is
+    chunked.
+
+    The chunking of a ``ChunkString`` can be modified with the ``xform()``
+    method, which uses a regular expression to transform the string
+    representation.  These transformations should only add and remove
+    braces; they should *not* modify the sequence of angle-bracket
+    delimited tags.
+
+    :type _str: str
+    :ivar _str: The internal string representation of the text's
+        encoding.  This string representation contains a sequence of
+        angle-bracket delimited tags, with chunking indicated by
+        braces.  An example of this encoding is::
+
+            {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+    :type _pieces: list(tagged tokens and chunks)
+    :ivar _pieces: The tagged tokens and chunks encoded by this ``ChunkString``.
+    :ivar _debug: The debug level.  See the constructor docs.
+
+    :cvar IN_CHUNK_PATTERN: A zero-width regexp pattern string that
+        will only match positions that are in chunks.
+    :cvar IN_STRIP_PATTERN: A zero-width regexp pattern string that
+        will only match positions that are in strips.
+    """
+
+    CHUNK_TAG_CHAR = r"[^\{\}<>]"
+    CHUNK_TAG = r"(<%s+?>)" % CHUNK_TAG_CHAR
+
+    IN_CHUNK_PATTERN = r"(?=[^\{]*\})"
+    IN_STRIP_PATTERN = r"(?=[^\}]*(\{|$))"
+
+    # These are used by _verify
+    _CHUNK = r"(\{%s+?\})+?" % CHUNK_TAG
+    _STRIP = r"(%s+?)+?" % CHUNK_TAG
+    _VALID = re.compile(r"^(\{?%s\}?)*?$" % CHUNK_TAG)
+    _BRACKETS = re.compile(r"[^\{\}]+")
+    _BALANCED_BRACKETS = re.compile(r"(\{\})*$")
+
+    def __init__(self, chunk_struct, debug_level=1):
+        """
+        Construct a new ``ChunkString`` that encodes the chunking of
+        the text ``tagged_tokens``.
+
+        :type chunk_struct: Tree
+        :param chunk_struct: The chunk structure to be further chunked.
+        :type debug_level: int
+        :param debug_level: The level of debugging which should be
+            applied to transformations on the ``ChunkString``.  The
+            valid levels are:
+
+                - 0: no checks
+                - 1: full check on to_chunkstruct
+                - 2: full check on to_chunkstruct and cursory check after
+                  each transformation.
+                - 3: full check on to_chunkstruct and full check after
+                  each transformation.
+
+            We recommend you use at least level 1.  You should
+            probably use level 3 if you use any non-standard
+            subclasses of ``RegexpChunkRule``.
+        """
+        self._root_label = chunk_struct.label()
+        self._pieces = chunk_struct[:]
+        tags = [self._tag(tok) for tok in self._pieces]
+        self._str = "<" + "><".join(tags) + ">"
+        self._debug = debug_level
+
+    def _tag(self, tok):
+        if isinstance(tok, tuple):
+            return tok[1]
+        elif isinstance(tok, Tree):
+            return tok.label()
+        else:
+            raise ValueError("chunk structures must contain tagged " "tokens or trees")
+
+    def _verify(self, s, verify_tags):
+        """
+        Check to make sure that ``s`` still corresponds to some chunked
+        version of ``_pieces``.
+
+        :type verify_tags: bool
+        :param verify_tags: Whether the individual tags should be
+            checked.  If this is false, ``_verify`` will check to make
+            sure that ``_str`` encodes a chunked version of *some*
+            list of tokens.  If this is true, then ``_verify`` will
+            check to make sure that the tags in ``_str`` match those in
+            ``_pieces``.
+
+        :raise ValueError: if the internal string representation of
+            this ``ChunkString`` is invalid or not consistent with _pieces.
+        """
+        # Check overall form
+        if not ChunkString._VALID.match(s):
+            raise ValueError(
+                "Transformation generated invalid " "chunkstring:\n  %s" % s
+            )
+
+        # Check that parens are balanced.  If the string is long, we
+        # have to do this in pieces, to avoid a maximum recursion
+        # depth limit for regular expressions.
+        brackets = ChunkString._BRACKETS.sub("", s)
+        for i in range(1 + len(brackets) // 5000):
+            substr = brackets[i * 5000 : i * 5000 + 5000]
+            if not ChunkString._BALANCED_BRACKETS.match(substr):
+                raise ValueError(
+                    "Transformation generated invalid " "chunkstring:\n  %s" % s
+                )
+
+        if verify_tags <= 0:
+            return
+
+        tags1 = (re.split(r"[\{\}<>]+", s))[1:-1]
+        tags2 = [self._tag(piece) for piece in self._pieces]
+        if tags1 != tags2:
+            raise ValueError(
+                "Transformation generated invalid " "chunkstring: tag changed"
+            )
+
+    def to_chunkstruct(self, chunk_label="CHUNK"):
+        """
+        Return the chunk structure encoded by this ``ChunkString``.
+
+        :rtype: Tree
+        :raise ValueError: If a transformation has generated an
+            invalid chunkstring.
+        """
+        if self._debug > 0:
+            self._verify(self._str, 1)
+
+        # Use this alternating list to create the chunkstruct.
+        pieces = []
+        index = 0
+        piece_in_chunk = 0
+        for piece in re.split("[{}]", self._str):
+
+            # Find the list of tokens contained in this piece.
+            length = piece.count("<")
+            subsequence = self._pieces[index : index + length]
+
+            # Add this list of tokens to our pieces.
+            if piece_in_chunk:
+                pieces.append(Tree(chunk_label, subsequence))
+            else:
+                pieces += subsequence
+
+            # Update index, piece_in_chunk
+            index += length
+            piece_in_chunk = not piece_in_chunk
+
+        return Tree(self._root_label, pieces)
+
+    def xform(self, regexp, repl):
+        """
+        Apply the given transformation to the string encoding of this
+        ``ChunkString``.  In particular, find all occurrences that match
+        ``regexp``, and replace them using ``repl`` (as done by
+        ``re.sub``).
+
+        This transformation should only add and remove braces; it
+        should *not* modify the sequence of angle-bracket delimited
+        tags.  Furthermore, this transformation may not result in
+        improper bracketing.  Note, in particular, that bracketing may
+        not be nested.
+
+        :type regexp: str or regexp
+        :param regexp: A regular expression matching the substring
+            that should be replaced.  This will typically include a
+            named group, which can be used by ``repl``.
+        :type repl: str
+        :param repl: An expression specifying what should replace the
+            matched substring.  Typically, this will include a named
+            replacement group, specified by ``regexp``.
+        :rtype: None
+        :raise ValueError: If this transformation generated an
+            invalid chunkstring.
+        """
+        # Do the actual substitution
+        s = re.sub(regexp, repl, self._str)
+
+        # The substitution might have generated "empty chunks"
+        # (substrings of the form "{}").  Remove them, so they don't
+        # interfere with other transformations.
+        s = re.sub(r"\{\}", "", s)
+
+        # Make sure that the transformation was legal.
+        if self._debug > 1:
+            self._verify(s, self._debug - 2)
+
+        # Commit the transformation.
+        self._str = s
+
+    def __repr__(self):
+        """
+        Return a string representation of this ``ChunkString``.
+        It has the form::
+
+            <ChunkString: '{<DT><JJ><NN>}<VBN><IN>{<DT><NN>}'>
+
+        :rtype: str
+        """
+        return "<ChunkString: %s>" % repr(self._str)
+
+    def __str__(self):
+        """
+        Return a formatted representation of this ``ChunkString``.
+        This representation will include extra spaces to ensure that
+        tags will line up with the representation of other
+        ``ChunkStrings`` for the same text, regardless of the chunking.
+
+        :rtype: str
+        """
+        # Add spaces to make everything line up.
+        str = re.sub(r">(?!\})", r"> ", self._str)
+        str = re.sub(r"([^\{])<", r"\1 <", str)
+        if str[0] == "<":
+            str = " " + str
+        return str
+
+
+# //////////////////////////////////////////////////////
+# Chunking Rules
+# //////////////////////////////////////////////////////
+
+
+class RegexpChunkRule:
+    """
+    A rule specifying how to modify the chunking in a ``ChunkString``,
+    using a transformational regular expression.  The
+    ``RegexpChunkRule`` class itself can be used to implement any
+    transformational rule based on regular expressions.  There are
+    also a number of subclasses, which can be used to implement
+    simpler types of rules, based on matching regular expressions.
+
+    Each ``RegexpChunkRule`` has a regular expression and a
+    replacement expression.  When a ``RegexpChunkRule`` is "applied"
+    to a ``ChunkString``, it searches the ``ChunkString`` for any
+    substring that matches the regular expression, and replaces it
+    using the replacement expression.  This search/replace operation
+    has the same semantics as ``re.sub``.
+
+    Each ``RegexpChunkRule`` also has a description string, which
+    gives a short (typically less than 75 characters) description of
+    the purpose of the rule.
+
+    This transformation defined by this ``RegexpChunkRule`` should
+    only add and remove braces; it should *not* modify the sequence
+    of angle-bracket delimited tags.  Furthermore, this transformation
+    may not result in nested or mismatched bracketing.
+    """
+
+    def __init__(self, regexp, repl, descr):
+        """
+        Construct a new RegexpChunkRule.
+
+        :type regexp: regexp or str
+        :param regexp: The regular expression for this ``RegexpChunkRule``.
+            When this rule is applied to a ``ChunkString``, any
+            substring that matches ``regexp`` will be replaced using
+            the replacement string ``repl``.  Note that this must be a
+            normal regular expression, not a tag pattern.
+        :type repl: str
+        :param repl: The replacement expression for this ``RegexpChunkRule``.
+            When this rule is applied to a ``ChunkString``, any substring
+            that matches ``regexp`` will be replaced using ``repl``.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        if isinstance(regexp, str):
+            regexp = re.compile(regexp)
+        self._repl = repl
+        self._descr = descr
+        self._regexp = regexp
+
+    def apply(self, chunkstr):
+        # Keep docstring generic so we can inherit it.
+        """
+        Apply this rule to the given ``ChunkString``.  See the
+        class reference documentation for a description of what it
+        means to apply a rule.
+
+        :type chunkstr: ChunkString
+        :param chunkstr: The chunkstring to which this rule is applied.
+        :rtype: None
+        :raise ValueError: If this transformation generated an
+            invalid chunkstring.
+        """
+        chunkstr.xform(self._regexp, self._repl)
+
+    def descr(self):
+        """
+        Return a short description of the purpose and/or effect of
+        this rule.
+
+        :rtype: str
+        """
+        return self._descr
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <RegexpChunkRule: '{<IN|VB.*>}'->'<IN>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return (
+            "<RegexpChunkRule: "
+            + repr(self._regexp.pattern)
+            + "->"
+            + repr(self._repl)
+            + ">"
+        )
+
+    @staticmethod
+    def fromstring(s):
+        """
+        Create a RegexpChunkRule from a string description.
+        Currently, the following formats are supported::
+
+          {regexp}         # chunk rule
+          }regexp{         # strip rule
+          regexp}{regexp   # split rule
+          regexp{}regexp   # merge rule
+
+        Where ``regexp`` is a regular expression for the rule.  Any
+        text following the comment marker (``#``) will be used as
+        the rule's description:
+
+        >>> from nltk.chunk.regexp import RegexpChunkRule
+        >>> RegexpChunkRule.fromstring('{<DT>?<NN.*>+}')
+        <ChunkRule: '<DT>?<NN.*>+'>
+        """
+        # Split off the comment (but don't split on '\#')
+        m = re.match(r"(?P<rule>(\\.|[^#])*)(?P<comment>#.*)?", s)
+        rule = m.group("rule").strip()
+        comment = (m.group("comment") or "")[1:].strip()
+
+        # Pattern bodies: chunk, strip, split, merge
+        try:
+            if not rule:
+                raise ValueError("Empty chunk pattern")
+            if rule[0] == "{" and rule[-1] == "}":
+                return ChunkRule(rule[1:-1], comment)
+            elif rule[0] == "}" and rule[-1] == "{":
+                return StripRule(rule[1:-1], comment)
+            elif "}{" in rule:
+                left, right = rule.split("}{")
+                return SplitRule(left, right, comment)
+            elif "{}" in rule:
+                left, right = rule.split("{}")
+                return MergeRule(left, right, comment)
+            elif re.match("[^{}]*{[^{}]*}[^{}]*", rule):
+                left, chunk, right = re.split("[{}]", rule)
+                return ChunkRuleWithContext(left, chunk, right, comment)
+            else:
+                raise ValueError("Illegal chunk pattern: %s" % rule)
+        except (ValueError, re.error) as e:
+            raise ValueError("Illegal chunk pattern: %s" % rule) from e
+
+
+class ChunkRule(RegexpChunkRule):
+    """
+    A rule specifying how to add chunks to a ``ChunkString``, using a
+    matching tag pattern.  When applied to a ``ChunkString``, it will
+    find any substring that matches this tag pattern and that is not
+    already part of a chunk, and create a new chunk containing that
+    substring.
+    """
+
+    def __init__(self, tag_pattern, descr):
+        """
+        Construct a new ``ChunkRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            chunk any substring that matches this tag pattern and that
+            is not already part of a chunk.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile(
+            "(?P<chunk>%s)%s"
+            % (tag_pattern2re_pattern(tag_pattern), ChunkString.IN_STRIP_PATTERN)
+        )
+        RegexpChunkRule.__init__(self, regexp, r"{\g<chunk>}", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ChunkRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return "<ChunkRule: " + repr(self._pattern) + ">"
+
+
+class StripRule(RegexpChunkRule):
+    """
+    A rule specifying how to remove strips to a ``ChunkString``,
+    using a matching tag pattern.  When applied to a
+    ``ChunkString``, it will find any substring that matches this
+    tag pattern and that is contained in a chunk, and remove it
+    from that chunk, thus creating two new chunks.
+    """
+
+    def __init__(self, tag_pattern, descr):
+        """
+        Construct a new ``StripRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            find any substring that matches this tag pattern and that
+            is contained in a chunk, and remove it from that chunk,
+            thus creating two new chunks.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile(
+            "(?P<strip>%s)%s"
+            % (tag_pattern2re_pattern(tag_pattern), ChunkString.IN_CHUNK_PATTERN)
+        )
+        RegexpChunkRule.__init__(self, regexp, r"}\g<strip>{", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <StripRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return "<StripRule: " + repr(self._pattern) + ">"
+
+
+class UnChunkRule(RegexpChunkRule):
+    """
+    A rule specifying how to remove chunks to a ``ChunkString``,
+    using a matching tag pattern.  When applied to a
+    ``ChunkString``, it will find any complete chunk that matches this
+    tag pattern, and un-chunk it.
+    """
+
+    def __init__(self, tag_pattern, descr):
+        """
+        Construct a new ``UnChunkRule``.
+
+        :type tag_pattern: str
+        :param tag_pattern: This rule's tag pattern.  When
+            applied to a ``ChunkString``, this rule will
+            find any complete chunk that matches this tag pattern,
+            and un-chunk it.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        self._pattern = tag_pattern
+        regexp = re.compile(r"\{(?P<chunk>%s)\}" % tag_pattern2re_pattern(tag_pattern))
+        RegexpChunkRule.__init__(self, regexp, r"\g<chunk>", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <UnChunkRule: '<IN|VB.*>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return "<UnChunkRule: " + repr(self._pattern) + ">"
+
+
+class MergeRule(RegexpChunkRule):
+    """
+    A rule specifying how to merge chunks in a ``ChunkString``, using
+    two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk whose end
+    matches left pattern, and immediately followed by a chunk whose
+    beginning matches right pattern.  It will then merge those two
+    chunks into a single chunk.
+    """
+
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``MergeRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            ``left_tag_pattern``, and immediately followed by a chunk
+            whose beginning matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            this pattern, and immediately followed by a chunk
+            whose beginning matches ``right_tag_pattern``.  It will
+            then merge those two chunks into a single chunk.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile(
+            "(?P<left>%s)}{(?=%s)"
+            % (
+                tag_pattern2re_pattern(left_tag_pattern),
+                tag_pattern2re_pattern(right_tag_pattern),
+            )
+        )
+        RegexpChunkRule.__init__(self, regexp, r"\g<left>", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <MergeRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return (
+            "<MergeRule: "
+            + repr(self._left_tag_pattern)
+            + ", "
+            + repr(self._right_tag_pattern)
+            + ">"
+        )
+
+
+class SplitRule(RegexpChunkRule):
+    """
+    A rule specifying how to split chunks in a ``ChunkString``, using
+    two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk that
+    matches the left pattern followed by the right pattern.  It will
+    then split the chunk into two new chunks, at the point between the
+    two pattern matches.
+    """
+
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``SplitRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this rule will
+            find any chunk containing a substring that matches
+            ``left_tag_pattern`` followed by this pattern.  It will
+            then split the chunk into two new chunks at the point
+            between these two matching patterns.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this rule will
+            find any chunk containing a substring that matches this
+            pattern followed by ``right_tag_pattern``.  It will then
+            split the chunk into two new chunks at the point between
+            these two matching patterns.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile(
+            "(?P<left>%s)(?=%s)"
+            % (
+                tag_pattern2re_pattern(left_tag_pattern),
+                tag_pattern2re_pattern(right_tag_pattern),
+            )
+        )
+        RegexpChunkRule.__init__(self, regexp, r"\g<left>}{", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <SplitRule: '<NN>', '<DT>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return (
+            "<SplitRule: "
+            + repr(self._left_tag_pattern)
+            + ", "
+            + repr(self._right_tag_pattern)
+            + ">"
+        )
+
+
+class ExpandLeftRule(RegexpChunkRule):
+    """
+    A rule specifying how to expand chunks in a ``ChunkString`` to the left,
+    using two matching tag patterns: a left pattern, and a right pattern.
+    When applied to a ``ChunkString``, it will find any chunk whose beginning
+    matches right pattern, and immediately preceded by a strip whose
+    end matches left pattern.  It will then expand the chunk to incorporate
+    the new material on the left.
+    """
+
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``ExpandRightRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose beginning matches
+            ``right_tag_pattern``, and immediately preceded by a strip
+            whose end matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose beginning matches
+            this pattern, and immediately preceded by a strip
+            whose end matches ``left_tag_pattern``.  It will
+            then expand the chunk to incorporate the new material on the left.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile(
+            r"(?P<left>%s)\{(?P<right>%s)"
+            % (
+                tag_pattern2re_pattern(left_tag_pattern),
+                tag_pattern2re_pattern(right_tag_pattern),
+            )
+        )
+        RegexpChunkRule.__init__(self, regexp, r"{\g<left>\g<right>", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ExpandLeftRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return (
+            "<ExpandLeftRule: "
+            + repr(self._left_tag_pattern)
+            + ", "
+            + repr(self._right_tag_pattern)
+            + ">"
+        )
+
+
+class ExpandRightRule(RegexpChunkRule):
+    """
+    A rule specifying how to expand chunks in a ``ChunkString`` to the
+    right, using two matching tag patterns: a left pattern, and a
+    right pattern.  When applied to a ``ChunkString``, it will find any
+    chunk whose end matches left pattern, and immediately followed by
+    a strip whose beginning matches right pattern.  It will then
+    expand the chunk to incorporate the new material on the right.
+    """
+
+    def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+        """
+        Construct a new ``ExpandRightRule``.
+
+        :type right_tag_pattern: str
+        :param right_tag_pattern: This rule's right tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            ``left_tag_pattern``, and immediately followed by a strip
+            whose beginning matches this pattern.  It will
+            then merge those two chunks into a single chunk.
+        :type left_tag_pattern: str
+        :param left_tag_pattern: This rule's left tag
+            pattern.  When applied to a ``ChunkString``, this
+            rule will find any chunk whose end matches
+            this pattern, and immediately followed by a strip
+            whose beginning matches ``right_tag_pattern``.  It will
+            then expand the chunk to incorporate the new material on the right.
+
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+        self._left_tag_pattern = left_tag_pattern
+        self._right_tag_pattern = right_tag_pattern
+        regexp = re.compile(
+            r"(?P<left>%s)\}(?P<right>%s)"
+            % (
+                tag_pattern2re_pattern(left_tag_pattern),
+                tag_pattern2re_pattern(right_tag_pattern),
+            )
+        )
+        RegexpChunkRule.__init__(self, regexp, r"\g<left>\g<right>}", descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ExpandRightRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return (
+            "<ExpandRightRule: "
+            + repr(self._left_tag_pattern)
+            + ", "
+            + repr(self._right_tag_pattern)
+            + ">"
+        )
+
+
+class ChunkRuleWithContext(RegexpChunkRule):
+    """
+    A rule specifying how to add chunks to a ``ChunkString``, using
+    three matching tag patterns: one for the left context, one for the
+    chunk, and one for the right context.  When applied to a
+    ``ChunkString``, it will find any substring that matches the chunk
+    tag pattern, is surrounded by substrings that match the two
+    context patterns, and is not already part of a chunk; and create a
+    new chunk containing the substring that matched the chunk tag
+    pattern.
+
+    Caveat: Both the left and right context are consumed when this
+    rule matches; therefore, if you need to find overlapping matches,
+    you will need to apply your rule more than once.
+    """
+
+    def __init__(
+        self,
+        left_context_tag_pattern,
+        chunk_tag_pattern,
+        right_context_tag_pattern,
+        descr,
+    ):
+        """
+        Construct a new ``ChunkRuleWithContext``.
+
+        :type left_context_tag_pattern: str
+        :param left_context_tag_pattern: A tag pattern that must match
+            the left context of ``chunk_tag_pattern`` for this rule to
+            apply.
+        :type chunk_tag_pattern: str
+        :param chunk_tag_pattern: A tag pattern that must match for this
+            rule to apply.  If the rule does apply, then this pattern
+            also identifies the substring that will be made into a chunk.
+        :type right_context_tag_pattern: str
+        :param right_context_tag_pattern: A tag pattern that must match
+            the right context of ``chunk_tag_pattern`` for this rule to
+            apply.
+        :type descr: str
+        :param descr: A short description of the purpose and/or effect
+            of this rule.
+        """
+        # Ensure that the individual patterns are coherent.  E.g., if
+        # left='(' and right=')', then this will raise an exception:
+        re.compile(tag_pattern2re_pattern(left_context_tag_pattern))
+        re.compile(tag_pattern2re_pattern(chunk_tag_pattern))
+        re.compile(tag_pattern2re_pattern(right_context_tag_pattern))
+
+        self._left_context_tag_pattern = left_context_tag_pattern
+        self._chunk_tag_pattern = chunk_tag_pattern
+        self._right_context_tag_pattern = right_context_tag_pattern
+        regexp = re.compile(
+            "(?P<left>%s)(?P<chunk>%s)(?P<right>%s)%s"
+            % (
+                tag_pattern2re_pattern(left_context_tag_pattern),
+                tag_pattern2re_pattern(chunk_tag_pattern),
+                tag_pattern2re_pattern(right_context_tag_pattern),
+                ChunkString.IN_STRIP_PATTERN,
+            )
+        )
+        replacement = r"\g<left>{\g<chunk>}\g<right>"
+        RegexpChunkRule.__init__(self, regexp, replacement, descr)
+
+    def __repr__(self):
+        """
+        Return a string representation of this rule.  It has the form::
+
+            <ChunkRuleWithContext: '<IN>', '<NN>', '<DT>'>
+
+        Note that this representation does not include the
+        description string; that string can be accessed
+        separately with the ``descr()`` method.
+
+        :rtype: str
+        """
+        return "<ChunkRuleWithContext:  {!r}, {!r}, {!r}>".format(
+            self._left_context_tag_pattern,
+            self._chunk_tag_pattern,
+            self._right_context_tag_pattern,
+        )
+
+
+# //////////////////////////////////////////////////////
+# Tag Pattern Format Conversion
+# //////////////////////////////////////////////////////
+
+# this should probably be made more strict than it is -- e.g., it
+# currently accepts 'foo'.
+CHUNK_TAG_PATTERN = re.compile(
+    r"^(({}|<{}>)*)$".format(r"([^\{\}<>]|\{\d+,?\}|\{\d*,\d+\})+", r"[^\{\}<>]+")
+)
+
+
+def tag_pattern2re_pattern(tag_pattern):
+    """
+    Convert a tag pattern to a regular expression pattern.  A "tag
+    pattern" is a modified version of a regular expression, designed
+    for matching sequences of tags.  The differences between regular
+    expression patterns and tag patterns are:
+
+        - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
+          ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
+          ``'<NN'`` followed by one or more repetitions of ``'>'``.
+        - Whitespace in tag patterns is ignored.  So
+          ``'<DT> | <NN>'`` is equivalent to ``'<DT>|<NN>'``
+        - In tag patterns, ``'.'`` is equivalent to ``'[^{}<>]'``; so
+          ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
+
+    In particular, ``tag_pattern2re_pattern`` performs the following
+    transformations on the given pattern:
+
+        - Replace '.' with '[^<>{}]'
+        - Remove any whitespace
+        - Add extra parens around '<' and '>', to make '<' and '>' act
+          like parentheses.  E.g., so that in '<NN>+', the '+' has scope
+          over the entire '<NN>'; and so that in '<NN|IN>', the '|' has
+          scope over 'NN' and 'IN', but not '<' or '>'.
+        - Check to make sure the resulting pattern is valid.
+
+    :type tag_pattern: str
+    :param tag_pattern: The tag pattern to convert to a regular
+        expression pattern.
+    :raise ValueError: If ``tag_pattern`` is not a valid tag pattern.
+        In particular, ``tag_pattern`` should not include braces; and it
+        should not contain nested or mismatched angle-brackets.
+    :rtype: str
+    :return: A regular expression pattern corresponding to
+        ``tag_pattern``.
+    """
+    # Clean up the regular expression
+    tag_pattern = re.sub(r"\s", "", tag_pattern)
+    tag_pattern = re.sub(r"<", "(<(", tag_pattern)
+    tag_pattern = re.sub(r">", ")>)", tag_pattern)
+
+    # Check the regular expression
+    if not CHUNK_TAG_PATTERN.match(tag_pattern):
+        raise ValueError("Bad tag pattern: %r" % tag_pattern)
+
+    # Replace "." with CHUNK_TAG_CHAR.
+    # We have to do this after, since it adds {}[]<>s, which would
+    # confuse CHUNK_TAG_PATTERN.
+    # PRE doesn't have lookback assertions, so reverse twice, and do
+    # the pattern backwards (with lookahead assertions).  This can be
+    # made much cleaner once we can switch back to SRE.
+    def reverse_str(str):
+        lst = list(str)
+        lst.reverse()
+        return "".join(lst)
+
+    tc_rev = reverse_str(ChunkString.CHUNK_TAG_CHAR)
+    reversed = reverse_str(tag_pattern)
+    reversed = re.sub(r"\.(?!\\(\\\\)*($|[^\\]))", tc_rev, reversed)
+    tag_pattern = reverse_str(reversed)
+
+    return tag_pattern
+
+
+# //////////////////////////////////////////////////////
+# RegexpChunkParser
+# //////////////////////////////////////////////////////
+
+
+class RegexpChunkParser(ChunkParserI):
+    """
+    A regular expression based chunk parser.  ``RegexpChunkParser`` uses a
+    sequence of "rules" to find chunks of a single type within a
+    text.  The chunking of the text is encoded using a ``ChunkString``,
+    and each rule acts by modifying the chunking in the
+    ``ChunkString``.  The rules are all implemented using regular
+    expression matching and substitution.
+
+    The ``RegexpChunkRule`` class and its subclasses (``ChunkRule``,
+    ``StripRule``, ``UnChunkRule``, ``MergeRule``, and ``SplitRule``)
+    define the rules that are used by ``RegexpChunkParser``.  Each rule
+    defines an ``apply()`` method, which modifies the chunking encoded
+    by a given ``ChunkString``.
+
+    :type _rules: list(RegexpChunkRule)
+    :ivar _rules: The list of rules that should be applied to a text.
+    :type _trace: int
+    :ivar _trace: The default level of tracing.
+
+    """
+
+    def __init__(self, rules, chunk_label="NP", root_label="S", trace=0):
+        """
+        Construct a new ``RegexpChunkParser``.
+
+        :type rules: list(RegexpChunkRule)
+        :param rules: The sequence of rules that should be used to
+            generate the chunking for a tagged text.
+        :type chunk_label: str
+        :param chunk_label: The node value that should be used for
+            chunk subtrees.  This is typically a short string
+            describing the type of information contained by the chunk,
+            such as ``"NP"`` for base noun phrases.
+        :type root_label: str
+        :param root_label: The node value that should be used for the
+            top node of the chunk structure.
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.
+        """
+        self._rules = rules
+        self._trace = trace
+        self._chunk_label = chunk_label
+        self._root_label = root_label
+
+    def _trace_apply(self, chunkstr, verbose):
+        """
+        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+        turn.  Generate trace output between each rule.  If ``verbose``
+        is true, then generate verbose output.
+
+        :type chunkstr: ChunkString
+        :param chunkstr: The chunk string to which each rule should be
+            applied.
+        :type verbose: bool
+        :param verbose: Whether output should be verbose.
+        :rtype: None
+        """
+        print("# Input:")
+        print(chunkstr)
+        for rule in self._rules:
+            rule.apply(chunkstr)
+            if verbose:
+                print("#", rule.descr() + " (" + repr(rule) + "):")
+            else:
+                print("#", rule.descr() + ":")
+            print(chunkstr)
+
+    def _notrace_apply(self, chunkstr):
+        """
+        Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+        turn.
+
+        :param chunkstr: The chunk string to which each rule should be
+            applied.
+        :type chunkstr: ChunkString
+        :rtype: None
+        """
+
+        for rule in self._rules:
+            rule.apply(chunkstr)
+
+    def parse(self, chunk_struct, trace=None):
+        """
+        :type chunk_struct: Tree
+        :param chunk_struct: the chunk structure to be (further) chunked
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.  This value
+            overrides the trace level value that was given to the
+            constructor.
+        :rtype: Tree
+        :return: a chunk structure that encodes the chunks in a given
+            tagged sentence.  A chunk is a non-overlapping linguistic
+            group, such as a noun phrase.  The set of chunks
+            identified in the chunk structure depends on the rules
+            used to define this ``RegexpChunkParser``.
+        """
+        if len(chunk_struct) == 0:
+            print("Warning: parsing empty text")
+            return Tree(self._root_label, [])
+
+        try:
+            chunk_struct.label()
+        except AttributeError:
+            chunk_struct = Tree(self._root_label, chunk_struct)
+
+        # Use the default trace value?
+        if trace is None:
+            trace = self._trace
+
+        chunkstr = ChunkString(chunk_struct)
+
+        # Apply the sequence of rules to the chunkstring.
+        if trace:
+            verbose = trace > 1
+            self._trace_apply(chunkstr, verbose)
+        else:
+            self._notrace_apply(chunkstr)
+
+        # Use the chunkstring to create a chunk structure.
+        return chunkstr.to_chunkstruct(self._chunk_label)
+
+    def rules(self):
+        """
+        :return: the sequence of rules used by ``RegexpChunkParser``.
+        :rtype: list(RegexpChunkRule)
+        """
+        return self._rules
+
+    def __repr__(self):
+        """
+        :return: a concise string representation of this
+            ``RegexpChunkParser``.
+        :rtype: str
+        """
+        return "<RegexpChunkParser with %d rules>" % len(self._rules)
+
+    def __str__(self):
+        """
+        :return: a verbose string representation of this ``RegexpChunkParser``.
+        :rtype: str
+        """
+        s = "RegexpChunkParser with %d rules:\n" % len(self._rules)
+        margin = 0
+        for rule in self._rules:
+            margin = max(margin, len(rule.descr()))
+        if margin < 35:
+            format = "    %" + repr(-(margin + 3)) + "s%s\n"
+        else:
+            format = "    %s\n      %s\n"
+        for rule in self._rules:
+            s += format % (rule.descr(), repr(rule))
+        return s[:-1]
+
+
+# //////////////////////////////////////////////////////
+# Chunk Grammar
+# //////////////////////////////////////////////////////
+
+
+class RegexpParser(ChunkParserI):
+    r"""
+    A grammar based chunk parser.  ``chunk.RegexpParser`` uses a set of
+    regular expression patterns to specify the behavior of the parser.
+    The chunking of the text is encoded using a ``ChunkString``, and
+    each rule acts by modifying the chunking in the ``ChunkString``.
+    The rules are all implemented using regular expression matching
+    and substitution.
+
+    A grammar contains one or more clauses in the following form::
+
+     NP:
+       {<DT|JJ>}          # chunk determiners and adjectives
+       }<[\.VI].*>+{      # strip any tag beginning with V, I, or .
+       <.*>}{<DT>         # split a chunk at a determiner
+       <DT|JJ>{}<NN.*>    # merge chunk ending with det/adj
+                          # with one starting with a noun
+
+    The patterns of a clause are executed in order.  An earlier
+    pattern may introduce a chunk boundary that prevents a later
+    pattern from executing.  Sometimes an individual pattern will
+    match on multiple, overlapping extents of the input.  As with
+    regular expression substitution more generally, the chunker will
+    identify the first match possible, then continue looking for matches
+    after this one has ended.
+
+    The clauses of a grammar are also executed in order.  A cascaded
+    chunk parser is one having more than one clause.  The maximum depth
+    of a parse tree created by this chunk parser is the same as the
+    number of clauses in the grammar.
+
+    When tracing is turned on, the comment portion of a line is displayed
+    each time the corresponding pattern is applied.
+
+    :type _start: str
+    :ivar _start: The start symbol of the grammar (the root node of
+        resulting trees)
+    :type _stages: int
+    :ivar _stages: The list of parsing stages corresponding to the grammar
+
+    """
+
+    def __init__(self, grammar, root_label="S", loop=1, trace=0):
+        """
+        Create a new chunk parser, from the given start state
+        and set of chunk patterns.
+
+        :param grammar: The grammar, or a list of RegexpChunkParser objects
+        :type grammar: str or list(RegexpChunkParser)
+        :param root_label: The top node of the tree being created
+        :type root_label: str or Nonterminal
+        :param loop: The number of times to run through the patterns
+        :type loop: int
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.
+        """
+        self._trace = trace
+        self._stages = []
+        self._grammar = grammar
+        self._loop = loop
+
+        if isinstance(grammar, str):
+            self._read_grammar(grammar, root_label, trace)
+        else:
+            # Make sur the grammar looks like it has the right type:
+            type_err = (
+                "Expected string or list of RegexpChunkParsers " "for the grammar."
+            )
+            try:
+                grammar = list(grammar)
+            except BaseException as e:
+                raise TypeError(type_err) from e
+            for elt in grammar:
+                if not isinstance(elt, RegexpChunkParser):
+                    raise TypeError(type_err)
+            self._stages = grammar
+
+    def _read_grammar(self, grammar, root_label, trace):
+        """
+        Helper function for __init__: read the grammar if it is a
+        string.
+        """
+        rules = []
+        lhs = None
+        pattern = regex.compile("(?P<nonterminal>(\\.|[^:])*)(:(?P<rule>.*))")
+        for line in grammar.split("\n"):
+            line = line.strip()
+
+            # New stage begins if there's an unescaped ':'
+            m = pattern.match(line)
+            if m:
+                # Record the stage that we just completed.
+                self._add_stage(rules, lhs, root_label, trace)
+                # Start a new stage.
+                lhs = m.group("nonterminal").strip()
+                rules = []
+                line = m.group("rule").strip()
+
+            # Skip blank & comment-only lines
+            if line == "" or line.startswith("#"):
+                continue
+
+            # Add the rule
+            rules.append(RegexpChunkRule.fromstring(line))
+
+        # Record the final stage
+        self._add_stage(rules, lhs, root_label, trace)
+
+    def _add_stage(self, rules, lhs, root_label, trace):
+        """
+        Helper function for __init__: add a new stage to the parser.
+        """
+        if rules != []:
+            if not lhs:
+                raise ValueError("Expected stage marker (eg NP:)")
+            parser = RegexpChunkParser(
+                rules, chunk_label=lhs, root_label=root_label, trace=trace
+            )
+            self._stages.append(parser)
+
+    def parse(self, chunk_struct, trace=None):
+        """
+        Apply the chunk parser to this input.
+
+        :type chunk_struct: Tree
+        :param chunk_struct: the chunk structure to be (further) chunked
+            (this tree is modified, and is also returned)
+        :type trace: int
+        :param trace: The level of tracing that should be used when
+            parsing a text.  ``0`` will generate no tracing output;
+            ``1`` will generate normal tracing output; and ``2`` or
+            higher will generate verbose tracing output.  This value
+            overrides the trace level value that was given to the
+            constructor.
+        :return: the chunked output.
+        :rtype: Tree
+        """
+        if trace is None:
+            trace = self._trace
+        for i in range(self._loop):
+            for parser in self._stages:
+                chunk_struct = parser.parse(chunk_struct, trace=trace)
+        return chunk_struct
+
+    def __repr__(self):
+        """
+        :return: a concise string representation of this ``chunk.RegexpParser``.
+        :rtype: str
+        """
+        return "<chunk.RegexpParser with %d stages>" % len(self._stages)
+
+    def __str__(self):
+        """
+        :return: a verbose string representation of this
+            ``RegexpParser``.
+        :rtype: str
+        """
+        s = "chunk.RegexpParser with %d stages:\n" % len(self._stages)
+        margin = 0
+        for parser in self._stages:
+            s += "%s\n" % parser
+        return s[:-1]
+
+
+# //////////////////////////////////////////////////////
+# Demonstration code
+# //////////////////////////////////////////////////////
+
+
+def demo_eval(chunkparser, text):
+    """
+    Demonstration code for evaluating a chunk parser, using a
+    ``ChunkScore``.  This function assumes that ``text`` contains one
+    sentence per line, and that each sentence has the form expected by
+    ``tree.chunk``.  It runs the given chunk parser on each sentence in
+    the text, and scores the result.  It prints the final score
+    (precision, recall, and f-measure); and reports the set of chunks
+    that were missed and the set of chunks that were incorrect.  (At
+    most 10 missing chunks and 10 incorrect chunks are reported).
+
+    :param chunkparser: The chunkparser to be tested
+    :type chunkparser: ChunkParserI
+    :param text: The chunked tagged text that should be used for
+        evaluation.
+    :type text: str
+    """
+    from nltk import chunk
+    from nltk.tree import Tree
+
+    # Evaluate our chunk parser.
+    chunkscore = chunk.ChunkScore()
+
+    for sentence in text.split("\n"):
+        print(sentence)
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+        gold = chunk.tagstr2tree(sentence)
+        tokens = gold.leaves()
+        test = chunkparser.parse(Tree("S", tokens), trace=1)
+        chunkscore.score(gold, test)
+        print()
+
+    print("/" + ("=" * 75) + "\\")
+    print("Scoring", chunkparser)
+    print("-" * 77)
+    print("Precision: %5.1f%%" % (chunkscore.precision() * 100), " " * 4, end=" ")
+    print("Recall: %5.1f%%" % (chunkscore.recall() * 100), " " * 6, end=" ")
+    print("F-Measure: %5.1f%%" % (chunkscore.f_measure() * 100))
+
+    # Missed chunks.
+    if chunkscore.missed():
+        print("Missed:")
+        missed = chunkscore.missed()
+        for chunk in missed[:10]:
+            print("  ", " ".join(map(str, chunk)))
+        if len(chunkscore.missed()) > 10:
+            print("  ...")
+
+    # Incorrect chunks.
+    if chunkscore.incorrect():
+        print("Incorrect:")
+        incorrect = chunkscore.incorrect()
+        for chunk in incorrect[:10]:
+            print("  ", " ".join(map(str, chunk)))
+        if len(chunkscore.incorrect()) > 10:
+            print("  ...")
+
+    print("\\" + ("=" * 75) + "/")
+    print()
+
+
+def demo():
+    """
+    A demonstration for the ``RegexpChunkParser`` class.  A single text is
+    parsed with four different chunk parsers, using a variety of rules
+    and strategies.
+    """
+
+    from nltk import Tree, chunk
+
+    text = """\
+    [ the/DT little/JJ cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] ./.
+    [ John/NNP ] saw/VBD [the/DT cats/NNS] [the/DT dog/NN] chased/VBD ./.
+    [ John/NNP ] thinks/VBZ [ Mary/NN ] saw/VBD [ the/DT cat/NN ] sit/VB on/IN [ the/DT mat/NN ]./.
+    """
+
+    print("*" * 75)
+    print("Evaluation text:")
+    print(text)
+    print("*" * 75)
+    print()
+
+    grammar = r"""
+    NP:                   # NP stage
+      {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
+      {<NNP>+}            # chunk proper nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP:
+      {<.*>}              # start by chunking each tag
+      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
+      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP: {<DT>?<JJ>*<NN>}    # chunk determiners, adjectives and nouns
+    VP: {<TO>?<VB.*>}       # VP = verb words
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    grammar = r"""
+    NP: {<.*>*}             # start by chunking everything
+        }<[\.VI].*>+{       # strip any verbs, prepositions or periods
+        <.*>}{<DT>          # separate on determiners
+    PP: {<IN><NP>}          # PP = preposition + noun phrase
+    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
+    """
+    cp = chunk.RegexpParser(grammar)
+    demo_eval(cp, text)
+
+    # Evaluation
+
+    from nltk.corpus import conll2000
+
+    print()
+    print("Demonstration of empty grammar:")
+
+    cp = chunk.RegexpParser("")
+    print(chunk.accuracy(cp, conll2000.chunked_sents("test.txt", chunk_types=("NP",))))
+
+    print()
+    print("Demonstration of accuracy evaluation using CoNLL tags:")
+
+    grammar = r"""
+    NP:
+      {<.*>}              # start by chunking each tag
+      }<[\.VI].*>+{       # unchunk any verbs, prepositions or periods
+      <DT|JJ>{}<NN.*>     # merge det/adj with nouns
+    """
+    cp = chunk.RegexpParser(grammar)
+    print(chunk.accuracy(cp, conll2000.chunked_sents("test.txt")[:5]))
+
+    print()
+    print("Demonstration of tagged token input")
+
+    grammar = r"""
+    NP: {<.*>*}             # start by chunking everything
+        }<[\.VI].*>+{       # strip any verbs, prepositions or periods
+        <.*>}{<DT>          # separate on determiners
+    PP: {<IN><NP>}          # PP = preposition + noun phrase
+    VP: {<VB.*><NP|PP>*}    # VP = verb words + NPs and PPs
+    """
+    cp = chunk.RegexpParser(grammar)
+    print(
+        cp.parse(
+            [
+                ("the", "DT"),
+                ("little", "JJ"),
+                ("cat", "NN"),
+                ("sat", "VBD"),
+                ("on", "IN"),
+                ("the", "DT"),
+                ("mat", "NN"),
+                (".", "."),
+            ]
+        )
+    )
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/chunk/util.py

@@ -0,0 +1,643 @@
+# Natural Language Toolkit: Chunk format conversions
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+
+from nltk.metrics import accuracy as _accuracy
+from nltk.tag.mapping import map_tag
+from nltk.tag.util import str2tuple
+from nltk.tree import Tree
+
+##//////////////////////////////////////////////////////
+## EVALUATION
+##//////////////////////////////////////////////////////
+
+
+def accuracy(chunker, gold):
+    """
+    Score the accuracy of the chunker against the gold standard.
+    Strip the chunk information from the gold standard and rechunk it using
+    the chunker, then compute the accuracy score.
+
+    :type chunker: ChunkParserI
+    :param chunker: The chunker being evaluated.
+    :type gold: tree
+    :param gold: The chunk structures to score the chunker on.
+    :rtype: float
+    """
+
+    gold_tags = []
+    test_tags = []
+    for gold_tree in gold:
+        test_tree = chunker.parse(gold_tree.flatten())
+        gold_tags += tree2conlltags(gold_tree)
+        test_tags += tree2conlltags(test_tree)
+
+    #    print 'GOLD:', gold_tags[:50]
+    #    print 'TEST:', test_tags[:50]
+    return _accuracy(gold_tags, test_tags)
+
+
+# Patched for increased performance by Yoav Goldberg <yoavg@cs.bgu.ac.il>, 2006-01-13
+#  -- statistics are evaluated only on demand, instead of at every sentence evaluation
+#
+# SB: use nltk.metrics for precision/recall scoring?
+#
+class ChunkScore:
+    """
+    A utility class for scoring chunk parsers.  ``ChunkScore`` can
+    evaluate a chunk parser's output, based on a number of statistics
+    (precision, recall, f-measure, misssed chunks, incorrect chunks).
+    It can also combine the scores from the parsing of multiple texts;
+    this makes it significantly easier to evaluate a chunk parser that
+    operates one sentence at a time.
+
+    Texts are evaluated with the ``score`` method.  The results of
+    evaluation can be accessed via a number of accessor methods, such
+    as ``precision`` and ``f_measure``.  A typical use of the
+    ``ChunkScore`` class is::
+
+        >>> chunkscore = ChunkScore()           # doctest: +SKIP
+        >>> for correct in correct_sentences:   # doctest: +SKIP
+        ...     guess = chunkparser.parse(correct.leaves())   # doctest: +SKIP
+        ...     chunkscore.score(correct, guess)              # doctest: +SKIP
+        >>> print('F Measure:', chunkscore.f_measure())       # doctest: +SKIP
+        F Measure: 0.823
+
+    :ivar kwargs: Keyword arguments:
+
+        - max_tp_examples: The maximum number actual examples of true
+          positives to record.  This affects the ``correct`` member
+          function: ``correct`` will not return more than this number
+          of true positive examples.  This does *not* affect any of
+          the numerical metrics (precision, recall, or f-measure)
+
+        - max_fp_examples: The maximum number actual examples of false
+          positives to record.  This affects the ``incorrect`` member
+          function and the ``guessed`` member function: ``incorrect``
+          will not return more than this number of examples, and
+          ``guessed`` will not return more than this number of true
+          positive examples.  This does *not* affect any of the
+          numerical metrics (precision, recall, or f-measure)
+
+        - max_fn_examples: The maximum number actual examples of false
+          negatives to record.  This affects the ``missed`` member
+          function and the ``correct`` member function: ``missed``
+          will not return more than this number of examples, and
+          ``correct`` will not return more than this number of true
+          negative examples.  This does *not* affect any of the
+          numerical metrics (precision, recall, or f-measure)
+
+        - chunk_label: A regular expression indicating which chunks
+          should be compared.  Defaults to ``'.*'`` (i.e., all chunks).
+
+    :type _tp: list(Token)
+    :ivar _tp: List of true positives
+    :type _fp: list(Token)
+    :ivar _fp: List of false positives
+    :type _fn: list(Token)
+    :ivar _fn: List of false negatives
+
+    :type _tp_num: int
+    :ivar _tp_num: Number of true positives
+    :type _fp_num: int
+    :ivar _fp_num: Number of false positives
+    :type _fn_num: int
+    :ivar _fn_num: Number of false negatives.
+    """
+
+    def __init__(self, **kwargs):
+        self._correct = set()
+        self._guessed = set()
+        self._tp = set()
+        self._fp = set()
+        self._fn = set()
+        self._max_tp = kwargs.get("max_tp_examples", 100)
+        self._max_fp = kwargs.get("max_fp_examples", 100)
+        self._max_fn = kwargs.get("max_fn_examples", 100)
+        self._chunk_label = kwargs.get("chunk_label", ".*")
+        self._tp_num = 0
+        self._fp_num = 0
+        self._fn_num = 0
+        self._count = 0
+        self._tags_correct = 0.0
+        self._tags_total = 0.0
+
+        self._measuresNeedUpdate = False
+
+    def _updateMeasures(self):
+        if self._measuresNeedUpdate:
+            self._tp = self._guessed & self._correct
+            self._fn = self._correct - self._guessed
+            self._fp = self._guessed - self._correct
+            self._tp_num = len(self._tp)
+            self._fp_num = len(self._fp)
+            self._fn_num = len(self._fn)
+            self._measuresNeedUpdate = False
+
+    def score(self, correct, guessed):
+        """
+        Given a correctly chunked sentence, score another chunked
+        version of the same sentence.
+
+        :type correct: chunk structure
+        :param correct: The known-correct ("gold standard") chunked
+            sentence.
+        :type guessed: chunk structure
+        :param guessed: The chunked sentence to be scored.
+        """
+        self._correct |= _chunksets(correct, self._count, self._chunk_label)
+        self._guessed |= _chunksets(guessed, self._count, self._chunk_label)
+        self._count += 1
+        self._measuresNeedUpdate = True
+        # Keep track of per-tag accuracy (if possible)
+        try:
+            correct_tags = tree2conlltags(correct)
+            guessed_tags = tree2conlltags(guessed)
+        except ValueError:
+            # This exception case is for nested chunk structures,
+            # where tree2conlltags will fail with a ValueError: "Tree
+            # is too deeply nested to be printed in CoNLL format."
+            correct_tags = guessed_tags = ()
+        self._tags_total += len(correct_tags)
+        self._tags_correct += sum(
+            1 for (t, g) in zip(guessed_tags, correct_tags) if t == g
+        )
+
+    def accuracy(self):
+        """
+        Return the overall tag-based accuracy for all text that have
+        been scored by this ``ChunkScore``, using the IOB (conll2000)
+        tag encoding.
+
+        :rtype: float
+        """
+        if self._tags_total == 0:
+            return 1
+        return self._tags_correct / self._tags_total
+
+    def precision(self):
+        """
+        Return the overall precision for all texts that have been
+        scored by this ``ChunkScore``.
+
+        :rtype: float
+        """
+        self._updateMeasures()
+        div = self._tp_num + self._fp_num
+        if div == 0:
+            return 0
+        else:
+            return self._tp_num / div
+
+    def recall(self):
+        """
+        Return the overall recall for all texts that have been
+        scored by this ``ChunkScore``.
+
+        :rtype: float
+        """
+        self._updateMeasures()
+        div = self._tp_num + self._fn_num
+        if div == 0:
+            return 0
+        else:
+            return self._tp_num / div
+
+    def f_measure(self, alpha=0.5):
+        """
+        Return the overall F measure for all texts that have been
+        scored by this ``ChunkScore``.
+
+        :param alpha: the relative weighting of precision and recall.
+            Larger alpha biases the score towards the precision value,
+            while smaller alpha biases the score towards the recall
+            value.  ``alpha`` should have a value in the range [0,1].
+        :type alpha: float
+        :rtype: float
+        """
+        self._updateMeasures()
+        p = self.precision()
+        r = self.recall()
+        if p == 0 or r == 0:  # what if alpha is 0 or 1?
+            return 0
+        return 1 / (alpha / p + (1 - alpha) / r)
+
+    def missed(self):
+        """
+        Return the chunks which were included in the
+        correct chunk structures, but not in the guessed chunk
+        structures, listed in input order.
+
+        :rtype: list of chunks
+        """
+        self._updateMeasures()
+        chunks = list(self._fn)
+        return [c[1] for c in chunks]  # discard position information
+
+    def incorrect(self):
+        """
+        Return the chunks which were included in the guessed chunk structures,
+        but not in the correct chunk structures, listed in input order.
+
+        :rtype: list of chunks
+        """
+        self._updateMeasures()
+        chunks = list(self._fp)
+        return [c[1] for c in chunks]  # discard position information
+
+    def correct(self):
+        """
+        Return the chunks which were included in the correct
+        chunk structures, listed in input order.
+
+        :rtype: list of chunks
+        """
+        chunks = list(self._correct)
+        return [c[1] for c in chunks]  # discard position information
+
+    def guessed(self):
+        """
+        Return the chunks which were included in the guessed
+        chunk structures, listed in input order.
+
+        :rtype: list of chunks
+        """
+        chunks = list(self._guessed)
+        return [c[1] for c in chunks]  # discard position information
+
+    def __len__(self):
+        self._updateMeasures()
+        return self._tp_num + self._fn_num
+
+    def __repr__(self):
+        """
+        Return a concise representation of this ``ChunkScoring``.
+
+        :rtype: str
+        """
+        return "<ChunkScoring of " + repr(len(self)) + " chunks>"
+
+    def __str__(self):
+        """
+        Return a verbose representation of this ``ChunkScoring``.
+        This representation includes the precision, recall, and
+        f-measure scores.  For other information about the score,
+        use the accessor methods (e.g., ``missed()`` and ``incorrect()``).
+
+        :rtype: str
+        """
+        return (
+            "ChunkParse score:\n"
+            + (f"    IOB Accuracy: {self.accuracy() * 100:5.1f}%%\n")
+            + (f"    Precision:    {self.precision() * 100:5.1f}%%\n")
+            + (f"    Recall:       {self.recall() * 100:5.1f}%%\n")
+            + (f"    F-Measure:    {self.f_measure() * 100:5.1f}%%")
+        )
+
+
+# extract chunks, and assign unique id, the absolute position of
+# the first word of the chunk
+def _chunksets(t, count, chunk_label):
+    pos = 0
+    chunks = []
+    for child in t:
+        if isinstance(child, Tree):
+            if re.match(chunk_label, child.label()):
+                chunks.append(((count, pos), child.freeze()))
+            pos += len(child.leaves())
+        else:
+            pos += 1
+    return set(chunks)
+
+
+def tagstr2tree(
+    s, chunk_label="NP", root_label="S", sep="/", source_tagset=None, target_tagset=None
+):
+    """
+    Divide a string of bracketted tagged text into
+    chunks and unchunked tokens, and produce a Tree.
+    Chunks are marked by square brackets (``[...]``).  Words are
+    delimited by whitespace, and each word should have the form
+    ``text/tag``.  Words that do not contain a slash are
+    assigned a ``tag`` of None.
+
+    :param s: The string to be converted
+    :type s: str
+    :param chunk_label: The label to use for chunk nodes
+    :type chunk_label: str
+    :param root_label: The label to use for the root of the tree
+    :type root_label: str
+    :rtype: Tree
+    """
+
+    WORD_OR_BRACKET = re.compile(r"\[|\]|[^\[\]\s]+")
+
+    stack = [Tree(root_label, [])]
+    for match in WORD_OR_BRACKET.finditer(s):
+        text = match.group()
+        if text[0] == "[":
+            if len(stack) != 1:
+                raise ValueError(f"Unexpected [ at char {match.start():d}")
+            chunk = Tree(chunk_label, [])
+            stack[-1].append(chunk)
+            stack.append(chunk)
+        elif text[0] == "]":
+            if len(stack) != 2:
+                raise ValueError(f"Unexpected ] at char {match.start():d}")
+            stack.pop()
+        else:
+            if sep is None:
+                stack[-1].append(text)
+            else:
+                word, tag = str2tuple(text, sep)
+                if source_tagset and target_tagset:
+                    tag = map_tag(source_tagset, target_tagset, tag)
+                stack[-1].append((word, tag))
+
+    if len(stack) != 1:
+        raise ValueError(f"Expected ] at char {len(s):d}")
+    return stack[0]
+
+
+### CONLL
+
+_LINE_RE = re.compile(r"(\S+)\s+(\S+)\s+([IOB])-?(\S+)?")
+
+
+def conllstr2tree(s, chunk_types=("NP", "PP", "VP"), root_label="S"):
+    """
+    Return a chunk structure for a single sentence
+    encoded in the given CONLL 2000 style string.
+    This function converts a CoNLL IOB string into a tree.
+    It uses the specified chunk types
+    (defaults to NP, PP and VP), and creates a tree rooted at a node
+    labeled S (by default).
+
+    :param s: The CoNLL string to be converted.
+    :type s: str
+    :param chunk_types: The chunk types to be converted.
+    :type chunk_types: tuple
+    :param root_label: The node label to use for the root.
+    :type root_label: str
+    :rtype: Tree
+    """
+
+    stack = [Tree(root_label, [])]
+
+    for lineno, line in enumerate(s.split("\n")):
+        if not line.strip():
+            continue
+
+        # Decode the line.
+        match = _LINE_RE.match(line)
+        if match is None:
+            raise ValueError(f"Error on line {lineno:d}")
+        (word, tag, state, chunk_type) = match.groups()
+
+        # If it's a chunk type we don't care about, treat it as O.
+        if chunk_types is not None and chunk_type not in chunk_types:
+            state = "O"
+
+        # For "Begin"/"Outside", finish any completed chunks -
+        # also do so for "Inside" which don't match the previous token.
+        mismatch_I = state == "I" and chunk_type != stack[-1].label()
+        if state in "BO" or mismatch_I:
+            if len(stack) == 2:
+                stack.pop()
+
+        # For "Begin", start a new chunk.
+        if state == "B" or mismatch_I:
+            chunk = Tree(chunk_type, [])
+            stack[-1].append(chunk)
+            stack.append(chunk)
+
+        # Add the new word token.
+        stack[-1].append((word, tag))
+
+    return stack[0]
+
+
+def tree2conlltags(t):
+    """
+    Return a list of 3-tuples containing ``(word, tag, IOB-tag)``.
+    Convert a tree to the CoNLL IOB tag format.
+
+    :param t: The tree to be converted.
+    :type t: Tree
+    :rtype: list(tuple)
+    """
+
+    tags = []
+    for child in t:
+        try:
+            category = child.label()
+            prefix = "B-"
+            for contents in child:
+                if isinstance(contents, Tree):
+                    raise ValueError(
+                        "Tree is too deeply nested to be printed in CoNLL format"
+                    )
+                tags.append((contents[0], contents[1], prefix + category))
+                prefix = "I-"
+        except AttributeError:
+            tags.append((child[0], child[1], "O"))
+    return tags
+
+
+def conlltags2tree(
+    sentence, chunk_types=("NP", "PP", "VP"), root_label="S", strict=False
+):
+    """
+    Convert the CoNLL IOB format to a tree.
+    """
+    tree = Tree(root_label, [])
+    for (word, postag, chunktag) in sentence:
+        if chunktag is None:
+            if strict:
+                raise ValueError("Bad conll tag sequence")
+            else:
+                # Treat as O
+                tree.append((word, postag))
+        elif chunktag.startswith("B-"):
+            tree.append(Tree(chunktag[2:], [(word, postag)]))
+        elif chunktag.startswith("I-"):
+            if (
+                len(tree) == 0
+                or not isinstance(tree[-1], Tree)
+                or tree[-1].label() != chunktag[2:]
+            ):
+                if strict:
+                    raise ValueError("Bad conll tag sequence")
+                else:
+                    # Treat as B-*
+                    tree.append(Tree(chunktag[2:], [(word, postag)]))
+            else:
+                tree[-1].append((word, postag))
+        elif chunktag == "O":
+            tree.append((word, postag))
+        else:
+            raise ValueError(f"Bad conll tag {chunktag!r}")
+    return tree
+
+
+def tree2conllstr(t):
+    """
+    Return a multiline string where each line contains a word, tag and IOB tag.
+    Convert a tree to the CoNLL IOB string format
+
+    :param t: The tree to be converted.
+    :type t: Tree
+    :rtype: str
+    """
+    lines = [" ".join(token) for token in tree2conlltags(t)]
+    return "\n".join(lines)
+
+
+### IEER
+
+_IEER_DOC_RE = re.compile(
+    r"<DOC>\s*"
+    r"(<DOCNO>\s*(?P<docno>.+?)\s*</DOCNO>\s*)?"
+    r"(<DOCTYPE>\s*(?P<doctype>.+?)\s*</DOCTYPE>\s*)?"
+    r"(<DATE_TIME>\s*(?P<date_time>.+?)\s*</DATE_TIME>\s*)?"
+    r"<BODY>\s*"
+    r"(<HEADLINE>\s*(?P<headline>.+?)\s*</HEADLINE>\s*)?"
+    r"<TEXT>(?P<text>.*?)</TEXT>\s*"
+    r"</BODY>\s*</DOC>\s*",
+    re.DOTALL,
+)
+
+_IEER_TYPE_RE = re.compile(r'<b_\w+\s+[^>]*?type="(?P<type>\w+)"')
+
+
+def _ieer_read_text(s, root_label):
+    stack = [Tree(root_label, [])]
+    # s will be None if there is no headline in the text
+    # return the empty list in place of a Tree
+    if s is None:
+        return []
+    for piece_m in re.finditer(r"<[^>]+>|[^\s<]+", s):
+        piece = piece_m.group()
+        try:
+            if piece.startswith("<b_"):
+                m = _IEER_TYPE_RE.match(piece)
+                if m is None:
+                    print("XXXX", piece)
+                chunk = Tree(m.group("type"), [])
+                stack[-1].append(chunk)
+                stack.append(chunk)
+            elif piece.startswith("<e_"):
+                stack.pop()
+            #           elif piece.startswith('<'):
+            #               print "ERROR:", piece
+            #               raise ValueError # Unexpected HTML
+            else:
+                stack[-1].append(piece)
+        except (IndexError, ValueError) as e:
+            raise ValueError(
+                f"Bad IEER string (error at character {piece_m.start():d})"
+            ) from e
+    if len(stack) != 1:
+        raise ValueError("Bad IEER string")
+    return stack[0]
+
+
+def ieerstr2tree(
+    s,
+    chunk_types=[
+        "LOCATION",
+        "ORGANIZATION",
+        "PERSON",
+        "DURATION",
+        "DATE",
+        "CARDINAL",
+        "PERCENT",
+        "MONEY",
+        "MEASURE",
+    ],
+    root_label="S",
+):
+    """
+    Return a chunk structure containing the chunked tagged text that is
+    encoded in the given IEER style string.
+    Convert a string of chunked tagged text in the IEER named
+    entity format into a chunk structure.  Chunks are of several
+    types, LOCATION, ORGANIZATION, PERSON, DURATION, DATE, CARDINAL,
+    PERCENT, MONEY, and MEASURE.
+
+    :rtype: Tree
+    """
+
+    # Try looking for a single document.  If that doesn't work, then just
+    # treat everything as if it was within the <TEXT>...</TEXT>.
+    m = _IEER_DOC_RE.match(s)
+    if m:
+        return {
+            "text": _ieer_read_text(m.group("text"), root_label),
+            "docno": m.group("docno"),
+            "doctype": m.group("doctype"),
+            "date_time": m.group("date_time"),
+            #'headline': m.group('headline')
+            # we want to capture NEs in the headline too!
+            "headline": _ieer_read_text(m.group("headline"), root_label),
+        }
+    else:
+        return _ieer_read_text(s, root_label)
+
+
+def demo():
+
+    s = "[ Pierre/NNP Vinken/NNP ] ,/, [ 61/CD years/NNS ] old/JJ ,/, will/MD join/VB [ the/DT board/NN ] ./."
+    import nltk
+
+    t = nltk.chunk.tagstr2tree(s, chunk_label="NP")
+    t.pprint()
+    print()
+
+    s = """
+These DT B-NP
+research NN I-NP
+protocols NNS I-NP
+offer VBP B-VP
+to TO B-PP
+the DT B-NP
+patient NN I-NP
+not RB O
+only RB O
+the DT B-NP
+very RB I-NP
+best JJS I-NP
+therapy NN I-NP
+which WDT B-NP
+we PRP B-NP
+have VBP B-VP
+established VBN I-VP
+today NN B-NP
+but CC B-NP
+also RB I-NP
+the DT B-NP
+hope NN I-NP
+of IN B-PP
+something NN B-NP
+still RB B-ADJP
+better JJR I-ADJP
+. . O
+"""
+
+    conll_tree = conllstr2tree(s, chunk_types=("NP", "PP"))
+    conll_tree.pprint()
+
+    # Demonstrate CoNLL output
+    print("CoNLL output:")
+    print(nltk.chunk.tree2conllstr(conll_tree))
+    print()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/__init__.py

@@ -0,0 +1,101 @@
+# Natural Language Toolkit: Classifiers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes and interfaces for labeling tokens with category labels (or
+"class labels").  Typically, labels are represented with strings
+(such as ``'health'`` or ``'sports'``).  Classifiers can be used to
+perform a wide range of classification tasks.  For example,
+classifiers can be used...
+
+- to classify documents by topic
+- to classify ambiguous words by which word sense is intended
+- to classify acoustic signals by which phoneme they represent
+- to classify sentences by their author
+
+Features
+========
+In order to decide which category label is appropriate for a given
+token, classifiers examine one or more 'features' of the token.  These
+"features" are typically chosen by hand, and indicate which aspects
+of the token are relevant to the classification decision.  For
+example, a document classifier might use a separate feature for each
+word, recording how often that word occurred in the document.
+
+Featuresets
+===========
+The features describing a token are encoded using a "featureset",
+which is a dictionary that maps from "feature names" to "feature
+values".  Feature names are unique strings that indicate what aspect
+of the token is encoded by the feature.  Examples include
+``'prevword'``, for a feature whose value is the previous word; and
+``'contains-word(library)'`` for a feature that is true when a document
+contains the word ``'library'``.  Feature values are typically
+booleans, numbers, or strings, depending on which feature they
+describe.
+
+Featuresets are typically constructed using a "feature detector"
+(also known as a "feature extractor").  A feature detector is a
+function that takes a token (and sometimes information about its
+context) as its input, and returns a featureset describing that token.
+For example, the following feature detector converts a document
+(stored as a list of words) to a featureset describing the set of
+words included in the document:
+
+    >>> # Define a feature detector function.
+    >>> def document_features(document):
+    ...     return dict([('contains-word(%s)' % w, True) for w in document])
+
+Feature detectors are typically applied to each token before it is fed
+to the classifier:
+
+    >>> # Classify each Gutenberg document.
+    >>> from nltk.corpus import gutenberg
+    >>> for fileid in gutenberg.fileids(): # doctest: +SKIP
+    ...     doc = gutenberg.words(fileid) # doctest: +SKIP
+    ...     print(fileid, classifier.classify(document_features(doc))) # doctest: +SKIP
+
+The parameters that a feature detector expects will vary, depending on
+the task and the needs of the feature detector.  For example, a
+feature detector for word sense disambiguation (WSD) might take as its
+input a sentence, and the index of a word that should be classified,
+and return a featureset for that word.  The following feature detector
+for WSD includes features describing the left and right contexts of
+the target word:
+
+    >>> def wsd_features(sentence, index):
+    ...     featureset = {}
+    ...     for i in range(max(0, index-3), index):
+    ...         featureset['left-context(%s)' % sentence[i]] = True
+    ...     for i in range(index, max(index+3, len(sentence))):
+    ...         featureset['right-context(%s)' % sentence[i]] = True
+    ...     return featureset
+
+Training Classifiers
+====================
+Most classifiers are built by training them on a list of hand-labeled
+examples, known as the "training set".  Training sets are represented
+as lists of ``(featuredict, label)`` tuples.
+"""
+
+from nltk.classify.api import ClassifierI, MultiClassifierI
+from nltk.classify.decisiontree import DecisionTreeClassifier
+from nltk.classify.maxent import (
+    BinaryMaxentFeatureEncoding,
+    ConditionalExponentialClassifier,
+    MaxentClassifier,
+    TypedMaxentFeatureEncoding,
+)
+from nltk.classify.megam import call_megam, config_megam
+from nltk.classify.naivebayes import NaiveBayesClassifier
+from nltk.classify.positivenaivebayes import PositiveNaiveBayesClassifier
+from nltk.classify.rte_classify import RTEFeatureExtractor, rte_classifier, rte_features
+from nltk.classify.scikitlearn import SklearnClassifier
+from nltk.classify.senna import Senna
+from nltk.classify.textcat import TextCat
+from nltk.classify.util import accuracy, apply_features, log_likelihood
+from nltk.classify.weka import WekaClassifier, config_weka

.eggs/nltk-3.8-py3.10.egg/nltk/classify/api.py

@@ -0,0 +1,195 @@
+# Natural Language Toolkit: Classifier Interface
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interfaces for labeling tokens with category labels (or "class labels").
+
+``ClassifierI`` is a standard interface for "single-category
+classification", in which the set of categories is known, the number
+of categories is finite, and each text belongs to exactly one
+category.
+
+``MultiClassifierI`` is a standard interface for "multi-category
+classification", which is like single-category classification except
+that each text belongs to zero or more categories.
+"""
+from nltk.internals import overridden
+
+##//////////////////////////////////////////////////////
+# { Classification Interfaces
+##//////////////////////////////////////////////////////
+
+
+class ClassifierI:
+    """
+    A processing interface for labeling tokens with a single category
+    label (or "class").  Labels are typically strs or
+    ints, but can be any immutable type.  The set of labels
+    that the classifier chooses from must be fixed and finite.
+
+    Subclasses must define:
+      - ``labels()``
+      - either ``classify()`` or ``classify_many()`` (or both)
+
+    Subclasses may define:
+      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+    """
+
+    def labels(self):
+        """
+        :return: the list of category labels used by this classifier.
+        :rtype: list of (immutable)
+        """
+        raise NotImplementedError()
+
+    def classify(self, featureset):
+        """
+        :return: the most appropriate label for the given featureset.
+        :rtype: label
+        """
+        if overridden(self.classify_many):
+            return self.classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def prob_classify(self, featureset):
+        """
+        :return: a probability distribution over labels for the given
+            featureset.
+        :rtype: ProbDistI
+        """
+        if overridden(self.prob_classify_many):
+            return self.prob_classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def classify_many(self, featuresets):
+        """
+        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.classify(fs) for fs in featuresets]
+
+        :rtype: list(label)
+        """
+        return [self.classify(fs) for fs in featuresets]
+
+    def prob_classify_many(self, featuresets):
+        """
+        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.prob_classify(fs) for fs in featuresets]
+
+        :rtype: list(ProbDistI)
+        """
+        return [self.prob_classify(fs) for fs in featuresets]
+
+
+class MultiClassifierI:
+    """
+    A processing interface for labeling tokens with zero or more
+    category labels (or "labels").  Labels are typically strs
+    or ints, but can be any immutable type.  The set of labels
+    that the multi-classifier chooses from must be fixed and finite.
+
+    Subclasses must define:
+      - ``labels()``
+      - either ``classify()`` or ``classify_many()`` (or both)
+
+    Subclasses may define:
+      - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+    """
+
+    def labels(self):
+        """
+        :return: the list of category labels used by this classifier.
+        :rtype: list of (immutable)
+        """
+        raise NotImplementedError()
+
+    def classify(self, featureset):
+        """
+        :return: the most appropriate set of labels for the given featureset.
+        :rtype: set(label)
+        """
+        if overridden(self.classify_many):
+            return self.classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def prob_classify(self, featureset):
+        """
+        :return: a probability distribution over sets of labels for the
+            given featureset.
+        :rtype: ProbDistI
+        """
+        if overridden(self.prob_classify_many):
+            return self.prob_classify_many([featureset])[0]
+        else:
+            raise NotImplementedError()
+
+    def classify_many(self, featuresets):
+        """
+        Apply ``self.classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.classify(fs) for fs in featuresets]
+
+        :rtype: list(set(label))
+        """
+        return [self.classify(fs) for fs in featuresets]
+
+    def prob_classify_many(self, featuresets):
+        """
+        Apply ``self.prob_classify()`` to each element of ``featuresets``.  I.e.:
+
+            return [self.prob_classify(fs) for fs in featuresets]
+
+        :rtype: list(ProbDistI)
+        """
+        return [self.prob_classify(fs) for fs in featuresets]
+
+
+# # [XX] IN PROGRESS:
+# class SequenceClassifierI:
+#     """
+#     A processing interface for labeling sequences of tokens with a
+#     single category label (or "class").  Labels are typically
+#     strs or ints, but can be any immutable type.  The set
+#     of labels that the classifier chooses from must be fixed and
+#     finite.
+#     """
+#     def labels(self):
+#         """
+#         :return: the list of category labels used by this classifier.
+#         :rtype: list of (immutable)
+#         """
+#         raise NotImplementedError()
+
+#     def prob_classify(self, featureset):
+#         """
+#         Return a probability distribution over labels for the given
+#         featureset.
+
+#         If ``featureset`` is a list of featuresets, then return a
+#         corresponding list containing the probability distribution
+#         over labels for each of the given featuresets, where the
+#         *i*\ th element of this list is the most appropriate label for
+#         the *i*\ th element of ``featuresets``.
+#         """
+#         raise NotImplementedError()
+
+#     def classify(self, featureset):
+#         """
+#         Return the most appropriate label for the given featureset.
+
+#         If ``featureset`` is a list of featuresets, then return a
+#         corresponding list containing the most appropriate label for
+#         each of the given featuresets, where the *i*\ th element of
+#         this list is the most appropriate label for the *i*\ th element
+#         of ``featuresets``.
+#         """
+#         raise NotImplementedError()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/decisiontree.py

@@ -0,0 +1,349 @@
+# Natural Language Toolkit: Decision Tree Classifiers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier model that decides which label to assign to a token on
+the basis of a tree structure, where branches correspond to conditions
+on feature values, and leaves correspond to label assignments.
+"""
+
+from collections import defaultdict
+
+from nltk.classify.api import ClassifierI
+from nltk.probability import FreqDist, MLEProbDist, entropy
+
+
+class DecisionTreeClassifier(ClassifierI):
+    def __init__(self, label, feature_name=None, decisions=None, default=None):
+        """
+        :param label: The most likely label for tokens that reach
+            this node in the decision tree.  If this decision tree
+            has no children, then this label will be assigned to
+            any token that reaches this decision tree.
+        :param feature_name: The name of the feature that this
+            decision tree selects for.
+        :param decisions: A dictionary mapping from feature values
+            for the feature identified by ``feature_name`` to
+            child decision trees.
+        :param default: The child that will be used if the value of
+            feature ``feature_name`` does not match any of the keys in
+            ``decisions``.  This is used when constructing binary
+            decision trees.
+        """
+        self._label = label
+        self._fname = feature_name
+        self._decisions = decisions
+        self._default = default
+
+    def labels(self):
+        labels = [self._label]
+        if self._decisions is not None:
+            for dt in self._decisions.values():
+                labels.extend(dt.labels())
+        if self._default is not None:
+            labels.extend(self._default.labels())
+        return list(set(labels))
+
+    def classify(self, featureset):
+        # Decision leaf:
+        if self._fname is None:
+            return self._label
+
+        # Decision tree:
+        fval = featureset.get(self._fname)
+        if fval in self._decisions:
+            return self._decisions[fval].classify(featureset)
+        elif self._default is not None:
+            return self._default.classify(featureset)
+        else:
+            return self._label
+
+    def error(self, labeled_featuresets):
+        errors = 0
+        for featureset, label in labeled_featuresets:
+            if self.classify(featureset) != label:
+                errors += 1
+        return errors / len(labeled_featuresets)
+
+    def pretty_format(self, width=70, prefix="", depth=4):
+        """
+        Return a string containing a pretty-printed version of this
+        decision tree.  Each line in this string corresponds to a
+        single decision tree node or leaf, and indentation is used to
+        display the structure of the decision tree.
+        """
+        # [xx] display default!!
+        if self._fname is None:
+            n = width - len(prefix) - 15
+            return "{}{} {}\n".format(prefix, "." * n, self._label)
+        s = ""
+        for i, (fval, result) in enumerate(
+            sorted(
+                self._decisions.items(),
+                key=lambda item: (item[0] in [None, False, True], str(item[0]).lower()),
+            )
+        ):
+            hdr = f"{prefix}{self._fname}={fval}? "
+            n = width - 15 - len(hdr)
+            s += "{}{} {}\n".format(hdr, "." * (n), result._label)
+            if result._fname is not None and depth > 1:
+                s += result.pretty_format(width, prefix + "  ", depth - 1)
+        if self._default is not None:
+            n = width - len(prefix) - 21
+            s += "{}else: {} {}\n".format(prefix, "." * n, self._default._label)
+            if self._default._fname is not None and depth > 1:
+                s += self._default.pretty_format(width, prefix + "  ", depth - 1)
+        return s
+
+    def pseudocode(self, prefix="", depth=4):
+        """
+        Return a string representation of this decision tree that
+        expresses the decisions it makes as a nested set of pseudocode
+        if statements.
+        """
+        if self._fname is None:
+            return f"{prefix}return {self._label!r}\n"
+        s = ""
+        for (fval, result) in sorted(
+            self._decisions.items(),
+            key=lambda item: (item[0] in [None, False, True], str(item[0]).lower()),
+        ):
+            s += f"{prefix}if {self._fname} == {fval!r}: "
+            if result._fname is not None and depth > 1:
+                s += "\n" + result.pseudocode(prefix + "  ", depth - 1)
+            else:
+                s += f"return {result._label!r}\n"
+        if self._default is not None:
+            if len(self._decisions) == 1:
+                s += "{}if {} != {!r}: ".format(
+                    prefix, self._fname, list(self._decisions.keys())[0]
+                )
+            else:
+                s += f"{prefix}else: "
+            if self._default._fname is not None and depth > 1:
+                s += "\n" + self._default.pseudocode(prefix + "  ", depth - 1)
+            else:
+                s += f"return {self._default._label!r}\n"
+        return s
+
+    def __str__(self):
+        return self.pretty_format()
+
+    @staticmethod
+    def train(
+        labeled_featuresets,
+        entropy_cutoff=0.05,
+        depth_cutoff=100,
+        support_cutoff=10,
+        binary=False,
+        feature_values=None,
+        verbose=False,
+    ):
+        """
+        :param binary: If true, then treat all feature/value pairs as
+            individual binary features, rather than using a single n-way
+            branch for each feature.
+        """
+        # Collect a list of all feature names.
+        feature_names = set()
+        for featureset, label in labeled_featuresets:
+            for fname in featureset:
+                feature_names.add(fname)
+
+        # Collect a list of the values each feature can take.
+        if feature_values is None and binary:
+            feature_values = defaultdict(set)
+            for featureset, label in labeled_featuresets:
+                for fname, fval in featureset.items():
+                    feature_values[fname].add(fval)
+
+        # Start with a stump.
+        if not binary:
+            tree = DecisionTreeClassifier.best_stump(
+                feature_names, labeled_featuresets, verbose
+            )
+        else:
+            tree = DecisionTreeClassifier.best_binary_stump(
+                feature_names, labeled_featuresets, feature_values, verbose
+            )
+
+        # Refine the stump.
+        tree.refine(
+            labeled_featuresets,
+            entropy_cutoff,
+            depth_cutoff - 1,
+            support_cutoff,
+            binary,
+            feature_values,
+            verbose,
+        )
+
+        # Return it
+        return tree
+
+    @staticmethod
+    def leaf(labeled_featuresets):
+        label = FreqDist(label for (featureset, label) in labeled_featuresets).max()
+        return DecisionTreeClassifier(label)
+
+    @staticmethod
+    def stump(feature_name, labeled_featuresets):
+        label = FreqDist(label for (featureset, label) in labeled_featuresets).max()
+
+        # Find the best label for each value.
+        freqs = defaultdict(FreqDist)  # freq(label|value)
+        for featureset, label in labeled_featuresets:
+            feature_value = featureset.get(feature_name)
+            freqs[feature_value][label] += 1
+
+        decisions = {val: DecisionTreeClassifier(freqs[val].max()) for val in freqs}
+        return DecisionTreeClassifier(label, feature_name, decisions)
+
+    def refine(
+        self,
+        labeled_featuresets,
+        entropy_cutoff,
+        depth_cutoff,
+        support_cutoff,
+        binary=False,
+        feature_values=None,
+        verbose=False,
+    ):
+        if len(labeled_featuresets) <= support_cutoff:
+            return
+        if self._fname is None:
+            return
+        if depth_cutoff <= 0:
+            return
+        for fval in self._decisions:
+            fval_featuresets = [
+                (featureset, label)
+                for (featureset, label) in labeled_featuresets
+                if featureset.get(self._fname) == fval
+            ]
+
+            label_freqs = FreqDist(label for (featureset, label) in fval_featuresets)
+            if entropy(MLEProbDist(label_freqs)) > entropy_cutoff:
+                self._decisions[fval] = DecisionTreeClassifier.train(
+                    fval_featuresets,
+                    entropy_cutoff,
+                    depth_cutoff,
+                    support_cutoff,
+                    binary,
+                    feature_values,
+                    verbose,
+                )
+        if self._default is not None:
+            default_featuresets = [
+                (featureset, label)
+                for (featureset, label) in labeled_featuresets
+                if featureset.get(self._fname) not in self._decisions
+            ]
+            label_freqs = FreqDist(label for (featureset, label) in default_featuresets)
+            if entropy(MLEProbDist(label_freqs)) > entropy_cutoff:
+                self._default = DecisionTreeClassifier.train(
+                    default_featuresets,
+                    entropy_cutoff,
+                    depth_cutoff,
+                    support_cutoff,
+                    binary,
+                    feature_values,
+                    verbose,
+                )
+
+    @staticmethod
+    def best_stump(feature_names, labeled_featuresets, verbose=False):
+        best_stump = DecisionTreeClassifier.leaf(labeled_featuresets)
+        best_error = best_stump.error(labeled_featuresets)
+        for fname in feature_names:
+            stump = DecisionTreeClassifier.stump(fname, labeled_featuresets)
+            stump_error = stump.error(labeled_featuresets)
+            if stump_error < best_error:
+                best_error = stump_error
+                best_stump = stump
+        if verbose:
+            print(
+                "best stump for {:6d} toks uses {:20} err={:6.4f}".format(
+                    len(labeled_featuresets), best_stump._fname, best_error
+                )
+            )
+        return best_stump
+
+    @staticmethod
+    def binary_stump(feature_name, feature_value, labeled_featuresets):
+        label = FreqDist(label for (featureset, label) in labeled_featuresets).max()
+
+        # Find the best label for each value.
+        pos_fdist = FreqDist()
+        neg_fdist = FreqDist()
+        for featureset, label in labeled_featuresets:
+            if featureset.get(feature_name) == feature_value:
+                pos_fdist[label] += 1
+            else:
+                neg_fdist[label] += 1
+
+        decisions = {}
+        default = label
+        # But hopefully we have observations!
+        if pos_fdist.N() > 0:
+            decisions = {feature_value: DecisionTreeClassifier(pos_fdist.max())}
+        if neg_fdist.N() > 0:
+            default = DecisionTreeClassifier(neg_fdist.max())
+
+        return DecisionTreeClassifier(label, feature_name, decisions, default)
+
+    @staticmethod
+    def best_binary_stump(
+        feature_names, labeled_featuresets, feature_values, verbose=False
+    ):
+        best_stump = DecisionTreeClassifier.leaf(labeled_featuresets)
+        best_error = best_stump.error(labeled_featuresets)
+        for fname in feature_names:
+            for fval in feature_values[fname]:
+                stump = DecisionTreeClassifier.binary_stump(
+                    fname, fval, labeled_featuresets
+                )
+                stump_error = stump.error(labeled_featuresets)
+                if stump_error < best_error:
+                    best_error = stump_error
+                    best_stump = stump
+        if verbose:
+            if best_stump._decisions:
+                descr = "{}={}".format(
+                    best_stump._fname, list(best_stump._decisions.keys())[0]
+                )
+            else:
+                descr = "(default)"
+            print(
+                "best stump for {:6d} toks uses {:20} err={:6.4f}".format(
+                    len(labeled_featuresets), descr, best_error
+                )
+            )
+        return best_stump
+
+
+##//////////////////////////////////////////////////////
+##  Demo
+##//////////////////////////////////////////////////////
+
+
+def f(x):
+    return DecisionTreeClassifier.train(x, binary=True, verbose=True)
+
+
+def demo():
+    from nltk.classify.util import binary_names_demo_features, names_demo
+
+    classifier = names_demo(
+        f, binary_names_demo_features  # DecisionTreeClassifier.train,
+    )
+    print(classifier.pretty_format(depth=7))
+    print(classifier.pseudocode(depth=7))
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/maxent.py

@@ -0,0 +1,1569 @@
+# Natural Language Toolkit: Maximum Entropy Classifiers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Dmitry Chichkov <dchichkov@gmail.com> (TypedMaxentFeatureEncoding)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier model based on maximum entropy modeling framework.  This
+framework considers all of the probability distributions that are
+empirically consistent with the training data; and chooses the
+distribution with the highest entropy.  A probability distribution is
+"empirically consistent" with a set of training data if its estimated
+frequency with which a class and a feature vector value co-occur is
+equal to the actual frequency in the data.
+
+Terminology: 'feature'
+======================
+The term *feature* is usually used to refer to some property of an
+unlabeled token.  For example, when performing word sense
+disambiguation, we might define a ``'prevword'`` feature whose value is
+the word preceding the target word.  However, in the context of
+maxent modeling, the term *feature* is typically used to refer to a
+property of a "labeled" token.  In order to prevent confusion, we
+will introduce two distinct terms to disambiguate these two different
+concepts:
+
+  - An "input-feature" is a property of an unlabeled token.
+  - A "joint-feature" is a property of a labeled token.
+
+In the rest of the ``nltk.classify`` module, the term "features" is
+used to refer to what we will call "input-features" in this module.
+
+In literature that describes and discusses maximum entropy models,
+input-features are typically called "contexts", and joint-features
+are simply referred to as "features".
+
+Converting Input-Features to Joint-Features
+-------------------------------------------
+In maximum entropy models, joint-features are required to have numeric
+values.  Typically, each input-feature ``input_feat`` is mapped to a
+set of joint-features of the form:
+
+|   joint_feat(token, label) = { 1 if input_feat(token) == feat_val
+|                              {      and label == some_label
+|                              {
+|                              { 0 otherwise
+
+For all values of ``feat_val`` and ``some_label``.  This mapping is
+performed by classes that implement the ``MaxentFeatureEncodingI``
+interface.
+"""
+try:
+    import numpy
+except ImportError:
+    pass
+
+import os
+import tempfile
+from collections import defaultdict
+
+from nltk.classify.api import ClassifierI
+from nltk.classify.megam import call_megam, parse_megam_weights, write_megam_file
+from nltk.classify.tadm import call_tadm, parse_tadm_weights, write_tadm_file
+from nltk.classify.util import CutoffChecker, accuracy, log_likelihood
+from nltk.data import gzip_open_unicode
+from nltk.probability import DictionaryProbDist
+from nltk.util import OrderedDict
+
+__docformat__ = "epytext en"
+
+######################################################################
+# { Classifier Model
+######################################################################
+
+
+class MaxentClassifier(ClassifierI):
+    """
+    A maximum entropy classifier (also known as a "conditional
+    exponential classifier").  This classifier is parameterized by a
+    set of "weights", which are used to combine the joint-features
+    that are generated from a featureset by an "encoding".  In
+    particular, the encoding maps each ``(featureset, label)`` pair to
+    a vector.  The probability of each label is then computed using
+    the following equation::
+
+                                dotprod(weights, encode(fs,label))
+      prob(fs|label) = ---------------------------------------------------
+                       sum(dotprod(weights, encode(fs,l)) for l in labels)
+
+    Where ``dotprod`` is the dot product::
+
+      dotprod(a,b) = sum(x*y for (x,y) in zip(a,b))
+    """
+
+    def __init__(self, encoding, weights, logarithmic=True):
+        """
+        Construct a new maxent classifier model.  Typically, new
+        classifier models are created using the ``train()`` method.
+
+        :type encoding: MaxentFeatureEncodingI
+        :param encoding: An encoding that is used to convert the
+            featuresets that are given to the ``classify`` method into
+            joint-feature vectors, which are used by the maxent
+            classifier model.
+
+        :type weights: list of float
+        :param weights:  The feature weight vector for this classifier.
+
+        :type logarithmic: bool
+        :param logarithmic: If false, then use non-logarithmic weights.
+        """
+        self._encoding = encoding
+        self._weights = weights
+        self._logarithmic = logarithmic
+        # self._logarithmic = False
+        assert encoding.length() == len(weights)
+
+    def labels(self):
+        return self._encoding.labels()
+
+    def set_weights(self, new_weights):
+        """
+        Set the feature weight vector for this classifier.
+        :param new_weights: The new feature weight vector.
+        :type new_weights: list of float
+        """
+        self._weights = new_weights
+        assert self._encoding.length() == len(new_weights)
+
+    def weights(self):
+        """
+        :return: The feature weight vector for this classifier.
+        :rtype: list of float
+        """
+        return self._weights
+
+    def classify(self, featureset):
+        return self.prob_classify(featureset).max()
+
+    def prob_classify(self, featureset):
+        prob_dict = {}
+        for label in self._encoding.labels():
+            feature_vector = self._encoding.encode(featureset, label)
+
+            if self._logarithmic:
+                total = 0.0
+                for (f_id, f_val) in feature_vector:
+                    total += self._weights[f_id] * f_val
+                prob_dict[label] = total
+
+            else:
+                prod = 1.0
+                for (f_id, f_val) in feature_vector:
+                    prod *= self._weights[f_id] ** f_val
+                prob_dict[label] = prod
+
+        # Normalize the dictionary to give a probability distribution
+        return DictionaryProbDist(prob_dict, log=self._logarithmic, normalize=True)
+
+    def explain(self, featureset, columns=4):
+        """
+        Print a table showing the effect of each of the features in
+        the given feature set, and how they combine to determine the
+        probabilities of each label for that featureset.
+        """
+        descr_width = 50
+        TEMPLATE = "  %-" + str(descr_width - 2) + "s%s%8.3f"
+
+        pdist = self.prob_classify(featureset)
+        labels = sorted(pdist.samples(), key=pdist.prob, reverse=True)
+        labels = labels[:columns]
+        print(
+            "  Feature".ljust(descr_width)
+            + "".join("%8s" % (("%s" % l)[:7]) for l in labels)
+        )
+        print("  " + "-" * (descr_width - 2 + 8 * len(labels)))
+        sums = defaultdict(int)
+        for i, label in enumerate(labels):
+            feature_vector = self._encoding.encode(featureset, label)
+            feature_vector.sort(
+                key=lambda fid__: abs(self._weights[fid__[0]]), reverse=True
+            )
+            for (f_id, f_val) in feature_vector:
+                if self._logarithmic:
+                    score = self._weights[f_id] * f_val
+                else:
+                    score = self._weights[f_id] ** f_val
+                descr = self._encoding.describe(f_id)
+                descr = descr.split(" and label is ")[0]  # hack
+                descr += " (%s)" % f_val  # hack
+                if len(descr) > 47:
+                    descr = descr[:44] + "..."
+                print(TEMPLATE % (descr, i * 8 * " ", score))
+                sums[label] += score
+        print("  " + "-" * (descr_width - 1 + 8 * len(labels)))
+        print(
+            "  TOTAL:".ljust(descr_width) + "".join("%8.3f" % sums[l] for l in labels)
+        )
+        print(
+            "  PROBS:".ljust(descr_width)
+            + "".join("%8.3f" % pdist.prob(l) for l in labels)
+        )
+
+    def most_informative_features(self, n=10):
+        """
+        Generates the ranked list of informative features from most to least.
+        """
+        if hasattr(self, "_most_informative_features"):
+            return self._most_informative_features[:n]
+        else:
+            self._most_informative_features = sorted(
+                list(range(len(self._weights))),
+                key=lambda fid: abs(self._weights[fid]),
+                reverse=True,
+            )
+            return self._most_informative_features[:n]
+
+    def show_most_informative_features(self, n=10, show="all"):
+        """
+        :param show: all, neg, or pos (for negative-only or positive-only)
+        :type show: str
+        :param n: The no. of top features
+        :type n: int
+        """
+        # Use None the full list of ranked features.
+        fids = self.most_informative_features(None)
+        if show == "pos":
+            fids = [fid for fid in fids if self._weights[fid] > 0]
+        elif show == "neg":
+            fids = [fid for fid in fids if self._weights[fid] < 0]
+        for fid in fids[:n]:
+            print(f"{self._weights[fid]:8.3f} {self._encoding.describe(fid)}")
+
+    def __repr__(self):
+        return "<ConditionalExponentialClassifier: %d labels, %d features>" % (
+            len(self._encoding.labels()),
+            self._encoding.length(),
+        )
+
+    #: A list of the algorithm names that are accepted for the
+    #: ``train()`` method's ``algorithm`` parameter.
+    ALGORITHMS = ["GIS", "IIS", "MEGAM", "TADM"]
+
+    @classmethod
+    def train(
+        cls,
+        train_toks,
+        algorithm=None,
+        trace=3,
+        encoding=None,
+        labels=None,
+        gaussian_prior_sigma=0,
+        **cutoffs,
+    ):
+        """
+        Train a new maxent classifier based on the given corpus of
+        training samples.  This classifier will have its weights
+        chosen to maximize entropy while remaining empirically
+        consistent with the training corpus.
+
+        :rtype: MaxentClassifier
+        :return: The new maxent classifier
+
+        :type train_toks: list
+        :param train_toks: Training data, represented as a list of
+            pairs, the first member of which is a featureset,
+            and the second of which is a classification label.
+
+        :type algorithm: str
+        :param algorithm: A case-insensitive string, specifying which
+            algorithm should be used to train the classifier.  The
+            following algorithms are currently available.
+
+            - Iterative Scaling Methods: Generalized Iterative Scaling (``'GIS'``),
+              Improved Iterative Scaling (``'IIS'``)
+            - External Libraries (requiring megam):
+              LM-BFGS algorithm, with training performed by Megam (``'megam'``)
+
+            The default algorithm is ``'IIS'``.
+
+        :type trace: int
+        :param trace: The level of diagnostic tracing output to produce.
+            Higher values produce more verbose output.
+        :type encoding: MaxentFeatureEncodingI
+        :param encoding: A feature encoding, used to convert featuresets
+            into feature vectors.  If none is specified, then a
+            ``BinaryMaxentFeatureEncoding`` will be built based on the
+            features that are attested in the training corpus.
+        :type labels: list(str)
+        :param labels: The set of possible labels.  If none is given, then
+            the set of all labels attested in the training data will be
+            used instead.
+        :param gaussian_prior_sigma: The sigma value for a gaussian
+            prior on model weights.  Currently, this is supported by
+            ``megam``. For other algorithms, its value is ignored.
+        :param cutoffs: Arguments specifying various conditions under
+            which the training should be halted.  (Some of the cutoff
+            conditions are not supported by some algorithms.)
+
+            - ``max_iter=v``: Terminate after ``v`` iterations.
+            - ``min_ll=v``: Terminate after the negative average
+              log-likelihood drops under ``v``.
+            - ``min_lldelta=v``: Terminate if a single iteration improves
+              log likelihood by less than ``v``.
+        """
+        if algorithm is None:
+            algorithm = "iis"
+        for key in cutoffs:
+            if key not in (
+                "max_iter",
+                "min_ll",
+                "min_lldelta",
+                "max_acc",
+                "min_accdelta",
+                "count_cutoff",
+                "norm",
+                "explicit",
+                "bernoulli",
+            ):
+                raise TypeError("Unexpected keyword arg %r" % key)
+        algorithm = algorithm.lower()
+        if algorithm == "iis":
+            return train_maxent_classifier_with_iis(
+                train_toks, trace, encoding, labels, **cutoffs
+            )
+        elif algorithm == "gis":
+            return train_maxent_classifier_with_gis(
+                train_toks, trace, encoding, labels, **cutoffs
+            )
+        elif algorithm == "megam":
+            return train_maxent_classifier_with_megam(
+                train_toks, trace, encoding, labels, gaussian_prior_sigma, **cutoffs
+            )
+        elif algorithm == "tadm":
+            kwargs = cutoffs
+            kwargs["trace"] = trace
+            kwargs["encoding"] = encoding
+            kwargs["labels"] = labels
+            kwargs["gaussian_prior_sigma"] = gaussian_prior_sigma
+            return TadmMaxentClassifier.train(train_toks, **kwargs)
+        else:
+            raise ValueError("Unknown algorithm %s" % algorithm)
+
+
+#: Alias for MaxentClassifier.
+ConditionalExponentialClassifier = MaxentClassifier
+
+
+######################################################################
+# { Feature Encodings
+######################################################################
+
+
+class MaxentFeatureEncodingI:
+    """
+    A mapping that converts a set of input-feature values to a vector
+    of joint-feature values, given a label.  This conversion is
+    necessary to translate featuresets into a format that can be used
+    by maximum entropy models.
+
+    The set of joint-features used by a given encoding is fixed, and
+    each index in the generated joint-feature vectors corresponds to a
+    single joint-feature.  The length of the generated joint-feature
+    vectors is therefore constant (for a given encoding).
+
+    Because the joint-feature vectors generated by
+    ``MaxentFeatureEncodingI`` are typically very sparse, they are
+    represented as a list of ``(index, value)`` tuples, specifying the
+    value of each non-zero joint-feature.
+
+    Feature encodings are generally created using the ``train()``
+    method, which generates an appropriate encoding based on the
+    input-feature values and labels that are present in a given
+    corpus.
+    """
+
+    def encode(self, featureset, label):
+        """
+        Given a (featureset, label) pair, return the corresponding
+        vector of joint-feature values.  This vector is represented as
+        a list of ``(index, value)`` tuples, specifying the value of
+        each non-zero joint-feature.
+
+        :type featureset: dict
+        :rtype: list(tuple(int, int))
+        """
+        raise NotImplementedError()
+
+    def length(self):
+        """
+        :return: The size of the fixed-length joint-feature vectors
+            that are generated by this encoding.
+        :rtype: int
+        """
+        raise NotImplementedError()
+
+    def labels(self):
+        """
+        :return: A list of the \"known labels\" -- i.e., all labels
+            ``l`` such that ``self.encode(fs,l)`` can be a nonzero
+            joint-feature vector for some value of ``fs``.
+        :rtype: list
+        """
+        raise NotImplementedError()
+
+    def describe(self, fid):
+        """
+        :return: A string describing the value of the joint-feature
+            whose index in the generated feature vectors is ``fid``.
+        :rtype: str
+        """
+        raise NotImplementedError()
+
+    def train(cls, train_toks):
+        """
+        Construct and return new feature encoding, based on a given
+        training corpus ``train_toks``.
+
+        :type train_toks: list(tuple(dict, str))
+        :param train_toks: Training data, represented as a list of
+            pairs, the first member of which is a feature dictionary,
+            and the second of which is a classification label.
+        """
+        raise NotImplementedError()
+
+
+class FunctionBackedMaxentFeatureEncoding(MaxentFeatureEncodingI):
+    """
+    A feature encoding that calls a user-supplied function to map a
+    given featureset/label pair to a sparse joint-feature vector.
+    """
+
+    def __init__(self, func, length, labels):
+        """
+        Construct a new feature encoding based on the given function.
+
+        :type func: (callable)
+        :param func: A function that takes two arguments, a featureset
+             and a label, and returns the sparse joint feature vector
+             that encodes them::
+
+                 func(featureset, label) -> feature_vector
+
+             This sparse joint feature vector (``feature_vector``) is a
+             list of ``(index,value)`` tuples.
+
+        :type length: int
+        :param length: The size of the fixed-length joint-feature
+            vectors that are generated by this encoding.
+
+        :type labels: list
+        :param labels: A list of the \"known labels\" for this
+            encoding -- i.e., all labels ``l`` such that
+            ``self.encode(fs,l)`` can be a nonzero joint-feature vector
+            for some value of ``fs``.
+        """
+        self._length = length
+        self._func = func
+        self._labels = labels
+
+    def encode(self, featureset, label):
+        return self._func(featureset, label)
+
+    def length(self):
+        return self._length
+
+    def labels(self):
+        return self._labels
+
+    def describe(self, fid):
+        return "no description available"
+
+
+class BinaryMaxentFeatureEncoding(MaxentFeatureEncodingI):
+    """
+    A feature encoding that generates vectors containing a binary
+    joint-features of the form:
+
+    |  joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
+    |                      {
+    |                      { 0 otherwise
+
+    Where ``fname`` is the name of an input-feature, ``fval`` is a value
+    for that input-feature, and ``label`` is a label.
+
+    Typically, these features are constructed based on a training
+    corpus, using the ``train()`` method.  This method will create one
+    feature for each combination of ``fname``, ``fval``, and ``label``
+    that occurs at least once in the training corpus.
+
+    The ``unseen_features`` parameter can be used to add "unseen-value
+    features", which are used whenever an input feature has a value
+    that was not encountered in the training corpus.  These features
+    have the form:
+
+    |  joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
+    |                      {      and l == label
+    |                      {
+    |                      { 0 otherwise
+
+    Where ``is_unseen(fname, fval)`` is true if the encoding does not
+    contain any joint features that are true when ``fs[fname]==fval``.
+
+    The ``alwayson_features`` parameter can be used to add "always-on
+    features", which have the form::
+
+    |  joint_feat(fs, l) = { 1 if (l == label)
+    |                      {
+    |                      { 0 otherwise
+
+    These always-on features allow the maxent model to directly model
+    the prior probabilities of each label.
+    """
+
+    def __init__(self, labels, mapping, unseen_features=False, alwayson_features=False):
+        """
+        :param labels: A list of the \"known labels\" for this encoding.
+
+        :param mapping: A dictionary mapping from ``(fname,fval,label)``
+            tuples to corresponding joint-feature indexes.  These
+            indexes must be the set of integers from 0...len(mapping).
+            If ``mapping[fname,fval,label]=id``, then
+            ``self.encode(..., fname:fval, ..., label)[id]`` is 1;
+            otherwise, it is 0.
+
+        :param unseen_features: If true, then include unseen value
+           features in the generated joint-feature vectors.
+
+        :param alwayson_features: If true, then include always-on
+           features in the generated joint-feature vectors.
+        """
+        if set(mapping.values()) != set(range(len(mapping))):
+            raise ValueError(
+                "Mapping values must be exactly the "
+                "set of integers from 0...len(mapping)"
+            )
+
+        self._labels = list(labels)
+        """A list of attested labels."""
+
+        self._mapping = mapping
+        """dict mapping from (fname,fval,label) -> fid"""
+
+        self._length = len(mapping)
+        """The length of generated joint feature vectors."""
+
+        self._alwayson = None
+        """dict mapping from label -> fid"""
+
+        self._unseen = None
+        """dict mapping from fname -> fid"""
+
+        if alwayson_features:
+            self._alwayson = {
+                label: i + self._length for (i, label) in enumerate(labels)
+            }
+            self._length += len(self._alwayson)
+
+        if unseen_features:
+            fnames = {fname for (fname, fval, label) in mapping}
+            self._unseen = {fname: i + self._length for (i, fname) in enumerate(fnames)}
+            self._length += len(fnames)
+
+    def encode(self, featureset, label):
+        # Inherit docs.
+        encoding = []
+
+        # Convert input-features to joint-features:
+        for fname, fval in featureset.items():
+            # Known feature name & value:
+            if (fname, fval, label) in self._mapping:
+                encoding.append((self._mapping[fname, fval, label], 1))
+
+            # Otherwise, we might want to fire an "unseen-value feature".
+            elif self._unseen:
+                # Have we seen this fname/fval combination with any label?
+                for label2 in self._labels:
+                    if (fname, fval, label2) in self._mapping:
+                        break  # we've seen this fname/fval combo
+                # We haven't -- fire the unseen-value feature
+                else:
+                    if fname in self._unseen:
+                        encoding.append((self._unseen[fname], 1))
+
+        # Add always-on features:
+        if self._alwayson and label in self._alwayson:
+            encoding.append((self._alwayson[label], 1))
+
+        return encoding
+
+    def describe(self, f_id):
+        # Inherit docs.
+        if not isinstance(f_id, int):
+            raise TypeError("describe() expected an int")
+        try:
+            self._inv_mapping
+        except AttributeError:
+            self._inv_mapping = [-1] * len(self._mapping)
+            for (info, i) in self._mapping.items():
+                self._inv_mapping[i] = info
+
+        if f_id < len(self._mapping):
+            (fname, fval, label) = self._inv_mapping[f_id]
+            return f"{fname}=={fval!r} and label is {label!r}"
+        elif self._alwayson and f_id in self._alwayson.values():
+            for (label, f_id2) in self._alwayson.items():
+                if f_id == f_id2:
+                    return "label is %r" % label
+        elif self._unseen and f_id in self._unseen.values():
+            for (fname, f_id2) in self._unseen.items():
+                if f_id == f_id2:
+                    return "%s is unseen" % fname
+        else:
+            raise ValueError("Bad feature id")
+
+    def labels(self):
+        # Inherit docs.
+        return self._labels
+
+    def length(self):
+        # Inherit docs.
+        return self._length
+
+    @classmethod
+    def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+        """
+        Construct and return new feature encoding, based on a given
+        training corpus ``train_toks``.  See the class description
+        ``BinaryMaxentFeatureEncoding`` for a description of the
+        joint-features that will be included in this encoding.
+
+        :type train_toks: list(tuple(dict, str))
+        :param train_toks: Training data, represented as a list of
+            pairs, the first member of which is a feature dictionary,
+            and the second of which is a classification label.
+
+        :type count_cutoff: int
+        :param count_cutoff: A cutoff value that is used to discard
+            rare joint-features.  If a joint-feature's value is 1
+            fewer than ``count_cutoff`` times in the training corpus,
+            then that joint-feature is not included in the generated
+            encoding.
+
+        :type labels: list
+        :param labels: A list of labels that should be used by the
+            classifier.  If not specified, then the set of labels
+            attested in ``train_toks`` will be used.
+
+        :param options: Extra parameters for the constructor, such as
+            ``unseen_features`` and ``alwayson_features``.
+        """
+        mapping = {}  # maps (fname, fval, label) -> fid
+        seen_labels = set()  # The set of labels we've encountered
+        count = defaultdict(int)  # maps (fname, fval) -> count
+
+        for (tok, label) in train_toks:
+            if labels and label not in labels:
+                raise ValueError("Unexpected label %s" % label)
+            seen_labels.add(label)
+
+            # Record each of the features.
+            for (fname, fval) in tok.items():
+
+                # If a count cutoff is given, then only add a joint
+                # feature once the corresponding (fname, fval, label)
+                # tuple exceeds that cutoff.
+                count[fname, fval] += 1
+                if count[fname, fval] >= count_cutoff:
+                    if (fname, fval, label) not in mapping:
+                        mapping[fname, fval, label] = len(mapping)
+
+        if labels is None:
+            labels = seen_labels
+        return cls(labels, mapping, **options)
+
+
+class GISEncoding(BinaryMaxentFeatureEncoding):
+    """
+    A binary feature encoding which adds one new joint-feature to the
+    joint-features defined by ``BinaryMaxentFeatureEncoding``: a
+    correction feature, whose value is chosen to ensure that the
+    sparse vector always sums to a constant non-negative number.  This
+    new feature is used to ensure two preconditions for the GIS
+    training algorithm:
+
+      - At least one feature vector index must be nonzero for every
+        token.
+      - The feature vector must sum to a constant non-negative number
+        for every token.
+    """
+
+    def __init__(
+        self, labels, mapping, unseen_features=False, alwayson_features=False, C=None
+    ):
+        """
+        :param C: The correction constant.  The value of the correction
+            feature is based on this value.  In particular, its value is
+            ``C - sum([v for (f,v) in encoding])``.
+        :seealso: ``BinaryMaxentFeatureEncoding.__init__``
+        """
+        BinaryMaxentFeatureEncoding.__init__(
+            self, labels, mapping, unseen_features, alwayson_features
+        )
+        if C is None:
+            C = len({fname for (fname, fval, label) in mapping}) + 1
+        self._C = C
+
+    @property
+    def C(self):
+        """The non-negative constant that all encoded feature vectors
+        will sum to."""
+        return self._C
+
+    def encode(self, featureset, label):
+        # Get the basic encoding.
+        encoding = BinaryMaxentFeatureEncoding.encode(self, featureset, label)
+        base_length = BinaryMaxentFeatureEncoding.length(self)
+
+        # Add a correction feature.
+        total = sum(v for (f, v) in encoding)
+        if total >= self._C:
+            raise ValueError("Correction feature is not high enough!")
+        encoding.append((base_length, self._C - total))
+
+        # Return the result
+        return encoding
+
+    def length(self):
+        return BinaryMaxentFeatureEncoding.length(self) + 1
+
+    def describe(self, f_id):
+        if f_id == BinaryMaxentFeatureEncoding.length(self):
+            return "Correction feature (%s)" % self._C
+        else:
+            return BinaryMaxentFeatureEncoding.describe(self, f_id)
+
+
+class TadmEventMaxentFeatureEncoding(BinaryMaxentFeatureEncoding):
+    def __init__(self, labels, mapping, unseen_features=False, alwayson_features=False):
+        self._mapping = OrderedDict(mapping)
+        self._label_mapping = OrderedDict()
+        BinaryMaxentFeatureEncoding.__init__(
+            self, labels, self._mapping, unseen_features, alwayson_features
+        )
+
+    def encode(self, featureset, label):
+        encoding = []
+        for feature, value in featureset.items():
+            if (feature, label) not in self._mapping:
+                self._mapping[(feature, label)] = len(self._mapping)
+            if value not in self._label_mapping:
+                if not isinstance(value, int):
+                    self._label_mapping[value] = len(self._label_mapping)
+                else:
+                    self._label_mapping[value] = value
+            encoding.append(
+                (self._mapping[(feature, label)], self._label_mapping[value])
+            )
+        return encoding
+
+    def labels(self):
+        return self._labels
+
+    def describe(self, fid):
+        for (feature, label) in self._mapping:
+            if self._mapping[(feature, label)] == fid:
+                return (feature, label)
+
+    def length(self):
+        return len(self._mapping)
+
+    @classmethod
+    def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+        mapping = OrderedDict()
+        if not labels:
+            labels = []
+
+        # This gets read twice, so compute the values in case it's lazy.
+        train_toks = list(train_toks)
+
+        for (featureset, label) in train_toks:
+            if label not in labels:
+                labels.append(label)
+
+        for (featureset, label) in train_toks:
+            for label in labels:
+                for feature in featureset:
+                    if (feature, label) not in mapping:
+                        mapping[(feature, label)] = len(mapping)
+
+        return cls(labels, mapping, **options)
+
+
+class TypedMaxentFeatureEncoding(MaxentFeatureEncodingI):
+    """
+    A feature encoding that generates vectors containing integer,
+    float and binary joint-features of the form:
+
+    Binary (for string and boolean features):
+
+    |  joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
+    |                      {
+    |                      { 0 otherwise
+
+    Value (for integer and float features):
+
+    |  joint_feat(fs, l) = { fval if     (fs[fname] == type(fval))
+    |                      {         and (l == label)
+    |                      {
+    |                      { not encoded otherwise
+
+    Where ``fname`` is the name of an input-feature, ``fval`` is a value
+    for that input-feature, and ``label`` is a label.
+
+    Typically, these features are constructed based on a training
+    corpus, using the ``train()`` method.
+
+    For string and boolean features [type(fval) not in (int, float)]
+    this method will create one feature for each combination of
+    ``fname``, ``fval``, and ``label`` that occurs at least once in the
+    training corpus.
+
+    For integer and float features [type(fval) in (int, float)] this
+    method will create one feature for each combination of ``fname``
+    and ``label`` that occurs at least once in the training corpus.
+
+    For binary features the ``unseen_features`` parameter can be used
+    to add "unseen-value features", which are used whenever an input
+    feature has a value that was not encountered in the training
+    corpus.  These features have the form:
+
+    |  joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
+    |                      {      and l == label
+    |                      {
+    |                      { 0 otherwise
+
+    Where ``is_unseen(fname, fval)`` is true if the encoding does not
+    contain any joint features that are true when ``fs[fname]==fval``.
+
+    The ``alwayson_features`` parameter can be used to add "always-on
+    features", which have the form:
+
+    |  joint_feat(fs, l) = { 1 if (l == label)
+    |                      {
+    |                      { 0 otherwise
+
+    These always-on features allow the maxent model to directly model
+    the prior probabilities of each label.
+    """
+
+    def __init__(self, labels, mapping, unseen_features=False, alwayson_features=False):
+        """
+        :param labels: A list of the \"known labels\" for this encoding.
+
+        :param mapping: A dictionary mapping from ``(fname,fval,label)``
+            tuples to corresponding joint-feature indexes.  These
+            indexes must be the set of integers from 0...len(mapping).
+            If ``mapping[fname,fval,label]=id``, then
+            ``self.encode({..., fname:fval, ...``, label)[id]} is 1;
+            otherwise, it is 0.
+
+        :param unseen_features: If true, then include unseen value
+           features in the generated joint-feature vectors.
+
+        :param alwayson_features: If true, then include always-on
+           features in the generated joint-feature vectors.
+        """
+        if set(mapping.values()) != set(range(len(mapping))):
+            raise ValueError(
+                "Mapping values must be exactly the "
+                "set of integers from 0...len(mapping)"
+            )
+
+        self._labels = list(labels)
+        """A list of attested labels."""
+
+        self._mapping = mapping
+        """dict mapping from (fname,fval,label) -> fid"""
+
+        self._length = len(mapping)
+        """The length of generated joint feature vectors."""
+
+        self._alwayson = None
+        """dict mapping from label -> fid"""
+
+        self._unseen = None
+        """dict mapping from fname -> fid"""
+
+        if alwayson_features:
+            self._alwayson = {
+                label: i + self._length for (i, label) in enumerate(labels)
+            }
+            self._length += len(self._alwayson)
+
+        if unseen_features:
+            fnames = {fname for (fname, fval, label) in mapping}
+            self._unseen = {fname: i + self._length for (i, fname) in enumerate(fnames)}
+            self._length += len(fnames)
+
+    def encode(self, featureset, label):
+        # Inherit docs.
+        encoding = []
+
+        # Convert input-features to joint-features:
+        for fname, fval in featureset.items():
+            if isinstance(fval, (int, float)):
+                # Known feature name & value:
+                if (fname, type(fval), label) in self._mapping:
+                    encoding.append((self._mapping[fname, type(fval), label], fval))
+            else:
+                # Known feature name & value:
+                if (fname, fval, label) in self._mapping:
+                    encoding.append((self._mapping[fname, fval, label], 1))
+
+                # Otherwise, we might want to fire an "unseen-value feature".
+                elif self._unseen:
+                    # Have we seen this fname/fval combination with any label?
+                    for label2 in self._labels:
+                        if (fname, fval, label2) in self._mapping:
+                            break  # we've seen this fname/fval combo
+                    # We haven't -- fire the unseen-value feature
+                    else:
+                        if fname in self._unseen:
+                            encoding.append((self._unseen[fname], 1))
+
+        # Add always-on features:
+        if self._alwayson and label in self._alwayson:
+            encoding.append((self._alwayson[label], 1))
+
+        return encoding
+
+    def describe(self, f_id):
+        # Inherit docs.
+        if not isinstance(f_id, int):
+            raise TypeError("describe() expected an int")
+        try:
+            self._inv_mapping
+        except AttributeError:
+            self._inv_mapping = [-1] * len(self._mapping)
+            for (info, i) in self._mapping.items():
+                self._inv_mapping[i] = info
+
+        if f_id < len(self._mapping):
+            (fname, fval, label) = self._inv_mapping[f_id]
+            return f"{fname}=={fval!r} and label is {label!r}"
+        elif self._alwayson and f_id in self._alwayson.values():
+            for (label, f_id2) in self._alwayson.items():
+                if f_id == f_id2:
+                    return "label is %r" % label
+        elif self._unseen and f_id in self._unseen.values():
+            for (fname, f_id2) in self._unseen.items():
+                if f_id == f_id2:
+                    return "%s is unseen" % fname
+        else:
+            raise ValueError("Bad feature id")
+
+    def labels(self):
+        # Inherit docs.
+        return self._labels
+
+    def length(self):
+        # Inherit docs.
+        return self._length
+
+    @classmethod
+    def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+        """
+        Construct and return new feature encoding, based on a given
+        training corpus ``train_toks``.  See the class description
+        ``TypedMaxentFeatureEncoding`` for a description of the
+        joint-features that will be included in this encoding.
+
+        Note: recognized feature values types are (int, float), over
+        types are interpreted as regular binary features.
+
+        :type train_toks: list(tuple(dict, str))
+        :param train_toks: Training data, represented as a list of
+            pairs, the first member of which is a feature dictionary,
+            and the second of which is a classification label.
+
+        :type count_cutoff: int
+        :param count_cutoff: A cutoff value that is used to discard
+            rare joint-features.  If a joint-feature's value is 1
+            fewer than ``count_cutoff`` times in the training corpus,
+            then that joint-feature is not included in the generated
+            encoding.
+
+        :type labels: list
+        :param labels: A list of labels that should be used by the
+            classifier.  If not specified, then the set of labels
+            attested in ``train_toks`` will be used.
+
+        :param options: Extra parameters for the constructor, such as
+            ``unseen_features`` and ``alwayson_features``.
+        """
+        mapping = {}  # maps (fname, fval, label) -> fid
+        seen_labels = set()  # The set of labels we've encountered
+        count = defaultdict(int)  # maps (fname, fval) -> count
+
+        for (tok, label) in train_toks:
+            if labels and label not in labels:
+                raise ValueError("Unexpected label %s" % label)
+            seen_labels.add(label)
+
+            # Record each of the features.
+            for (fname, fval) in tok.items():
+                if type(fval) in (int, float):
+                    fval = type(fval)
+                # If a count cutoff is given, then only add a joint
+                # feature once the corresponding (fname, fval, label)
+                # tuple exceeds that cutoff.
+                count[fname, fval] += 1
+                if count[fname, fval] >= count_cutoff:
+                    if (fname, fval, label) not in mapping:
+                        mapping[fname, fval, label] = len(mapping)
+
+        if labels is None:
+            labels = seen_labels
+        return cls(labels, mapping, **options)
+
+
+######################################################################
+# { Classifier Trainer: Generalized Iterative Scaling
+######################################################################
+
+
+def train_maxent_classifier_with_gis(
+    train_toks, trace=3, encoding=None, labels=None, **cutoffs
+):
+    """
+    Train a new ``ConditionalExponentialClassifier``, using the given
+    training samples, using the Generalized Iterative Scaling
+    algorithm.  This ``ConditionalExponentialClassifier`` will encode
+    the model that maximizes entropy from all the models that are
+    empirically consistent with ``train_toks``.
+
+    :see: ``train_maxent_classifier()`` for parameter descriptions.
+    """
+    cutoffs.setdefault("max_iter", 100)
+    cutoffchecker = CutoffChecker(cutoffs)
+
+    # Construct an encoding from the training data.
+    if encoding is None:
+        encoding = GISEncoding.train(train_toks, labels=labels)
+
+    if not hasattr(encoding, "C"):
+        raise TypeError(
+            "The GIS algorithm requires an encoding that "
+            "defines C (e.g., GISEncoding)."
+        )
+
+    # Cinv is the inverse of the sum of each joint feature vector.
+    # This controls the learning rate: higher Cinv (or lower C) gives
+    # faster learning.
+    Cinv = 1.0 / encoding.C
+
+    # Count how many times each feature occurs in the training data.
+    empirical_fcount = calculate_empirical_fcount(train_toks, encoding)
+
+    # Check for any features that are not attested in train_toks.
+    unattested = set(numpy.nonzero(empirical_fcount == 0)[0])
+
+    # Build the classifier.  Start with weight=0 for each attested
+    # feature, and weight=-infinity for each unattested feature.
+    weights = numpy.zeros(len(empirical_fcount), "d")
+    for fid in unattested:
+        weights[fid] = numpy.NINF
+    classifier = ConditionalExponentialClassifier(encoding, weights)
+
+    # Take the log of the empirical fcount.
+    log_empirical_fcount = numpy.log2(empirical_fcount)
+    del empirical_fcount
+
+    if trace > 0:
+        print("  ==> Training (%d iterations)" % cutoffs["max_iter"])
+    if trace > 2:
+        print()
+        print("      Iteration    Log Likelihood    Accuracy")
+        print("      ---------------------------------------")
+
+    # Train the classifier.
+    try:
+        while True:
+            if trace > 2:
+                ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
+                acc = cutoffchecker.acc or accuracy(classifier, train_toks)
+                iternum = cutoffchecker.iter
+                print("     %9d    %14.5f    %9.3f" % (iternum, ll, acc))
+
+            # Use the model to estimate the number of times each
+            # feature should occur in the training data.
+            estimated_fcount = calculate_estimated_fcount(
+                classifier, train_toks, encoding
+            )
+
+            # Take the log of estimated fcount (avoid taking log(0).)
+            for fid in unattested:
+                estimated_fcount[fid] += 1
+            log_estimated_fcount = numpy.log2(estimated_fcount)
+            del estimated_fcount
+
+            # Update the classifier weights
+            weights = classifier.weights()
+            weights += (log_empirical_fcount - log_estimated_fcount) * Cinv
+            classifier.set_weights(weights)
+
+            # Check the log-likelihood & accuracy cutoffs.
+            if cutoffchecker.check(classifier, train_toks):
+                break
+
+    except KeyboardInterrupt:
+        print("      Training stopped: keyboard interrupt")
+    except:
+        raise
+
+    if trace > 2:
+        ll = log_likelihood(classifier, train_toks)
+        acc = accuracy(classifier, train_toks)
+        print(f"         Final    {ll:14.5f}    {acc:9.3f}")
+
+    # Return the classifier.
+    return classifier
+
+
+def calculate_empirical_fcount(train_toks, encoding):
+    fcount = numpy.zeros(encoding.length(), "d")
+
+    for tok, label in train_toks:
+        for (index, val) in encoding.encode(tok, label):
+            fcount[index] += val
+
+    return fcount
+
+
+def calculate_estimated_fcount(classifier, train_toks, encoding):
+    fcount = numpy.zeros(encoding.length(), "d")
+
+    for tok, label in train_toks:
+        pdist = classifier.prob_classify(tok)
+        for label in pdist.samples():
+            prob = pdist.prob(label)
+            for (fid, fval) in encoding.encode(tok, label):
+                fcount[fid] += prob * fval
+
+    return fcount
+
+
+######################################################################
+# { Classifier Trainer: Improved Iterative Scaling
+######################################################################
+
+
+def train_maxent_classifier_with_iis(
+    train_toks, trace=3, encoding=None, labels=None, **cutoffs
+):
+    """
+    Train a new ``ConditionalExponentialClassifier``, using the given
+    training samples, using the Improved Iterative Scaling algorithm.
+    This ``ConditionalExponentialClassifier`` will encode the model
+    that maximizes entropy from all the models that are empirically
+    consistent with ``train_toks``.
+
+    :see: ``train_maxent_classifier()`` for parameter descriptions.
+    """
+    cutoffs.setdefault("max_iter", 100)
+    cutoffchecker = CutoffChecker(cutoffs)
+
+    # Construct an encoding from the training data.
+    if encoding is None:
+        encoding = BinaryMaxentFeatureEncoding.train(train_toks, labels=labels)
+
+    # Count how many times each feature occurs in the training data.
+    empirical_ffreq = calculate_empirical_fcount(train_toks, encoding) / len(train_toks)
+
+    # Find the nf map, and related variables nfarray and nfident.
+    # nf is the sum of the features for a given labeled text.
+    # nfmap compresses this sparse set of values to a dense list.
+    # nfarray performs the reverse operation.  nfident is
+    # nfarray multiplied by an identity matrix.
+    nfmap = calculate_nfmap(train_toks, encoding)
+    nfarray = numpy.array(sorted(nfmap, key=nfmap.__getitem__), "d")
+    nftranspose = numpy.reshape(nfarray, (len(nfarray), 1))
+
+    # Check for any features that are not attested in train_toks.
+    unattested = set(numpy.nonzero(empirical_ffreq == 0)[0])
+
+    # Build the classifier.  Start with weight=0 for each attested
+    # feature, and weight=-infinity for each unattested feature.
+    weights = numpy.zeros(len(empirical_ffreq), "d")
+    for fid in unattested:
+        weights[fid] = numpy.NINF
+    classifier = ConditionalExponentialClassifier(encoding, weights)
+
+    if trace > 0:
+        print("  ==> Training (%d iterations)" % cutoffs["max_iter"])
+    if trace > 2:
+        print()
+        print("      Iteration    Log Likelihood    Accuracy")
+        print("      ---------------------------------------")
+
+    # Train the classifier.
+    try:
+        while True:
+            if trace > 2:
+                ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
+                acc = cutoffchecker.acc or accuracy(classifier, train_toks)
+                iternum = cutoffchecker.iter
+                print("     %9d    %14.5f    %9.3f" % (iternum, ll, acc))
+
+            # Calculate the deltas for this iteration, using Newton's method.
+            deltas = calculate_deltas(
+                train_toks,
+                classifier,
+                unattested,
+                empirical_ffreq,
+                nfmap,
+                nfarray,
+                nftranspose,
+                encoding,
+            )
+
+            # Use the deltas to update our weights.
+            weights = classifier.weights()
+            weights += deltas
+            classifier.set_weights(weights)
+
+            # Check the log-likelihood & accuracy cutoffs.
+            if cutoffchecker.check(classifier, train_toks):
+                break
+
+    except KeyboardInterrupt:
+        print("      Training stopped: keyboard interrupt")
+    except:
+        raise
+
+    if trace > 2:
+        ll = log_likelihood(classifier, train_toks)
+        acc = accuracy(classifier, train_toks)
+        print(f"         Final    {ll:14.5f}    {acc:9.3f}")
+
+    # Return the classifier.
+    return classifier
+
+
+def calculate_nfmap(train_toks, encoding):
+    """
+    Construct a map that can be used to compress ``nf`` (which is
+    typically sparse).
+
+    *nf(feature_vector)* is the sum of the feature values for
+    *feature_vector*.
+
+    This represents the number of features that are active for a
+    given labeled text.  This method finds all values of *nf(t)*
+    that are attested for at least one token in the given list of
+    training tokens; and constructs a dictionary mapping these
+    attested values to a continuous range *0...N*.  For example,
+    if the only values of *nf()* that were attested were 3, 5, and
+    7, then ``_nfmap`` might return the dictionary ``{3:0, 5:1, 7:2}``.
+
+    :return: A map that can be used to compress ``nf`` to a dense
+        vector.
+    :rtype: dict(int -> int)
+    """
+    # Map from nf to indices.  This allows us to use smaller arrays.
+    nfset = set()
+    for tok, _ in train_toks:
+        for label in encoding.labels():
+            nfset.add(sum(val for (id, val) in encoding.encode(tok, label)))
+    return {nf: i for (i, nf) in enumerate(nfset)}
+
+
+def calculate_deltas(
+    train_toks,
+    classifier,
+    unattested,
+    ffreq_empirical,
+    nfmap,
+    nfarray,
+    nftranspose,
+    encoding,
+):
+    r"""
+    Calculate the update values for the classifier weights for
+    this iteration of IIS.  These update weights are the value of
+    ``delta`` that solves the equation::
+
+      ffreq_empirical[i]
+             =
+      SUM[fs,l] (classifier.prob_classify(fs).prob(l) *
+                 feature_vector(fs,l)[i] *
+                 exp(delta[i] * nf(feature_vector(fs,l))))
+
+    Where:
+        - *(fs,l)* is a (featureset, label) tuple from ``train_toks``
+        - *feature_vector(fs,l)* = ``encoding.encode(fs,l)``
+        - *nf(vector)* = ``sum([val for (id,val) in vector])``
+
+    This method uses Newton's method to solve this equation for
+    *delta[i]*.  In particular, it starts with a guess of
+    ``delta[i]`` = 1; and iteratively updates ``delta`` with:
+
+    | delta[i] -= (ffreq_empirical[i] - sum1[i])/(-sum2[i])
+
+    until convergence, where *sum1* and *sum2* are defined as:
+
+    |    sum1[i](delta) = SUM[fs,l] f[i](fs,l,delta)
+    |    sum2[i](delta) = SUM[fs,l] (f[i](fs,l,delta).nf(feature_vector(fs,l)))
+    |    f[i](fs,l,delta) = (classifier.prob_classify(fs).prob(l) .
+    |                        feature_vector(fs,l)[i] .
+    |                        exp(delta[i] . nf(feature_vector(fs,l))))
+
+    Note that *sum1* and *sum2* depend on ``delta``; so they need
+    to be re-computed each iteration.
+
+    The variables ``nfmap``, ``nfarray``, and ``nftranspose`` are
+    used to generate a dense encoding for *nf(ltext)*.  This
+    allows ``_deltas`` to calculate *sum1* and *sum2* using
+    matrices, which yields a significant performance improvement.
+
+    :param train_toks: The set of training tokens.
+    :type train_toks: list(tuple(dict, str))
+    :param classifier: The current classifier.
+    :type classifier: ClassifierI
+    :param ffreq_empirical: An array containing the empirical
+        frequency for each feature.  The *i*\ th element of this
+        array is the empirical frequency for feature *i*.
+    :type ffreq_empirical: sequence of float
+    :param unattested: An array that is 1 for features that are
+        not attested in the training data; and 0 for features that
+        are attested.  In other words, ``unattested[i]==0`` iff
+        ``ffreq_empirical[i]==0``.
+    :type unattested: sequence of int
+    :param nfmap: A map that can be used to compress ``nf`` to a dense
+        vector.
+    :type nfmap: dict(int -> int)
+    :param nfarray: An array that can be used to uncompress ``nf``
+        from a dense vector.
+    :type nfarray: array(float)
+    :param nftranspose: The transpose of ``nfarray``
+    :type nftranspose: array(float)
+    """
+    # These parameters control when we decide that we've
+    # converged.  It probably should be possible to set these
+    # manually, via keyword arguments to train.
+    NEWTON_CONVERGE = 1e-12
+    MAX_NEWTON = 300
+
+    deltas = numpy.ones(encoding.length(), "d")
+
+    # Precompute the A matrix:
+    # A[nf][id] = sum ( p(fs) * p(label|fs) * f(fs,label) )
+    # over all label,fs s.t. num_features[label,fs]=nf
+    A = numpy.zeros((len(nfmap), encoding.length()), "d")
+
+    for tok, label in train_toks:
+        dist = classifier.prob_classify(tok)
+
+        for label in encoding.labels():
+            # Generate the feature vector
+            feature_vector = encoding.encode(tok, label)
+            # Find the number of active features
+            nf = sum(val for (id, val) in feature_vector)
+            # Update the A matrix
+            for (id, val) in feature_vector:
+                A[nfmap[nf], id] += dist.prob(label) * val
+    A /= len(train_toks)
+
+    # Iteratively solve for delta.  Use the following variables:
+    #   - nf_delta[x][y] = nfarray[x] * delta[y]
+    #   - exp_nf_delta[x][y] = exp(nf[x] * delta[y])
+    #   - nf_exp_nf_delta[x][y] = nf[x] * exp(nf[x] * delta[y])
+    #   - sum1[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
+    #                       exp(delta[i]nf)
+    #   - sum2[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
+    #                       nf exp(delta[i]nf)
+    for rangenum in range(MAX_NEWTON):
+        nf_delta = numpy.outer(nfarray, deltas)
+        exp_nf_delta = 2**nf_delta
+        nf_exp_nf_delta = nftranspose * exp_nf_delta
+        sum1 = numpy.sum(exp_nf_delta * A, axis=0)
+        sum2 = numpy.sum(nf_exp_nf_delta * A, axis=0)
+
+        # Avoid division by zero.
+        for fid in unattested:
+            sum2[fid] += 1
+
+        # Update the deltas.
+        deltas -= (ffreq_empirical - sum1) / -sum2
+
+        # We can stop once we converge.
+        n_error = numpy.sum(abs(ffreq_empirical - sum1)) / numpy.sum(abs(deltas))
+        if n_error < NEWTON_CONVERGE:
+            return deltas
+
+    return deltas
+
+
+######################################################################
+# { Classifier Trainer: megam
+######################################################################
+
+# [xx] possible extension: add support for using implicit file format;
+# this would need to put requirements on what encoding is used.  But
+# we may need this for other maxent classifier trainers that require
+# implicit formats anyway.
+def train_maxent_classifier_with_megam(
+    train_toks, trace=3, encoding=None, labels=None, gaussian_prior_sigma=0, **kwargs
+):
+    """
+    Train a new ``ConditionalExponentialClassifier``, using the given
+    training samples, using the external ``megam`` library.  This
+    ``ConditionalExponentialClassifier`` will encode the model that
+    maximizes entropy from all the models that are empirically
+    consistent with ``train_toks``.
+
+    :see: ``train_maxent_classifier()`` for parameter descriptions.
+    :see: ``nltk.classify.megam``
+    """
+
+    explicit = True
+    bernoulli = True
+    if "explicit" in kwargs:
+        explicit = kwargs["explicit"]
+    if "bernoulli" in kwargs:
+        bernoulli = kwargs["bernoulli"]
+
+    # Construct an encoding from the training data.
+    if encoding is None:
+        # Count cutoff can also be controlled by megam with the -minfc
+        # option. Not sure where the best place for it is.
+        count_cutoff = kwargs.get("count_cutoff", 0)
+        encoding = BinaryMaxentFeatureEncoding.train(
+            train_toks, count_cutoff, labels=labels, alwayson_features=True
+        )
+    elif labels is not None:
+        raise ValueError("Specify encoding or labels, not both")
+
+    # Write a training file for megam.
+    try:
+        fd, trainfile_name = tempfile.mkstemp(prefix="nltk-")
+        with open(trainfile_name, "w") as trainfile:
+            write_megam_file(
+                train_toks, encoding, trainfile, explicit=explicit, bernoulli=bernoulli
+            )
+        os.close(fd)
+    except (OSError, ValueError) as e:
+        raise ValueError("Error while creating megam training file: %s" % e) from e
+
+    # Run megam on the training file.
+    options = []
+    options += ["-nobias", "-repeat", "10"]
+    if explicit:
+        options += ["-explicit"]
+    if not bernoulli:
+        options += ["-fvals"]
+    if gaussian_prior_sigma:
+        # Lambda is just the precision of the Gaussian prior, i.e. it's the
+        # inverse variance, so the parameter conversion is 1.0/sigma**2.
+        # See https://users.umiacs.umd.edu/~hal/docs/daume04cg-bfgs.pdf
+        inv_variance = 1.0 / gaussian_prior_sigma**2
+    else:
+        inv_variance = 0
+    options += ["-lambda", "%.2f" % inv_variance, "-tune"]
+    if trace < 3:
+        options += ["-quiet"]
+    if "max_iter" in kwargs:
+        options += ["-maxi", "%s" % kwargs["max_iter"]]
+    if "ll_delta" in kwargs:
+        # [xx] this is actually a perplexity delta, not a log
+        # likelihood delta
+        options += ["-dpp", "%s" % abs(kwargs["ll_delta"])]
+    if hasattr(encoding, "cost"):
+        options += ["-multilabel"]  # each possible la
+    options += ["multiclass", trainfile_name]
+    stdout = call_megam(options)
+    # print('./megam_i686.opt ', ' '.join(options))
+    # Delete the training file
+    try:
+        os.remove(trainfile_name)
+    except OSError as e:
+        print(f"Warning: unable to delete {trainfile_name}: {e}")
+
+    # Parse the generated weight vector.
+    weights = parse_megam_weights(stdout, encoding.length(), explicit)
+
+    # Convert from base-e to base-2 weights.
+    weights *= numpy.log2(numpy.e)
+
+    # Build the classifier
+    return MaxentClassifier(encoding, weights)
+
+
+######################################################################
+# { Classifier Trainer: tadm
+######################################################################
+
+
+class TadmMaxentClassifier(MaxentClassifier):
+    @classmethod
+    def train(cls, train_toks, **kwargs):
+        algorithm = kwargs.get("algorithm", "tao_lmvm")
+        trace = kwargs.get("trace", 3)
+        encoding = kwargs.get("encoding", None)
+        labels = kwargs.get("labels", None)
+        sigma = kwargs.get("gaussian_prior_sigma", 0)
+        count_cutoff = kwargs.get("count_cutoff", 0)
+        max_iter = kwargs.get("max_iter")
+        ll_delta = kwargs.get("min_lldelta")
+
+        # Construct an encoding from the training data.
+        if not encoding:
+            encoding = TadmEventMaxentFeatureEncoding.train(
+                train_toks, count_cutoff, labels=labels
+            )
+
+        trainfile_fd, trainfile_name = tempfile.mkstemp(
+            prefix="nltk-tadm-events-", suffix=".gz"
+        )
+        weightfile_fd, weightfile_name = tempfile.mkstemp(prefix="nltk-tadm-weights-")
+
+        trainfile = gzip_open_unicode(trainfile_name, "w")
+        write_tadm_file(train_toks, encoding, trainfile)
+        trainfile.close()
+
+        options = []
+        options.extend(["-monitor"])
+        options.extend(["-method", algorithm])
+        if sigma:
+            options.extend(["-l2", "%.6f" % sigma**2])
+        if max_iter:
+            options.extend(["-max_it", "%d" % max_iter])
+        if ll_delta:
+            options.extend(["-fatol", "%.6f" % abs(ll_delta)])
+        options.extend(["-events_in", trainfile_name])
+        options.extend(["-params_out", weightfile_name])
+        if trace < 3:
+            options.extend(["2>&1"])
+        else:
+            options.extend(["-summary"])
+
+        call_tadm(options)
+
+        with open(weightfile_name) as weightfile:
+            weights = parse_tadm_weights(weightfile)
+
+        os.remove(trainfile_name)
+        os.remove(weightfile_name)
+
+        # Convert from base-e to base-2 weights.
+        weights *= numpy.log2(numpy.e)
+
+        # Build the classifier
+        return cls(encoding, weights)
+
+
+######################################################################
+# { Demo
+######################################################################
+def demo():
+    from nltk.classify.util import names_demo
+
+    classifier = names_demo(MaxentClassifier.train)
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/megam.py

@@ -0,0 +1,184 @@
+# Natural Language Toolkit: Interface to Megam Classifier
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A set of functions used to interface with the external megam_ maxent
+optimization package. Before megam can be used, you should tell NLTK where it
+can find the megam binary, using the ``config_megam()`` function. Typical
+usage:
+
+    >>> from nltk.classify import megam
+    >>> megam.config_megam() # pass path to megam if not found in PATH # doctest: +SKIP
+    [Found megam: ...]
+
+Use with MaxentClassifier. Example below, see MaxentClassifier documentation
+for details.
+
+    nltk.classify.MaxentClassifier.train(corpus, 'megam')
+
+.. _megam: https://www.umiacs.umd.edu/~hal/megam/index.html
+"""
+import subprocess
+
+from nltk.internals import find_binary
+
+try:
+    import numpy
+except ImportError:
+    numpy = None
+
+######################################################################
+# { Configuration
+######################################################################
+
+_megam_bin = None
+
+
+def config_megam(bin=None):
+    """
+    Configure NLTK's interface to the ``megam`` maxent optimization
+    package.
+
+    :param bin: The full path to the ``megam`` binary.  If not specified,
+        then nltk will search the system for a ``megam`` binary; and if
+        one is not found, it will raise a ``LookupError`` exception.
+    :type bin: str
+    """
+    global _megam_bin
+    _megam_bin = find_binary(
+        "megam",
+        bin,
+        env_vars=["MEGAM"],
+        binary_names=["megam.opt", "megam", "megam_686", "megam_i686.opt"],
+        url="https://www.umiacs.umd.edu/~hal/megam/index.html",
+    )
+
+
+######################################################################
+# { Megam Interface Functions
+######################################################################
+
+
+def write_megam_file(train_toks, encoding, stream, bernoulli=True, explicit=True):
+    """
+    Generate an input file for ``megam`` based on the given corpus of
+    classified tokens.
+
+    :type train_toks: list(tuple(dict, str))
+    :param train_toks: Training data, represented as a list of
+        pairs, the first member of which is a feature dictionary,
+        and the second of which is a classification label.
+
+    :type encoding: MaxentFeatureEncodingI
+    :param encoding: A feature encoding, used to convert featuresets
+        into feature vectors. May optionally implement a cost() method
+        in order to assign different costs to different class predictions.
+
+    :type stream: stream
+    :param stream: The stream to which the megam input file should be
+        written.
+
+    :param bernoulli: If true, then use the 'bernoulli' format.  I.e.,
+        all joint features have binary values, and are listed iff they
+        are true.  Otherwise, list feature values explicitly.  If
+        ``bernoulli=False``, then you must call ``megam`` with the
+        ``-fvals`` option.
+
+    :param explicit: If true, then use the 'explicit' format.  I.e.,
+        list the features that would fire for any of the possible
+        labels, for each token.  If ``explicit=True``, then you must
+        call ``megam`` with the ``-explicit`` option.
+    """
+    # Look up the set of labels.
+    labels = encoding.labels()
+    labelnum = {label: i for (i, label) in enumerate(labels)}
+
+    # Write the file, which contains one line per instance.
+    for featureset, label in train_toks:
+        # First, the instance number (or, in the weighted multiclass case, the cost of each label).
+        if hasattr(encoding, "cost"):
+            stream.write(
+                ":".join(str(encoding.cost(featureset, label, l)) for l in labels)
+            )
+        else:
+            stream.write("%d" % labelnum[label])
+
+        # For implicit file formats, just list the features that fire
+        # for this instance's actual label.
+        if not explicit:
+            _write_megam_features(encoding.encode(featureset, label), stream, bernoulli)
+
+        # For explicit formats, list the features that would fire for
+        # any of the possible labels.
+        else:
+            for l in labels:
+                stream.write(" #")
+                _write_megam_features(encoding.encode(featureset, l), stream, bernoulli)
+
+        # End of the instance.
+        stream.write("\n")
+
+
+def parse_megam_weights(s, features_count, explicit=True):
+    """
+    Given the stdout output generated by ``megam`` when training a
+    model, return a ``numpy`` array containing the corresponding weight
+    vector.  This function does not currently handle bias features.
+    """
+    if numpy is None:
+        raise ValueError("This function requires that numpy be installed")
+    assert explicit, "non-explicit not supported yet"
+    lines = s.strip().split("\n")
+    weights = numpy.zeros(features_count, "d")
+    for line in lines:
+        if line.strip():
+            fid, weight = line.split()
+            weights[int(fid)] = float(weight)
+    return weights
+
+
+def _write_megam_features(vector, stream, bernoulli):
+    if not vector:
+        raise ValueError(
+            "MEGAM classifier requires the use of an " "always-on feature."
+        )
+    for (fid, fval) in vector:
+        if bernoulli:
+            if fval == 1:
+                stream.write(" %s" % fid)
+            elif fval != 0:
+                raise ValueError(
+                    "If bernoulli=True, then all" "features must be binary."
+                )
+        else:
+            stream.write(f" {fid} {fval}")
+
+
+def call_megam(args):
+    """
+    Call the ``megam`` binary with the given arguments.
+    """
+    if isinstance(args, str):
+        raise TypeError("args should be a list of strings")
+    if _megam_bin is None:
+        config_megam()
+
+    # Call megam via a subprocess
+    cmd = [_megam_bin] + args
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+
+    # Check the return code.
+    if p.returncode != 0:
+        print()
+        print(stderr)
+        raise OSError("megam command failed!")
+
+    if isinstance(stdout, str):
+        return stdout
+    else:
+        return stdout.decode("utf-8")

.eggs/nltk-3.8-py3.10.egg/nltk/classify/naivebayes.py

@@ -0,0 +1,260 @@
+# Natural Language Toolkit: Naive Bayes Classifiers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier based on the Naive Bayes algorithm.  In order to find the
+probability for a label, this algorithm first uses the Bayes rule to
+express P(label|features) in terms of P(label) and P(features|label):
+
+|                       P(label) * P(features|label)
+|  P(label|features) = ------------------------------
+|                              P(features)
+
+The algorithm then makes the 'naive' assumption that all features are
+independent, given the label:
+
+|                       P(label) * P(f1|label) * ... * P(fn|label)
+|  P(label|features) = --------------------------------------------
+|                                         P(features)
+
+Rather than computing P(features) explicitly, the algorithm just
+calculates the numerator for each label, and normalizes them so they
+sum to one:
+
+|                       P(label) * P(f1|label) * ... * P(fn|label)
+|  P(label|features) = --------------------------------------------
+|                        SUM[l]( P(l) * P(f1|l) * ... * P(fn|l) )
+"""
+
+from collections import defaultdict
+
+from nltk.classify.api import ClassifierI
+from nltk.probability import DictionaryProbDist, ELEProbDist, FreqDist, sum_logs
+
+##//////////////////////////////////////////////////////
+##  Naive Bayes Classifier
+##//////////////////////////////////////////////////////
+
+
+class NaiveBayesClassifier(ClassifierI):
+    """
+    A Naive Bayes classifier.  Naive Bayes classifiers are
+    paramaterized by two probability distributions:
+
+      - P(label) gives the probability that an input will receive each
+        label, given no information about the input's features.
+
+      - P(fname=fval|label) gives the probability that a given feature
+        (fname) will receive a given value (fval), given that the
+        label (label).
+
+    If the classifier encounters an input with a feature that has
+    never been seen with any label, then rather than assigning a
+    probability of 0 to all labels, it will ignore that feature.
+
+    The feature value 'None' is reserved for unseen feature values;
+    you generally should not use 'None' as a feature value for one of
+    your own features.
+    """
+
+    def __init__(self, label_probdist, feature_probdist):
+        """
+        :param label_probdist: P(label), the probability distribution
+            over labels.  It is expressed as a ``ProbDistI`` whose
+            samples are labels.  I.e., P(label) =
+            ``label_probdist.prob(label)``.
+
+        :param feature_probdist: P(fname=fval|label), the probability
+            distribution for feature values, given labels.  It is
+            expressed as a dictionary whose keys are ``(label, fname)``
+            pairs and whose values are ``ProbDistI`` objects over feature
+            values.  I.e., P(fname=fval|label) =
+            ``feature_probdist[label,fname].prob(fval)``.  If a given
+            ``(label,fname)`` is not a key in ``feature_probdist``, then
+            it is assumed that the corresponding P(fname=fval|label)
+            is 0 for all values of ``fval``.
+        """
+        self._label_probdist = label_probdist
+        self._feature_probdist = feature_probdist
+        self._labels = list(label_probdist.samples())
+
+    def labels(self):
+        return self._labels
+
+    def classify(self, featureset):
+        return self.prob_classify(featureset).max()
+
+    def prob_classify(self, featureset):
+        # Discard any feature names that we've never seen before.
+        # Otherwise, we'll just assign a probability of 0 to
+        # everything.
+        featureset = featureset.copy()
+        for fname in list(featureset.keys()):
+            for label in self._labels:
+                if (label, fname) in self._feature_probdist:
+                    break
+            else:
+                # print('Ignoring unseen feature %s' % fname)
+                del featureset[fname]
+
+        # Find the log probability of each label, given the features.
+        # Start with the log probability of the label itself.
+        logprob = {}
+        for label in self._labels:
+            logprob[label] = self._label_probdist.logprob(label)
+
+        # Then add in the log probability of features given labels.
+        for label in self._labels:
+            for (fname, fval) in featureset.items():
+                if (label, fname) in self._feature_probdist:
+                    feature_probs = self._feature_probdist[label, fname]
+                    logprob[label] += feature_probs.logprob(fval)
+                else:
+                    # nb: This case will never come up if the
+                    # classifier was created by
+                    # NaiveBayesClassifier.train().
+                    logprob[label] += sum_logs([])  # = -INF.
+
+        return DictionaryProbDist(logprob, normalize=True, log=True)
+
+    def show_most_informative_features(self, n=10):
+        # Determine the most relevant features, and display them.
+        cpdist = self._feature_probdist
+        print("Most Informative Features")
+
+        for (fname, fval) in self.most_informative_features(n):
+
+            def labelprob(l):
+                return cpdist[l, fname].prob(fval)
+
+            labels = sorted(
+                (l for l in self._labels if fval in cpdist[l, fname].samples()),
+                key=lambda element: (-labelprob(element), element),
+                reverse=True,
+            )
+            if len(labels) == 1:
+                continue
+            l0 = labels[0]
+            l1 = labels[-1]
+            if cpdist[l0, fname].prob(fval) == 0:
+                ratio = "INF"
+            else:
+                ratio = "%8.1f" % (
+                    cpdist[l1, fname].prob(fval) / cpdist[l0, fname].prob(fval)
+                )
+            print(
+                "%24s = %-14r %6s : %-6s = %s : 1.0"
+                % (fname, fval, ("%s" % l1)[:6], ("%s" % l0)[:6], ratio)
+            )
+
+    def most_informative_features(self, n=100):
+        """
+        Return a list of the 'most informative' features used by this
+        classifier.  For the purpose of this function, the
+        informativeness of a feature ``(fname,fval)`` is equal to the
+        highest value of P(fname=fval|label), for any label, divided by
+        the lowest value of P(fname=fval|label), for any label:
+
+        |  max[ P(fname=fval|label1) / P(fname=fval|label2) ]
+        """
+        if hasattr(self, "_most_informative_features"):
+            return self._most_informative_features[:n]
+        else:
+            # The set of (fname, fval) pairs used by this classifier.
+            features = set()
+            # The max & min probability associated w/ each (fname, fval)
+            # pair.  Maps (fname,fval) -> float.
+            maxprob = defaultdict(lambda: 0.0)
+            minprob = defaultdict(lambda: 1.0)
+
+            for (label, fname), probdist in self._feature_probdist.items():
+                for fval in probdist.samples():
+                    feature = (fname, fval)
+                    features.add(feature)
+                    p = probdist.prob(fval)
+                    maxprob[feature] = max(p, maxprob[feature])
+                    minprob[feature] = min(p, minprob[feature])
+                    if minprob[feature] == 0:
+                        features.discard(feature)
+
+            # Convert features to a list, & sort it by how informative
+            # features are.
+            self._most_informative_features = sorted(
+                features,
+                key=lambda feature_: (
+                    minprob[feature_] / maxprob[feature_],
+                    feature_[0],
+                    feature_[1] in [None, False, True],
+                    str(feature_[1]).lower(),
+                ),
+            )
+        return self._most_informative_features[:n]
+
+    @classmethod
+    def train(cls, labeled_featuresets, estimator=ELEProbDist):
+        """
+        :param labeled_featuresets: A list of classified featuresets,
+            i.e., a list of tuples ``(featureset, label)``.
+        """
+        label_freqdist = FreqDist()
+        feature_freqdist = defaultdict(FreqDist)
+        feature_values = defaultdict(set)
+        fnames = set()
+
+        # Count up how many times each feature value occurred, given
+        # the label and featurename.
+        for featureset, label in labeled_featuresets:
+            label_freqdist[label] += 1
+            for fname, fval in featureset.items():
+                # Increment freq(fval|label, fname)
+                feature_freqdist[label, fname][fval] += 1
+                # Record that fname can take the value fval.
+                feature_values[fname].add(fval)
+                # Keep a list of all feature names.
+                fnames.add(fname)
+
+        # If a feature didn't have a value given for an instance, then
+        # we assume that it gets the implicit value 'None.'  This loop
+        # counts up the number of 'missing' feature values for each
+        # (label,fname) pair, and increments the count of the fval
+        # 'None' by that amount.
+        for label in label_freqdist:
+            num_samples = label_freqdist[label]
+            for fname in fnames:
+                count = feature_freqdist[label, fname].N()
+                # Only add a None key when necessary, i.e. if there are
+                # any samples with feature 'fname' missing.
+                if num_samples - count > 0:
+                    feature_freqdist[label, fname][None] += num_samples - count
+                    feature_values[fname].add(None)
+
+        # Create the P(label) distribution
+        label_probdist = estimator(label_freqdist)
+
+        # Create the P(fval|label, fname) distribution
+        feature_probdist = {}
+        for ((label, fname), freqdist) in feature_freqdist.items():
+            probdist = estimator(freqdist, bins=len(feature_values[fname]))
+            feature_probdist[label, fname] = probdist
+
+        return cls(label_probdist, feature_probdist)
+
+
+##//////////////////////////////////////////////////////
+##  Demo
+##//////////////////////////////////////////////////////
+
+
+def demo():
+    from nltk.classify.util import names_demo
+
+    classifier = names_demo(NaiveBayesClassifier.train)
+    classifier.show_most_informative_features()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/positivenaivebayes.py

@@ -0,0 +1,180 @@
+# Natural Language Toolkit: Positive Naive Bayes Classifier
+#
+# Copyright (C) 2012 NLTK Project
+# Author: Alessandro Presta <alessandro.presta@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A variant of the Naive Bayes Classifier that performs binary classification with
+partially-labeled training sets. In other words, assume we want to build a classifier
+that assigns each example to one of two complementary classes (e.g., male names and
+female names).
+If we have a training set with labeled examples for both classes, we can use a
+standard Naive Bayes Classifier. However, consider the case when we only have labeled
+examples for one of the classes, and other, unlabeled, examples.
+Then, assuming a prior distribution on the two labels, we can use the unlabeled set
+to estimate the frequencies of the various features.
+
+Let the two possible labels be 1 and 0, and let's say we only have examples labeled 1
+and unlabeled examples. We are also given an estimate of P(1).
+
+We compute P(feature|1) exactly as in the standard case.
+
+To compute P(feature|0), we first estimate P(feature) from the unlabeled set (we are
+assuming that the unlabeled examples are drawn according to the given prior distribution)
+and then express the conditional probability as:
+
+|                  P(feature) - P(feature|1) * P(1)
+|  P(feature|0) = ----------------------------------
+|                               P(0)
+
+Example:
+
+    >>> from nltk.classify import PositiveNaiveBayesClassifier
+
+Some sentences about sports:
+
+    >>> sports_sentences = [ 'The team dominated the game',
+    ...                      'They lost the ball',
+    ...                      'The game was intense',
+    ...                      'The goalkeeper catched the ball',
+    ...                      'The other team controlled the ball' ]
+
+Mixed topics, including sports:
+
+    >>> various_sentences = [ 'The President did not comment',
+    ...                       'I lost the keys',
+    ...                       'The team won the game',
+    ...                       'Sara has two kids',
+    ...                       'The ball went off the court',
+    ...                       'They had the ball for the whole game',
+    ...                       'The show is over' ]
+
+The features of a sentence are simply the words it contains:
+
+    >>> def features(sentence):
+    ...     words = sentence.lower().split()
+    ...     return dict(('contains(%s)' % w, True) for w in words)
+
+We use the sports sentences as positive examples, the mixed ones ad unlabeled examples:
+
+    >>> positive_featuresets = map(features, sports_sentences)
+    >>> unlabeled_featuresets = map(features, various_sentences)
+    >>> classifier = PositiveNaiveBayesClassifier.train(positive_featuresets,
+    ...                                                 unlabeled_featuresets)
+
+Is the following sentence about sports?
+
+    >>> classifier.classify(features('The cat is on the table'))
+    False
+
+What about this one?
+
+    >>> classifier.classify(features('My team lost the game'))
+    True
+"""
+
+from collections import defaultdict
+
+from nltk.classify.naivebayes import NaiveBayesClassifier
+from nltk.probability import DictionaryProbDist, ELEProbDist, FreqDist
+
+##//////////////////////////////////////////////////////
+##  Positive Naive Bayes Classifier
+##//////////////////////////////////////////////////////
+
+
+class PositiveNaiveBayesClassifier(NaiveBayesClassifier):
+    @staticmethod
+    def train(
+        positive_featuresets,
+        unlabeled_featuresets,
+        positive_prob_prior=0.5,
+        estimator=ELEProbDist,
+    ):
+        """
+        :param positive_featuresets: An iterable of featuresets that are known as positive
+            examples (i.e., their label is ``True``).
+
+        :param unlabeled_featuresets: An iterable of featuresets whose label is unknown.
+
+        :param positive_prob_prior: A prior estimate of the probability of the label
+            ``True`` (default 0.5).
+        """
+        positive_feature_freqdist = defaultdict(FreqDist)
+        unlabeled_feature_freqdist = defaultdict(FreqDist)
+        feature_values = defaultdict(set)
+        fnames = set()
+
+        # Count up how many times each feature value occurred in positive examples.
+        num_positive_examples = 0
+        for featureset in positive_featuresets:
+            for fname, fval in featureset.items():
+                positive_feature_freqdist[fname][fval] += 1
+                feature_values[fname].add(fval)
+                fnames.add(fname)
+            num_positive_examples += 1
+
+        # Count up how many times each feature value occurred in unlabeled examples.
+        num_unlabeled_examples = 0
+        for featureset in unlabeled_featuresets:
+            for fname, fval in featureset.items():
+                unlabeled_feature_freqdist[fname][fval] += 1
+                feature_values[fname].add(fval)
+                fnames.add(fname)
+            num_unlabeled_examples += 1
+
+        # If a feature didn't have a value given for an instance, then we assume that
+        # it gets the implicit value 'None'.
+        for fname in fnames:
+            count = positive_feature_freqdist[fname].N()
+            positive_feature_freqdist[fname][None] += num_positive_examples - count
+            feature_values[fname].add(None)
+
+        for fname in fnames:
+            count = unlabeled_feature_freqdist[fname].N()
+            unlabeled_feature_freqdist[fname][None] += num_unlabeled_examples - count
+            feature_values[fname].add(None)
+
+        negative_prob_prior = 1.0 - positive_prob_prior
+
+        # Create the P(label) distribution.
+        label_probdist = DictionaryProbDist(
+            {True: positive_prob_prior, False: negative_prob_prior}
+        )
+
+        # Create the P(fval|label, fname) distribution.
+        feature_probdist = {}
+        for fname, freqdist in positive_feature_freqdist.items():
+            probdist = estimator(freqdist, bins=len(feature_values[fname]))
+            feature_probdist[True, fname] = probdist
+
+        for fname, freqdist in unlabeled_feature_freqdist.items():
+            global_probdist = estimator(freqdist, bins=len(feature_values[fname]))
+            negative_feature_probs = {}
+            for fval in feature_values[fname]:
+                prob = (
+                    global_probdist.prob(fval)
+                    - positive_prob_prior * feature_probdist[True, fname].prob(fval)
+                ) / negative_prob_prior
+                # TODO: We need to add some kind of smoothing here, instead of
+                # setting negative probabilities to zero and normalizing.
+                negative_feature_probs[fval] = max(prob, 0.0)
+            feature_probdist[False, fname] = DictionaryProbDist(
+                negative_feature_probs, normalize=True
+            )
+
+        return PositiveNaiveBayesClassifier(label_probdist, feature_probdist)
+
+
+##//////////////////////////////////////////////////////
+##  Demo
+##//////////////////////////////////////////////////////
+
+
+def demo():
+    from nltk.classify.util import partial_names_demo
+
+    classifier = partial_names_demo(PositiveNaiveBayesClassifier.train)
+    classifier.show_most_informative_features()

.eggs/nltk-3.8-py3.10.egg/nltk/classify/util.py

@@ -0,0 +1,346 @@
+# Natural Language Toolkit: Classifier Utility Functions
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Utility functions and classes for classifiers.
+"""
+
+import math
+
+# from nltk.util import Deprecated
+import nltk.classify.util  # for accuracy & log_likelihood
+from nltk.util import LazyMap
+
+######################################################################
+# { Helper Functions
+######################################################################
+
+# alternative name possibility: 'map_featurefunc()'?
+# alternative name possibility: 'detect_features()'?
+# alternative name possibility: 'map_featuredetect()'?
+# or.. just have users use LazyMap directly?
+def apply_features(feature_func, toks, labeled=None):
+    """
+    Use the ``LazyMap`` class to construct a lazy list-like
+    object that is analogous to ``map(feature_func, toks)``.  In
+    particular, if ``labeled=False``, then the returned list-like
+    object's values are equal to::
+
+        [feature_func(tok) for tok in toks]
+
+    If ``labeled=True``, then the returned list-like object's values
+    are equal to::
+
+        [(feature_func(tok), label) for (tok, label) in toks]
+
+    The primary purpose of this function is to avoid the memory
+    overhead involved in storing all the featuresets for every token
+    in a corpus.  Instead, these featuresets are constructed lazily,
+    as-needed.  The reduction in memory overhead can be especially
+    significant when the underlying list of tokens is itself lazy (as
+    is the case with many corpus readers).
+
+    :param feature_func: The function that will be applied to each
+        token.  It should return a featureset -- i.e., a dict
+        mapping feature names to feature values.
+    :param toks: The list of tokens to which ``feature_func`` should be
+        applied.  If ``labeled=True``, then the list elements will be
+        passed directly to ``feature_func()``.  If ``labeled=False``,
+        then the list elements should be tuples ``(tok,label)``, and
+        ``tok`` will be passed to ``feature_func()``.
+    :param labeled: If true, then ``toks`` contains labeled tokens --
+        i.e., tuples of the form ``(tok, label)``.  (Default:
+        auto-detect based on types.)
+    """
+    if labeled is None:
+        labeled = toks and isinstance(toks[0], (tuple, list))
+    if labeled:
+
+        def lazy_func(labeled_token):
+            return (feature_func(labeled_token[0]), labeled_token[1])
+
+        return LazyMap(lazy_func, toks)
+    else:
+        return LazyMap(feature_func, toks)
+
+
+def attested_labels(tokens):
+    """
+    :return: A list of all labels that are attested in the given list
+        of tokens.
+    :rtype: list of (immutable)
+    :param tokens: The list of classified tokens from which to extract
+        labels.  A classified token has the form ``(token, label)``.
+    :type tokens: list
+    """
+    return tuple({label for (tok, label) in tokens})
+
+
+def log_likelihood(classifier, gold):
+    results = classifier.prob_classify_many([fs for (fs, l) in gold])
+    ll = [pdist.prob(l) for ((fs, l), pdist) in zip(gold, results)]
+    return math.log(sum(ll) / len(ll))
+
+
+def accuracy(classifier, gold):
+    results = classifier.classify_many([fs for (fs, l) in gold])
+    correct = [l == r for ((fs, l), r) in zip(gold, results)]
+    if correct:
+        return sum(correct) / len(correct)
+    else:
+        return 0
+
+
+class CutoffChecker:
+    """
+    A helper class that implements cutoff checks based on number of
+    iterations and log likelihood.
+
+    Accuracy cutoffs are also implemented, but they're almost never
+    a good idea to use.
+    """
+
+    def __init__(self, cutoffs):
+        self.cutoffs = cutoffs.copy()
+        if "min_ll" in cutoffs:
+            cutoffs["min_ll"] = -abs(cutoffs["min_ll"])
+        if "min_lldelta" in cutoffs:
+            cutoffs["min_lldelta"] = abs(cutoffs["min_lldelta"])
+        self.ll = None
+        self.acc = None
+        self.iter = 1
+
+    def check(self, classifier, train_toks):
+        cutoffs = self.cutoffs
+        self.iter += 1
+        if "max_iter" in cutoffs and self.iter >= cutoffs["max_iter"]:
+            return True  # iteration cutoff.
+
+        new_ll = nltk.classify.util.log_likelihood(classifier, train_toks)
+        if math.isnan(new_ll):
+            return True
+
+        if "min_ll" in cutoffs or "min_lldelta" in cutoffs:
+            if "min_ll" in cutoffs and new_ll >= cutoffs["min_ll"]:
+                return True  # log likelihood cutoff
+            if (
+                "min_lldelta" in cutoffs
+                and self.ll
+                and ((new_ll - self.ll) <= abs(cutoffs["min_lldelta"]))
+            ):
+                return True  # log likelihood delta cutoff
+            self.ll = new_ll
+
+        if "max_acc" in cutoffs or "min_accdelta" in cutoffs:
+            new_acc = nltk.classify.util.log_likelihood(classifier, train_toks)
+            if "max_acc" in cutoffs and new_acc >= cutoffs["max_acc"]:
+                return True  # log likelihood cutoff
+            if (
+                "min_accdelta" in cutoffs
+                and self.acc
+                and ((new_acc - self.acc) <= abs(cutoffs["min_accdelta"]))
+            ):
+                return True  # log likelihood delta cutoff
+            self.acc = new_acc
+
+            return False  # no cutoff reached.
+
+
+######################################################################
+# { Demos
+######################################################################
+
+
+def names_demo_features(name):
+    features = {}
+    features["alwayson"] = True
+    features["startswith"] = name[0].lower()
+    features["endswith"] = name[-1].lower()
+    for letter in "abcdefghijklmnopqrstuvwxyz":
+        features["count(%s)" % letter] = name.lower().count(letter)
+        features["has(%s)" % letter] = letter in name.lower()
+    return features
+
+
+def binary_names_demo_features(name):
+    features = {}
+    features["alwayson"] = True
+    features["startswith(vowel)"] = name[0].lower() in "aeiouy"
+    features["endswith(vowel)"] = name[-1].lower() in "aeiouy"
+    for letter in "abcdefghijklmnopqrstuvwxyz":
+        features["count(%s)" % letter] = name.lower().count(letter)
+        features["has(%s)" % letter] = letter in name.lower()
+        features["startswith(%s)" % letter] = letter == name[0].lower()
+        features["endswith(%s)" % letter] = letter == name[-1].lower()
+    return features
+
+
+def names_demo(trainer, features=names_demo_features):
+    import random
+
+    from nltk.corpus import names
+
+    # Construct a list of classified names, using the names corpus.
+    namelist = [(name, "male") for name in names.words("male.txt")] + [
+        (name, "female") for name in names.words("female.txt")
+    ]
+
+    # Randomly split the names into a test & train set.
+    random.seed(123456)
+    random.shuffle(namelist)
+    train = namelist[:5000]
+    test = namelist[5000:5500]
+
+    # Train up a classifier.
+    print("Training classifier...")
+    classifier = trainer([(features(n), g) for (n, g) in train])
+
+    # Run the classifier on the test data.
+    print("Testing classifier...")
+    acc = accuracy(classifier, [(features(n), g) for (n, g) in test])
+    print("Accuracy: %6.4f" % acc)
+
+    # For classifiers that can find probabilities, show the log
+    # likelihood and some sample probability distributions.
+    try:
+        test_featuresets = [features(n) for (n, g) in test]
+        pdists = classifier.prob_classify_many(test_featuresets)
+        ll = [pdist.logprob(gold) for ((name, gold), pdist) in zip(test, pdists)]
+        print("Avg. log likelihood: %6.4f" % (sum(ll) / len(test)))
+        print()
+        print("Unseen Names      P(Male)  P(Female)\n" + "-" * 40)
+        for ((name, gender), pdist) in list(zip(test, pdists))[:5]:
+            if gender == "male":
+                fmt = "  %-15s *%6.4f   %6.4f"
+            else:
+                fmt = "  %-15s  %6.4f  *%6.4f"
+            print(fmt % (name, pdist.prob("male"), pdist.prob("female")))
+    except NotImplementedError:
+        pass
+
+    # Return the classifier
+    return classifier
+
+
+def partial_names_demo(trainer, features=names_demo_features):
+    import random
+
+    from nltk.corpus import names
+
+    male_names = names.words("male.txt")
+    female_names = names.words("female.txt")
+
+    random.seed(654321)
+    random.shuffle(male_names)
+    random.shuffle(female_names)
+
+    # Create a list of male names to be used as positive-labeled examples for training
+    positive = map(features, male_names[:2000])
+
+    # Create a list of male and female names to be used as unlabeled examples
+    unlabeled = map(features, male_names[2000:2500] + female_names[:500])
+
+    # Create a test set with correctly-labeled male and female names
+    test = [(name, True) for name in male_names[2500:2750]] + [
+        (name, False) for name in female_names[500:750]
+    ]
+
+    random.shuffle(test)
+
+    # Train up a classifier.
+    print("Training classifier...")
+    classifier = trainer(positive, unlabeled)
+
+    # Run the classifier on the test data.
+    print("Testing classifier...")
+    acc = accuracy(classifier, [(features(n), m) for (n, m) in test])
+    print("Accuracy: %6.4f" % acc)
+
+    # For classifiers that can find probabilities, show the log
+    # likelihood and some sample probability distributions.
+    try:
+        test_featuresets = [features(n) for (n, m) in test]
+        pdists = classifier.prob_classify_many(test_featuresets)
+        ll = [pdist.logprob(gold) for ((name, gold), pdist) in zip(test, pdists)]
+        print("Avg. log likelihood: %6.4f" % (sum(ll) / len(test)))
+        print()
+        print("Unseen Names      P(Male)  P(Female)\n" + "-" * 40)
+        for ((name, is_male), pdist) in zip(test, pdists)[:5]:
+            if is_male == True:
+                fmt = "  %-15s *%6.4f   %6.4f"
+            else:
+                fmt = "  %-15s  %6.4f  *%6.4f"
+            print(fmt % (name, pdist.prob(True), pdist.prob(False)))
+    except NotImplementedError:
+        pass
+
+    # Return the classifier
+    return classifier
+
+
+_inst_cache = {}
+
+
+def wsd_demo(trainer, word, features, n=1000):
+    import random
+
+    from nltk.corpus import senseval
+
+    # Get the instances.
+    print("Reading data...")
+    global _inst_cache
+    if word not in _inst_cache:
+        _inst_cache[word] = [(i, i.senses[0]) for i in senseval.instances(word)]
+    instances = _inst_cache[word][:]
+    if n > len(instances):
+        n = len(instances)
+    senses = list({l for (i, l) in instances})
+    print("  Senses: " + " ".join(senses))
+
+    # Randomly split the names into a test & train set.
+    print("Splitting into test & train...")
+    random.seed(123456)
+    random.shuffle(instances)
+    train = instances[: int(0.8 * n)]
+    test = instances[int(0.8 * n) : n]
+
+    # Train up a classifier.
+    print("Training classifier...")
+    classifier = trainer([(features(i), l) for (i, l) in train])
+
+    # Run the classifier on the test data.
+    print("Testing classifier...")
+    acc = accuracy(classifier, [(features(i), l) for (i, l) in test])
+    print("Accuracy: %6.4f" % acc)
+
+    # For classifiers that can find probabilities, show the log
+    # likelihood and some sample probability distributions.
+    try:
+        test_featuresets = [features(i) for (i, n) in test]
+        pdists = classifier.prob_classify_many(test_featuresets)
+        ll = [pdist.logprob(gold) for ((name, gold), pdist) in zip(test, pdists)]
+        print("Avg. log likelihood: %6.4f" % (sum(ll) / len(test)))
+    except NotImplementedError:
+        pass
+
+    # Return the classifier
+    return classifier
+
+
+def check_megam_config():
+    """
+    Checks whether the MEGAM binary is configured.
+    """
+    try:
+        _megam_bin
+    except NameError as e:
+        err_msg = str(
+            "Please configure your megam binary first, e.g.\n"
+            ">>> nltk.config_megam('/usr/bin/local/megam')"
+        )
+        raise NameError(err_msg) from e

.eggs/nltk-3.8-py3.10.egg/nltk/classify/weka.py

@@ -0,0 +1,377 @@
+# Natural Language Toolkit: Interface to Weka Classsifiers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classifiers that make use of the external 'Weka' package.
+"""
+
+import os
+import re
+import subprocess
+import tempfile
+import time
+import zipfile
+from sys import stdin
+
+from nltk.classify.api import ClassifierI
+from nltk.internals import config_java, java
+from nltk.probability import DictionaryProbDist
+
+_weka_classpath = None
+_weka_search = [
+    ".",
+    "/usr/share/weka",
+    "/usr/local/share/weka",
+    "/usr/lib/weka",
+    "/usr/local/lib/weka",
+]
+
+
+def config_weka(classpath=None):
+    global _weka_classpath
+
+    # Make sure java's configured first.
+    config_java()
+
+    if classpath is not None:
+        _weka_classpath = classpath
+
+    if _weka_classpath is None:
+        searchpath = _weka_search
+        if "WEKAHOME" in os.environ:
+            searchpath.insert(0, os.environ["WEKAHOME"])
+
+        for path in searchpath:
+            if os.path.exists(os.path.join(path, "weka.jar")):
+                _weka_classpath = os.path.join(path, "weka.jar")
+                version = _check_weka_version(_weka_classpath)
+                if version:
+                    print(f"[Found Weka: {_weka_classpath} (version {version})]")
+                else:
+                    print("[Found Weka: %s]" % _weka_classpath)
+                _check_weka_version(_weka_classpath)
+
+    if _weka_classpath is None:
+        raise LookupError(
+            "Unable to find weka.jar!  Use config_weka() "
+            "or set the WEKAHOME environment variable. "
+            "For more information about Weka, please see "
+            "https://www.cs.waikato.ac.nz/ml/weka/"
+        )
+
+
+def _check_weka_version(jar):
+    try:
+        zf = zipfile.ZipFile(jar)
+    except (SystemExit, KeyboardInterrupt):
+        raise
+    except:
+        return None
+    try:
+        try:
+            return zf.read("weka/core/version.txt")
+        except KeyError:
+            return None
+    finally:
+        zf.close()
+
+
+class WekaClassifier(ClassifierI):
+    def __init__(self, formatter, model_filename):
+        self._formatter = formatter
+        self._model = model_filename
+
+    def prob_classify_many(self, featuresets):
+        return self._classify_many(featuresets, ["-p", "0", "-distribution"])
+
+    def classify_many(self, featuresets):
+        return self._classify_many(featuresets, ["-p", "0"])
+
+    def _classify_many(self, featuresets, options):
+        # Make sure we can find java & weka.
+        config_weka()
+
+        temp_dir = tempfile.mkdtemp()
+        try:
+            # Write the test data file.
+            test_filename = os.path.join(temp_dir, "test.arff")
+            self._formatter.write(test_filename, featuresets)
+
+            # Call weka to classify the data.
+            cmd = [
+                "weka.classifiers.bayes.NaiveBayes",
+                "-l",
+                self._model,
+                "-T",
+                test_filename,
+            ] + options
+            (stdout, stderr) = java(
+                cmd,
+                classpath=_weka_classpath,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+
+            # Check if something went wrong:
+            if stderr and not stdout:
+                if "Illegal options: -distribution" in stderr:
+                    raise ValueError(
+                        "The installed version of weka does "
+                        "not support probability distribution "
+                        "output."
+                    )
+                else:
+                    raise ValueError("Weka failed to generate output:\n%s" % stderr)
+
+            # Parse weka's output.
+            return self.parse_weka_output(stdout.decode(stdin.encoding).split("\n"))
+
+        finally:
+            for f in os.listdir(temp_dir):
+                os.remove(os.path.join(temp_dir, f))
+            os.rmdir(temp_dir)
+
+    def parse_weka_distribution(self, s):
+        probs = [float(v) for v in re.split("[*,]+", s) if v.strip()]
+        probs = dict(zip(self._formatter.labels(), probs))
+        return DictionaryProbDist(probs)
+
+    def parse_weka_output(self, lines):
+        # Strip unwanted text from stdout
+        for i, line in enumerate(lines):
+            if line.strip().startswith("inst#"):
+                lines = lines[i:]
+                break
+
+        if lines[0].split() == ["inst#", "actual", "predicted", "error", "prediction"]:
+            return [line.split()[2].split(":")[1] for line in lines[1:] if line.strip()]
+        elif lines[0].split() == [
+            "inst#",
+            "actual",
+            "predicted",
+            "error",
+            "distribution",
+        ]:
+            return [
+                self.parse_weka_distribution(line.split()[-1])
+                for line in lines[1:]
+                if line.strip()
+            ]
+
+        # is this safe:?
+        elif re.match(r"^0 \w+ [01]\.[0-9]* \?\s*$", lines[0]):
+            return [line.split()[1] for line in lines if line.strip()]
+
+        else:
+            for line in lines[:10]:
+                print(line)
+            raise ValueError(
+                "Unhandled output format -- your version "
+                "of weka may not be supported.\n"
+                "  Header: %s" % lines[0]
+            )
+
+    # [xx] full list of classifiers (some may be abstract?):
+    # ADTree, AODE, BayesNet, ComplementNaiveBayes, ConjunctiveRule,
+    # DecisionStump, DecisionTable, HyperPipes, IB1, IBk, Id3, J48,
+    # JRip, KStar, LBR, LeastMedSq, LinearRegression, LMT, Logistic,
+    # LogisticBase, M5Base, MultilayerPerceptron,
+    # MultipleClassifiersCombiner, NaiveBayes, NaiveBayesMultinomial,
+    # NaiveBayesSimple, NBTree, NNge, OneR, PaceRegression, PART,
+    # PreConstructedLinearModel, Prism, RandomForest,
+    # RandomizableClassifier, RandomTree, RBFNetwork, REPTree, Ridor,
+    # RuleNode, SimpleLinearRegression, SimpleLogistic,
+    # SingleClassifierEnhancer, SMO, SMOreg, UserClassifier, VFI,
+    # VotedPerceptron, Winnow, ZeroR
+
+    _CLASSIFIER_CLASS = {
+        "naivebayes": "weka.classifiers.bayes.NaiveBayes",
+        "C4.5": "weka.classifiers.trees.J48",
+        "log_regression": "weka.classifiers.functions.Logistic",
+        "svm": "weka.classifiers.functions.SMO",
+        "kstar": "weka.classifiers.lazy.KStar",
+        "ripper": "weka.classifiers.rules.JRip",
+    }
+
+    @classmethod
+    def train(
+        cls,
+        model_filename,
+        featuresets,
+        classifier="naivebayes",
+        options=[],
+        quiet=True,
+    ):
+        # Make sure we can find java & weka.
+        config_weka()
+
+        # Build an ARFF formatter.
+        formatter = ARFF_Formatter.from_train(featuresets)
+
+        temp_dir = tempfile.mkdtemp()
+        try:
+            # Write the training data file.
+            train_filename = os.path.join(temp_dir, "train.arff")
+            formatter.write(train_filename, featuresets)
+
+            if classifier in cls._CLASSIFIER_CLASS:
+                javaclass = cls._CLASSIFIER_CLASS[classifier]
+            elif classifier in cls._CLASSIFIER_CLASS.values():
+                javaclass = classifier
+            else:
+                raise ValueError("Unknown classifier %s" % classifier)
+
+            # Train the weka model.
+            cmd = [javaclass, "-d", model_filename, "-t", train_filename]
+            cmd += list(options)
+            if quiet:
+                stdout = subprocess.PIPE
+            else:
+                stdout = None
+            java(cmd, classpath=_weka_classpath, stdout=stdout)
+
+            # Return the new classifier.
+            return WekaClassifier(formatter, model_filename)
+
+        finally:
+            for f in os.listdir(temp_dir):
+                os.remove(os.path.join(temp_dir, f))
+            os.rmdir(temp_dir)
+
+
+class ARFF_Formatter:
+    """
+    Converts featuresets and labeled featuresets to ARFF-formatted
+    strings, appropriate for input into Weka.
+
+    Features and classes can be specified manually in the constructor, or may
+    be determined from data using ``from_train``.
+    """
+
+    def __init__(self, labels, features):
+        """
+        :param labels: A list of all class labels that can be generated.
+        :param features: A list of feature specifications, where
+            each feature specification is a tuple (fname, ftype);
+            and ftype is an ARFF type string such as NUMERIC or
+            STRING.
+        """
+        self._labels = labels
+        self._features = features
+
+    def format(self, tokens):
+        """Returns a string representation of ARFF output for the given data."""
+        return self.header_section() + self.data_section(tokens)
+
+    def labels(self):
+        """Returns the list of classes."""
+        return list(self._labels)
+
+    def write(self, outfile, tokens):
+        """Writes ARFF data to a file for the given data."""
+        if not hasattr(outfile, "write"):
+            outfile = open(outfile, "w")
+        outfile.write(self.format(tokens))
+        outfile.close()
+
+    @staticmethod
+    def from_train(tokens):
+        """
+        Constructs an ARFF_Formatter instance with class labels and feature
+        types determined from the given data. Handles boolean, numeric and
+        string (note: not nominal) types.
+        """
+        # Find the set of all attested labels.
+        labels = {label for (tok, label) in tokens}
+
+        # Determine the types of all features.
+        features = {}
+        for tok, label in tokens:
+            for (fname, fval) in tok.items():
+                if issubclass(type(fval), bool):
+                    ftype = "{True, False}"
+                elif issubclass(type(fval), (int, float, bool)):
+                    ftype = "NUMERIC"
+                elif issubclass(type(fval), str):
+                    ftype = "STRING"
+                elif fval is None:
+                    continue  # can't tell the type.
+                else:
+                    raise ValueError("Unsupported value type %r" % ftype)
+
+                if features.get(fname, ftype) != ftype:
+                    raise ValueError("Inconsistent type for %s" % fname)
+                features[fname] = ftype
+        features = sorted(features.items())
+
+        return ARFF_Formatter(labels, features)
+
+    def header_section(self):
+        """Returns an ARFF header as a string."""
+        # Header comment.
+        s = (
+            "% Weka ARFF file\n"
+            + "% Generated automatically by NLTK\n"
+            + "%% %s\n\n" % time.ctime()
+        )
+
+        # Relation name
+        s += "@RELATION rel\n\n"
+
+        # Input attribute specifications
+        for fname, ftype in self._features:
+            s += "@ATTRIBUTE %-30r %s\n" % (fname, ftype)
+
+        # Label attribute specification
+        s += "@ATTRIBUTE %-30r {%s}\n" % ("-label-", ",".join(self._labels))
+
+        return s
+
+    def data_section(self, tokens, labeled=None):
+        """
+        Returns the ARFF data section for the given data.
+
+        :param tokens: a list of featuresets (dicts) or labelled featuresets
+            which are tuples (featureset, label).
+        :param labeled: Indicates whether the given tokens are labeled
+            or not.  If None, then the tokens will be assumed to be
+            labeled if the first token's value is a tuple or list.
+        """
+        # Check if the tokens are labeled or unlabeled.  If unlabeled,
+        # then use 'None'
+        if labeled is None:
+            labeled = tokens and isinstance(tokens[0], (tuple, list))
+        if not labeled:
+            tokens = [(tok, None) for tok in tokens]
+
+        # Data section
+        s = "\n@DATA\n"
+        for (tok, label) in tokens:
+            for fname, ftype in self._features:
+                s += "%s," % self._fmt_arff_val(tok.get(fname))
+            s += "%s\n" % self._fmt_arff_val(label)
+
+        return s
+
+    def _fmt_arff_val(self, fval):
+        if fval is None:
+            return "?"
+        elif isinstance(fval, (bool, int)):
+            return "%s" % fval
+        elif isinstance(fval, float):
+            return "%r" % fval
+        else:
+            return "%r" % fval
+
+
+if __name__ == "__main__":
+    from nltk.classify.util import binary_names_demo_features, names_demo
+
+    def make_classifier(featuresets):
+        return WekaClassifier.train("/tmp/name.model", featuresets, "C4.5")
+
+    classifier = names_demo(make_classifier, binary_names_demo_features)

.eggs/nltk-3.8-py3.10.egg/nltk/cluster/__init__.py

@@ -0,0 +1,92 @@
+# Natural Language Toolkit: Clusterers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This module contains a number of basic clustering algorithms. Clustering
+describes the task of discovering groups of similar items with a large
+collection. It is also describe as unsupervised machine learning, as the data
+from which it learns is unannotated with class information, as is the case for
+supervised learning.  Annotated data is difficult and expensive to obtain in
+the quantities required for the majority of supervised learning algorithms.
+This problem, the knowledge acquisition bottleneck, is common to most natural
+language processing tasks, thus fueling the need for quality unsupervised
+approaches.
+
+This module contains a k-means clusterer, E-M clusterer and a group average
+agglomerative clusterer (GAAC). All these clusterers involve finding good
+cluster groupings for a set of vectors in multi-dimensional space.
+
+The K-means clusterer starts with k arbitrary chosen means then allocates each
+vector to the cluster with the closest mean. It then recalculates the means of
+each cluster as the centroid of the vectors in the cluster. This process
+repeats until the cluster memberships stabilise. This is a hill-climbing
+algorithm which may converge to a local maximum. Hence the clustering is
+often repeated with random initial means and the most commonly occurring
+output means are chosen.
+
+The GAAC clusterer starts with each of the *N* vectors as singleton clusters.
+It then iteratively merges pairs of clusters which have the closest centroids.
+This continues until there is only one cluster. The order of merges gives rise
+to a dendrogram - a tree with the earlier merges lower than later merges. The
+membership of a given number of clusters *c*, *1 <= c <= N*, can be found by
+cutting the dendrogram at depth *c*.
+
+The Gaussian EM clusterer models the vectors as being produced by a mixture
+of k Gaussian sources. The parameters of these sources (prior probability,
+mean and covariance matrix) are then found to maximise the likelihood of the
+given data. This is done with the expectation maximisation algorithm. It
+starts with k arbitrarily chosen means, priors and covariance matrices. It
+then calculates the membership probabilities for each vector in each of the
+clusters - this is the 'E' step. The cluster parameters are then updated in
+the 'M' step using the maximum likelihood estimate from the cluster membership
+probabilities. This process continues until the likelihood of the data does
+not significantly increase.
+
+They all extend the ClusterI interface which defines common operations
+available with each clusterer. These operations include:
+
+- cluster: clusters a sequence of vectors
+- classify: assign a vector to a cluster
+- classification_probdist: give the probability distribution over cluster memberships
+
+The current existing classifiers also extend cluster.VectorSpace, an
+abstract class which allows for singular value decomposition (SVD) and vector
+normalisation. SVD is used to reduce the dimensionality of the vector space in
+such a manner as to preserve as much of the variation as possible, by
+reparameterising the axes in order of variability and discarding all bar the
+first d dimensions. Normalisation ensures that vectors fall in the unit
+hypersphere.
+
+Usage example (see also demo())::
+
+    from nltk import cluster
+    from nltk.cluster import euclidean_distance
+    from numpy import array
+
+    vectors = [array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0]]]
+
+    # initialise the clusterer (will also assign the vectors to clusters)
+    clusterer = cluster.KMeansClusterer(2, euclidean_distance)
+    clusterer.cluster(vectors, True)
+
+    # classify a new vector
+    print(clusterer.classify(array([3, 3])))
+
+Note that the vectors must use numpy array-like
+objects. nltk_contrib.unimelb.tacohn.SparseArrays may be used for
+efficiency when required.
+"""
+
+from nltk.cluster.em import EMClusterer
+from nltk.cluster.gaac import GAAClusterer
+from nltk.cluster.kmeans import KMeansClusterer
+from nltk.cluster.util import (
+    Dendrogram,
+    VectorSpaceClusterer,
+    cosine_distance,
+    euclidean_distance,
+)

.eggs/nltk-3.8-py3.10.egg/nltk/cluster/api.py

@@ -0,0 +1,74 @@
+# Natural Language Toolkit: Clusterer Interfaces
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# Porting: Steven Bird <stevenbird1@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from abc import ABCMeta, abstractmethod
+
+from nltk.probability import DictionaryProbDist
+
+
+class ClusterI(metaclass=ABCMeta):
+    """
+    Interface covering basic clustering functionality.
+    """
+
+    @abstractmethod
+    def cluster(self, vectors, assign_clusters=False):
+        """
+        Assigns the vectors to clusters, learning the clustering parameters
+        from the data. Returns a cluster identifier for each vector.
+        """
+
+    @abstractmethod
+    def classify(self, token):
+        """
+        Classifies the token into a cluster, setting the token's CLUSTER
+        parameter to that cluster identifier.
+        """
+
+    def likelihood(self, vector, label):
+        """
+        Returns the likelihood (a float) of the token having the
+        corresponding cluster.
+        """
+        if self.classify(vector) == label:
+            return 1.0
+        else:
+            return 0.0
+
+    def classification_probdist(self, vector):
+        """
+        Classifies the token into a cluster, returning
+        a probability distribution over the cluster identifiers.
+        """
+        likelihoods = {}
+        sum = 0.0
+        for cluster in self.cluster_names():
+            likelihoods[cluster] = self.likelihood(vector, cluster)
+            sum += likelihoods[cluster]
+        for cluster in self.cluster_names():
+            likelihoods[cluster] /= sum
+        return DictionaryProbDist(likelihoods)
+
+    @abstractmethod
+    def num_clusters(self):
+        """
+        Returns the number of clusters.
+        """
+
+    def cluster_names(self):
+        """
+        Returns the names of the clusters.
+        :rtype: list
+        """
+        return list(range(self.num_clusters()))
+
+    def cluster_name(self, index):
+        """
+        Returns the names of the cluster at index.
+        """
+        return index

.eggs/nltk-3.8-py3.10.egg/nltk/cluster/em.py

@@ -0,0 +1,219 @@
+# Natural Language Toolkit: Expectation Maximization Clusterer
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+try:
+    import numpy
+except ImportError:
+    pass
+
+from nltk.cluster.util import VectorSpaceClusterer
+
+
+class EMClusterer(VectorSpaceClusterer):
+    """
+    The Gaussian EM clusterer models the vectors as being produced by
+    a mixture of k Gaussian sources. The parameters of these sources
+    (prior probability, mean and covariance matrix) are then found to
+    maximise the likelihood of the given data. This is done with the
+    expectation maximisation algorithm. It starts with k arbitrarily
+    chosen means, priors and covariance matrices. It then calculates
+    the membership probabilities for each vector in each of the
+    clusters; this is the 'E' step. The cluster parameters are then
+    updated in the 'M' step using the maximum likelihood estimate from
+    the cluster membership probabilities. This process continues until
+    the likelihood of the data does not significantly increase.
+    """
+
+    def __init__(
+        self,
+        initial_means,
+        priors=None,
+        covariance_matrices=None,
+        conv_threshold=1e-6,
+        bias=0.1,
+        normalise=False,
+        svd_dimensions=None,
+    ):
+        """
+        Creates an EM clusterer with the given starting parameters,
+        convergence threshold and vector mangling parameters.
+
+        :param  initial_means: the means of the gaussian cluster centers
+        :type   initial_means: [seq of] numpy array or seq of SparseArray
+        :param  priors: the prior probability for each cluster
+        :type   priors: numpy array or seq of float
+        :param  covariance_matrices: the covariance matrix for each cluster
+        :type   covariance_matrices: [seq of] numpy array
+        :param  conv_threshold: maximum change in likelihood before deemed
+                    convergent
+        :type   conv_threshold: int or float
+        :param  bias: variance bias used to ensure non-singular covariance
+                      matrices
+        :type   bias: float
+        :param  normalise:  should vectors be normalised to length 1
+        :type   normalise:  boolean
+        :param  svd_dimensions: number of dimensions to use in reducing vector
+                               dimensionsionality with SVD
+        :type   svd_dimensions: int
+        """
+        VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+        self._means = numpy.array(initial_means, numpy.float64)
+        self._num_clusters = len(initial_means)
+        self._conv_threshold = conv_threshold
+        self._covariance_matrices = covariance_matrices
+        self._priors = priors
+        self._bias = bias
+
+    def num_clusters(self):
+        return self._num_clusters
+
+    def cluster_vectorspace(self, vectors, trace=False):
+        assert len(vectors) > 0
+
+        # set the parameters to initial values
+        dimensions = len(vectors[0])
+        means = self._means
+        priors = self._priors
+        if not priors:
+            priors = self._priors = (
+                numpy.ones(self._num_clusters, numpy.float64) / self._num_clusters
+            )
+        covariances = self._covariance_matrices
+        if not covariances:
+            covariances = self._covariance_matrices = [
+                numpy.identity(dimensions, numpy.float64)
+                for i in range(self._num_clusters)
+            ]
+
+        # do the E and M steps until the likelihood plateaus
+        lastl = self._loglikelihood(vectors, priors, means, covariances)
+        converged = False
+
+        while not converged:
+            if trace:
+                print("iteration; loglikelihood", lastl)
+            # E-step, calculate hidden variables, h[i,j]
+            h = numpy.zeros((len(vectors), self._num_clusters), numpy.float64)
+            for i in range(len(vectors)):
+                for j in range(self._num_clusters):
+                    h[i, j] = priors[j] * self._gaussian(
+                        means[j], covariances[j], vectors[i]
+                    )
+                h[i, :] /= sum(h[i, :])
+
+            # M-step, update parameters - cvm, p, mean
+            for j in range(self._num_clusters):
+                covariance_before = covariances[j]
+                new_covariance = numpy.zeros((dimensions, dimensions), numpy.float64)
+                new_mean = numpy.zeros(dimensions, numpy.float64)
+                sum_hj = 0.0
+                for i in range(len(vectors)):
+                    delta = vectors[i] - means[j]
+                    new_covariance += h[i, j] * numpy.multiply.outer(delta, delta)
+                    sum_hj += h[i, j]
+                    new_mean += h[i, j] * vectors[i]
+                covariances[j] = new_covariance / sum_hj
+                means[j] = new_mean / sum_hj
+                priors[j] = sum_hj / len(vectors)
+
+                # bias term to stop covariance matrix being singular
+                covariances[j] += self._bias * numpy.identity(dimensions, numpy.float64)
+
+            # calculate likelihood - FIXME: may be broken
+            l = self._loglikelihood(vectors, priors, means, covariances)
+
+            # check for convergence
+            if abs(lastl - l) < self._conv_threshold:
+                converged = True
+            lastl = l
+
+    def classify_vectorspace(self, vector):
+        best = None
+        for j in range(self._num_clusters):
+            p = self._priors[j] * self._gaussian(
+                self._means[j], self._covariance_matrices[j], vector
+            )
+            if not best or p > best[0]:
+                best = (p, j)
+        return best[1]
+
+    def likelihood_vectorspace(self, vector, cluster):
+        cid = self.cluster_names().index(cluster)
+        return self._priors[cluster] * self._gaussian(
+            self._means[cluster], self._covariance_matrices[cluster], vector
+        )
+
+    def _gaussian(self, mean, cvm, x):
+        m = len(mean)
+        assert cvm.shape == (m, m), "bad sized covariance matrix, %s" % str(cvm.shape)
+        try:
+            det = numpy.linalg.det(cvm)
+            inv = numpy.linalg.inv(cvm)
+            a = det**-0.5 * (2 * numpy.pi) ** (-m / 2.0)
+            dx = x - mean
+            print(dx, inv)
+            b = -0.5 * numpy.dot(numpy.dot(dx, inv), dx)
+            return a * numpy.exp(b)
+        except OverflowError:
+            # happens when the exponent is negative infinity - i.e. b = 0
+            # i.e. the inverse of cvm is huge (cvm is almost zero)
+            return 0
+
+    def _loglikelihood(self, vectors, priors, means, covariances):
+        llh = 0.0
+        for vector in vectors:
+            p = 0
+            for j in range(len(priors)):
+                p += priors[j] * self._gaussian(means[j], covariances[j], vector)
+            llh += numpy.log(p)
+        return llh
+
+    def __repr__(self):
+        return "<EMClusterer means=%s>" % list(self._means)
+
+
+def demo():
+    """
+    Non-interactive demonstration of the clusterers with simple 2-D data.
+    """
+
+    from nltk import cluster
+
+    # example from figure 14.10, page 519, Manning and Schutze
+
+    vectors = [numpy.array(f) for f in [[0.5, 0.5], [1.5, 0.5], [1, 3]]]
+    means = [[4, 2], [4, 2.01]]
+
+    clusterer = cluster.EMClusterer(means, bias=0.1)
+    clusters = clusterer.cluster(vectors, True, trace=True)
+
+    print("Clustered:", vectors)
+    print("As:       ", clusters)
+    print()
+
+    for c in range(2):
+        print("Cluster:", c)
+        print("Prior:  ", clusterer._priors[c])
+        print("Mean:   ", clusterer._means[c])
+        print("Covar:  ", clusterer._covariance_matrices[c])
+        print()
+
+    # classify a new vector
+    vector = numpy.array([2, 2])
+    print("classify(%s):" % vector, end=" ")
+    print(clusterer.classify(vector))
+
+    # show the classification probabilities
+    vector = numpy.array([2, 2])
+    print("classification_probdist(%s):" % vector)
+    pdist = clusterer.classification_probdist(vector)
+    for sample in pdist.samples():
+        print(f"{sample} => {pdist.prob(sample) * 100:.0f}%")
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/cluster/gaac.py

@@ -0,0 +1,170 @@
+# Natural Language Toolkit: Group Average Agglomerative Clusterer
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+try:
+    import numpy
+except ImportError:
+    pass
+
+from nltk.cluster.util import Dendrogram, VectorSpaceClusterer, cosine_distance
+
+
+class GAAClusterer(VectorSpaceClusterer):
+    """
+    The Group Average Agglomerative starts with each of the N vectors as singleton
+    clusters. It then iteratively merges pairs of clusters which have the
+    closest centroids.  This continues until there is only one cluster. The
+    order of merges gives rise to a dendrogram: a tree with the earlier merges
+    lower than later merges. The membership of a given number of clusters c, 1
+    <= c <= N, can be found by cutting the dendrogram at depth c.
+
+    This clusterer uses the cosine similarity metric only, which allows for
+    efficient speed-up in the clustering process.
+    """
+
+    def __init__(self, num_clusters=1, normalise=True, svd_dimensions=None):
+        VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+        self._num_clusters = num_clusters
+        self._dendrogram = None
+        self._groups_values = None
+
+    def cluster(self, vectors, assign_clusters=False, trace=False):
+        # stores the merge order
+        self._dendrogram = Dendrogram(
+            [numpy.array(vector, numpy.float64) for vector in vectors]
+        )
+        return VectorSpaceClusterer.cluster(self, vectors, assign_clusters, trace)
+
+    def cluster_vectorspace(self, vectors, trace=False):
+        # variables describing the initial situation
+        N = len(vectors)
+        cluster_len = [1] * N
+        cluster_count = N
+        index_map = numpy.arange(N)
+
+        # construct the similarity matrix
+        dims = (N, N)
+        dist = numpy.ones(dims, dtype=float) * numpy.inf
+        for i in range(N):
+            for j in range(i + 1, N):
+                dist[i, j] = cosine_distance(vectors[i], vectors[j])
+
+        while cluster_count > max(self._num_clusters, 1):
+            i, j = numpy.unravel_index(dist.argmin(), dims)
+            if trace:
+                print("merging %d and %d" % (i, j))
+
+            # update similarities for merging i and j
+            self._merge_similarities(dist, cluster_len, i, j)
+
+            # remove j
+            dist[:, j] = numpy.inf
+            dist[j, :] = numpy.inf
+
+            # merge the clusters
+            cluster_len[i] = cluster_len[i] + cluster_len[j]
+            self._dendrogram.merge(index_map[i], index_map[j])
+            cluster_count -= 1
+
+            # update the index map to reflect the indexes if we
+            # had removed j
+            index_map[j + 1 :] -= 1
+            index_map[j] = N
+
+        self.update_clusters(self._num_clusters)
+
+    def _merge_similarities(self, dist, cluster_len, i, j):
+        # the new cluster i merged from i and j adopts the average of
+        # i and j's similarity to each other cluster, weighted by the
+        # number of points in the clusters i and j
+        i_weight = cluster_len[i]
+        j_weight = cluster_len[j]
+        weight_sum = i_weight + j_weight
+
+        # update for x<i
+        dist[:i, i] = dist[:i, i] * i_weight + dist[:i, j] * j_weight
+        dist[:i, i] /= weight_sum
+        # update for i<x<j
+        dist[i, i + 1 : j] = (
+            dist[i, i + 1 : j] * i_weight + dist[i + 1 : j, j] * j_weight
+        )
+        # update for i<j<x
+        dist[i, j + 1 :] = dist[i, j + 1 :] * i_weight + dist[j, j + 1 :] * j_weight
+        dist[i, i + 1 :] /= weight_sum
+
+    def update_clusters(self, num_clusters):
+        clusters = self._dendrogram.groups(num_clusters)
+        self._centroids = []
+        for cluster in clusters:
+            assert len(cluster) > 0
+            if self._should_normalise:
+                centroid = self._normalise(cluster[0])
+            else:
+                centroid = numpy.array(cluster[0])
+            for vector in cluster[1:]:
+                if self._should_normalise:
+                    centroid += self._normalise(vector)
+                else:
+                    centroid += vector
+            centroid /= len(cluster)
+            self._centroids.append(centroid)
+        self._num_clusters = len(self._centroids)
+
+    def classify_vectorspace(self, vector):
+        best = None
+        for i in range(self._num_clusters):
+            centroid = self._centroids[i]
+            dist = cosine_distance(vector, centroid)
+            if not best or dist < best[0]:
+                best = (dist, i)
+        return best[1]
+
+    def dendrogram(self):
+        """
+        :return: The dendrogram representing the current clustering
+        :rtype:  Dendrogram
+        """
+        return self._dendrogram
+
+    def num_clusters(self):
+        return self._num_clusters
+
+    def __repr__(self):
+        return "<GroupAverageAgglomerative Clusterer n=%d>" % self._num_clusters
+
+
+def demo():
+    """
+    Non-interactive demonstration of the clusterers with simple 2-D data.
+    """
+
+    from nltk.cluster import GAAClusterer
+
+    # use a set of tokens with 2D indices
+    vectors = [numpy.array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0], [2, 3], [3, 1]]]
+
+    # test the GAAC clusterer with 4 clusters
+    clusterer = GAAClusterer(4)
+    clusters = clusterer.cluster(vectors, True)
+
+    print("Clusterer:", clusterer)
+    print("Clustered:", vectors)
+    print("As:", clusters)
+    print()
+
+    # show the dendrogram
+    clusterer.dendrogram().show()
+
+    # classify a new vector
+    vector = numpy.array([3, 3])
+    print("classify(%s):" % vector, end=" ")
+    print(clusterer.classify(vector))
+    print()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/cluster/kmeans.py

@@ -0,0 +1,231 @@
+# Natural Language Toolkit: K-Means Clusterer
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Trevor Cohn <tacohn@cs.mu.oz.au>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import copy
+import random
+import sys
+
+try:
+    import numpy
+except ImportError:
+    pass
+
+
+from nltk.cluster.util import VectorSpaceClusterer
+
+
+class KMeansClusterer(VectorSpaceClusterer):
+    """
+    The K-means clusterer starts with k arbitrary chosen means then allocates
+    each vector to the cluster with the closest mean. It then recalculates the
+    means of each cluster as the centroid of the vectors in the cluster. This
+    process repeats until the cluster memberships stabilise. This is a
+    hill-climbing algorithm which may converge to a local maximum. Hence the
+    clustering is often repeated with random initial means and the most
+    commonly occurring output means are chosen.
+    """
+
+    def __init__(
+        self,
+        num_means,
+        distance,
+        repeats=1,
+        conv_test=1e-6,
+        initial_means=None,
+        normalise=False,
+        svd_dimensions=None,
+        rng=None,
+        avoid_empty_clusters=False,
+    ):
+
+        """
+        :param  num_means:  the number of means to use (may use fewer)
+        :type   num_means:  int
+        :param  distance:   measure of distance between two vectors
+        :type   distance:   function taking two vectors and returning a float
+        :param  repeats:    number of randomised clustering trials to use
+        :type   repeats:    int
+        :param  conv_test:  maximum variation in mean differences before
+                            deemed convergent
+        :type   conv_test:  number
+        :param  initial_means: set of k initial means
+        :type   initial_means: sequence of vectors
+        :param  normalise:  should vectors be normalised to length 1
+        :type   normalise:  boolean
+        :param svd_dimensions: number of dimensions to use in reducing vector
+                               dimensionsionality with SVD
+        :type svd_dimensions: int
+        :param  rng:        random number generator (or None)
+        :type   rng:        Random
+        :param avoid_empty_clusters: include current centroid in computation
+                                     of next one; avoids undefined behavior
+                                     when clusters become empty
+        :type avoid_empty_clusters: boolean
+        """
+        VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+        self._num_means = num_means
+        self._distance = distance
+        self._max_difference = conv_test
+        assert not initial_means or len(initial_means) == num_means
+        self._means = initial_means
+        assert repeats >= 1
+        assert not (initial_means and repeats > 1)
+        self._repeats = repeats
+        self._rng = rng if rng else random.Random()
+        self._avoid_empty_clusters = avoid_empty_clusters
+
+    def cluster_vectorspace(self, vectors, trace=False):
+        if self._means and self._repeats > 1:
+            print("Warning: means will be discarded for subsequent trials")
+
+        meanss = []
+        for trial in range(self._repeats):
+            if trace:
+                print("k-means trial", trial)
+            if not self._means or trial > 1:
+                self._means = self._rng.sample(list(vectors), self._num_means)
+            self._cluster_vectorspace(vectors, trace)
+            meanss.append(self._means)
+
+        if len(meanss) > 1:
+            # sort the means first (so that different cluster numbering won't
+            # effect the distance comparison)
+            for means in meanss:
+                means.sort(key=sum)
+
+            # find the set of means that's minimally different from the others
+            min_difference = min_means = None
+            for i in range(len(meanss)):
+                d = 0
+                for j in range(len(meanss)):
+                    if i != j:
+                        d += self._sum_distances(meanss[i], meanss[j])
+                if min_difference is None or d < min_difference:
+                    min_difference, min_means = d, meanss[i]
+
+            # use the best means
+            self._means = min_means
+
+    def _cluster_vectorspace(self, vectors, trace=False):
+        if self._num_means < len(vectors):
+            # perform k-means clustering
+            converged = False
+            while not converged:
+                # assign the tokens to clusters based on minimum distance to
+                # the cluster means
+                clusters = [[] for m in range(self._num_means)]
+                for vector in vectors:
+                    index = self.classify_vectorspace(vector)
+                    clusters[index].append(vector)
+
+                if trace:
+                    print("iteration")
+                # for i in range(self._num_means):
+                # print '  mean', i, 'allocated', len(clusters[i]), 'vectors'
+
+                # recalculate cluster means by computing the centroid of each cluster
+                new_means = list(map(self._centroid, clusters, self._means))
+
+                # measure the degree of change from the previous step for convergence
+                difference = self._sum_distances(self._means, new_means)
+                if difference < self._max_difference:
+                    converged = True
+
+                # remember the new means
+                self._means = new_means
+
+    def classify_vectorspace(self, vector):
+        # finds the closest cluster centroid
+        # returns that cluster's index
+        best_distance = best_index = None
+        for index in range(len(self._means)):
+            mean = self._means[index]
+            dist = self._distance(vector, mean)
+            if best_distance is None or dist < best_distance:
+                best_index, best_distance = index, dist
+        return best_index
+
+    def num_clusters(self):
+        if self._means:
+            return len(self._means)
+        else:
+            return self._num_means
+
+    def means(self):
+        """
+        The means used for clustering.
+        """
+        return self._means
+
+    def _sum_distances(self, vectors1, vectors2):
+        difference = 0.0
+        for u, v in zip(vectors1, vectors2):
+            difference += self._distance(u, v)
+        return difference
+
+    def _centroid(self, cluster, mean):
+        if self._avoid_empty_clusters:
+            centroid = copy.copy(mean)
+            for vector in cluster:
+                centroid += vector
+            return centroid / (1 + len(cluster))
+        else:
+            if not len(cluster):
+                sys.stderr.write("Error: no centroid defined for empty cluster.\n")
+                sys.stderr.write(
+                    "Try setting argument 'avoid_empty_clusters' to True\n"
+                )
+                assert False
+            centroid = copy.copy(cluster[0])
+            for vector in cluster[1:]:
+                centroid += vector
+            return centroid / len(cluster)
+
+    def __repr__(self):
+        return "<KMeansClusterer means=%s repeats=%d>" % (self._means, self._repeats)
+
+
+#################################################################################
+
+
+def demo():
+    # example from figure 14.9, page 517, Manning and Schutze
+
+    from nltk.cluster import KMeansClusterer, euclidean_distance
+
+    vectors = [numpy.array(f) for f in [[2, 1], [1, 3], [4, 7], [6, 7]]]
+    means = [[4, 3], [5, 5]]
+
+    clusterer = KMeansClusterer(2, euclidean_distance, initial_means=means)
+    clusters = clusterer.cluster(vectors, True, trace=True)
+
+    print("Clustered:", vectors)
+    print("As:", clusters)
+    print("Means:", clusterer.means())
+    print()
+
+    vectors = [numpy.array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0], [2, 3], [3, 1]]]
+
+    # test k-means using the euclidean distance metric, 2 means and repeat
+    # clustering 10 times with random seeds
+
+    clusterer = KMeansClusterer(2, euclidean_distance, repeats=10)
+    clusters = clusterer.cluster(vectors, True)
+    print("Clustered:", vectors)
+    print("As:", clusters)
+    print("Means:", clusterer.means())
+    print()
+
+    # classify a new vector
+    vector = numpy.array([3, 3])
+    print("classify(%s):" % vector, end=" ")
+    print(clusterer.classify(vector))
+    print()
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/__init__.py

@@ -0,0 +1,529 @@
+# Natural Language Toolkit: Corpus Readers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+# TODO this docstring isn't up-to-date!
+"""
+NLTK corpus readers.  The modules in this package provide functions
+that can be used to read corpus files in a variety of formats.  These
+functions can be used to read both the corpus files that are
+distributed in the NLTK corpus package, and corpus files that are part
+of external corpora.
+
+Available Corpora
+=================
+
+Please see https://www.nltk.org/nltk_data/ for a complete list.
+Install corpora using nltk.download().
+
+Corpus Reader Functions
+=======================
+Each corpus module defines one or more "corpus reader functions",
+which can be used to read documents from that corpus.  These functions
+take an argument, ``item``, which is used to indicate which document
+should be read from the corpus:
+
+- If ``item`` is one of the unique identifiers listed in the corpus
+  module's ``items`` variable, then the corresponding document will
+  be loaded from the NLTK corpus package.
+- If ``item`` is a filename, then that file will be read.
+
+Additionally, corpus reader functions can be given lists of item
+names; in which case, they will return a concatenation of the
+corresponding documents.
+
+Corpus reader functions are named based on the type of information
+they return.  Some common examples, and their return types, are:
+
+- words(): list of str
+- sents(): list of (list of str)
+- paras(): list of (list of (list of str))
+- tagged_words(): list of (str,str) tuple
+- tagged_sents(): list of (list of (str,str))
+- tagged_paras(): list of (list of (list of (str,str)))
+- chunked_sents(): list of (Tree w/ (str,str) leaves)
+- parsed_sents(): list of (Tree with str leaves)
+- parsed_paras(): list of (list of (Tree with str leaves))
+- xml(): A single xml ElementTree
+- raw(): unprocessed corpus contents
+
+For example, to read a list of the words in the Brown Corpus, use
+``nltk.corpus.brown.words()``:
+
+    >>> from nltk.corpus import brown
+    >>> print(", ".join(brown.words())) # doctest: +ELLIPSIS
+    The, Fulton, County, Grand, Jury, said, ...
+
+"""
+
+import re
+
+from nltk.corpus.reader import *
+from nltk.corpus.util import LazyCorpusLoader
+from nltk.tokenize import RegexpTokenizer
+
+abc: PlaintextCorpusReader = LazyCorpusLoader(
+    "abc",
+    PlaintextCorpusReader,
+    r"(?!\.).*\.txt",
+    encoding=[("science", "latin_1"), ("rural", "utf8")],
+)
+alpino: AlpinoCorpusReader = LazyCorpusLoader(
+    "alpino", AlpinoCorpusReader, tagset="alpino"
+)
+bcp47: BCP47CorpusReader = LazyCorpusLoader(
+    "bcp47", BCP47CorpusReader, r"(cldr|iana)/*"
+)
+brown: CategorizedTaggedCorpusReader = LazyCorpusLoader(
+    "brown",
+    CategorizedTaggedCorpusReader,
+    r"c[a-z]\d\d",
+    cat_file="cats.txt",
+    tagset="brown",
+    encoding="ascii",
+)
+cess_cat: BracketParseCorpusReader = LazyCorpusLoader(
+    "cess_cat",
+    BracketParseCorpusReader,
+    r"(?!\.).*\.tbf",
+    tagset="unknown",
+    encoding="ISO-8859-15",
+)
+cess_esp: BracketParseCorpusReader = LazyCorpusLoader(
+    "cess_esp",
+    BracketParseCorpusReader,
+    r"(?!\.).*\.tbf",
+    tagset="unknown",
+    encoding="ISO-8859-15",
+)
+cmudict: CMUDictCorpusReader = LazyCorpusLoader(
+    "cmudict", CMUDictCorpusReader, ["cmudict"]
+)
+comtrans: AlignedCorpusReader = LazyCorpusLoader(
+    "comtrans", AlignedCorpusReader, r"(?!\.).*\.txt"
+)
+comparative_sentences: ComparativeSentencesCorpusReader = LazyCorpusLoader(
+    "comparative_sentences",
+    ComparativeSentencesCorpusReader,
+    r"labeledSentences\.txt",
+    encoding="latin-1",
+)
+conll2000: ConllChunkCorpusReader = LazyCorpusLoader(
+    "conll2000",
+    ConllChunkCorpusReader,
+    ["train.txt", "test.txt"],
+    ("NP", "VP", "PP"),
+    tagset="wsj",
+    encoding="ascii",
+)
+conll2002: ConllChunkCorpusReader = LazyCorpusLoader(
+    "conll2002",
+    ConllChunkCorpusReader,
+    r".*\.(test|train).*",
+    ("LOC", "PER", "ORG", "MISC"),
+    encoding="utf-8",
+)
+conll2007: DependencyCorpusReader = LazyCorpusLoader(
+    "conll2007",
+    DependencyCorpusReader,
+    r".*\.(test|train).*",
+    encoding=[("eus", "ISO-8859-2"), ("esp", "utf8")],
+)
+crubadan: CrubadanCorpusReader = LazyCorpusLoader(
+    "crubadan", CrubadanCorpusReader, r".*\.txt"
+)
+dependency_treebank: DependencyCorpusReader = LazyCorpusLoader(
+    "dependency_treebank", DependencyCorpusReader, r".*\.dp", encoding="ascii"
+)
+extended_omw: CorpusReader = LazyCorpusLoader(
+    "extended_omw", CorpusReader, r".*/wn-[a-z\-]*\.tab", encoding="utf8"
+)
+floresta: BracketParseCorpusReader = LazyCorpusLoader(
+    "floresta",
+    BracketParseCorpusReader,
+    r"(?!\.).*\.ptb",
+    "#",
+    tagset="unknown",
+    encoding="ISO-8859-15",
+)
+framenet15: FramenetCorpusReader = LazyCorpusLoader(
+    "framenet_v15",
+    FramenetCorpusReader,
+    [
+        "frRelation.xml",
+        "frameIndex.xml",
+        "fulltextIndex.xml",
+        "luIndex.xml",
+        "semTypes.xml",
+    ],
+)
+framenet: FramenetCorpusReader = LazyCorpusLoader(
+    "framenet_v17",
+    FramenetCorpusReader,
+    [
+        "frRelation.xml",
+        "frameIndex.xml",
+        "fulltextIndex.xml",
+        "luIndex.xml",
+        "semTypes.xml",
+    ],
+)
+gazetteers: WordListCorpusReader = LazyCorpusLoader(
+    "gazetteers", WordListCorpusReader, r"(?!LICENSE|\.).*\.txt", encoding="ISO-8859-2"
+)
+genesis: PlaintextCorpusReader = LazyCorpusLoader(
+    "genesis",
+    PlaintextCorpusReader,
+    r"(?!\.).*\.txt",
+    encoding=[
+        ("finnish|french|german", "latin_1"),
+        ("swedish", "cp865"),
+        (".*", "utf_8"),
+    ],
+)
+gutenberg: PlaintextCorpusReader = LazyCorpusLoader(
+    "gutenberg", PlaintextCorpusReader, r"(?!\.).*\.txt", encoding="latin1"
+)
+ieer: IEERCorpusReader = LazyCorpusLoader("ieer", IEERCorpusReader, r"(?!README|\.).*")
+inaugural: PlaintextCorpusReader = LazyCorpusLoader(
+    "inaugural", PlaintextCorpusReader, r"(?!\.).*\.txt", encoding="latin1"
+)
+# [XX] This should probably just use TaggedCorpusReader:
+indian: IndianCorpusReader = LazyCorpusLoader(
+    "indian", IndianCorpusReader, r"(?!\.).*\.pos", tagset="unknown", encoding="utf8"
+)
+
+jeita: ChasenCorpusReader = LazyCorpusLoader(
+    "jeita", ChasenCorpusReader, r".*\.chasen", encoding="utf-8"
+)
+knbc: KNBCorpusReader = LazyCorpusLoader(
+    "knbc/corpus1", KNBCorpusReader, r".*/KN.*", encoding="euc-jp"
+)
+lin_thesaurus: LinThesaurusCorpusReader = LazyCorpusLoader(
+    "lin_thesaurus", LinThesaurusCorpusReader, r".*\.lsp"
+)
+mac_morpho: MacMorphoCorpusReader = LazyCorpusLoader(
+    "mac_morpho",
+    MacMorphoCorpusReader,
+    r"(?!\.).*\.txt",
+    tagset="unknown",
+    encoding="latin-1",
+)
+machado: PortugueseCategorizedPlaintextCorpusReader = LazyCorpusLoader(
+    "machado",
+    PortugueseCategorizedPlaintextCorpusReader,
+    r"(?!\.).*\.txt",
+    cat_pattern=r"([a-z]*)/.*",
+    encoding="latin-1",
+)
+masc_tagged: CategorizedTaggedCorpusReader = LazyCorpusLoader(
+    "masc_tagged",
+    CategorizedTaggedCorpusReader,
+    r"(spoken|written)/.*\.txt",
+    cat_file="categories.txt",
+    tagset="wsj",
+    encoding="utf-8",
+    sep="_",
+)
+movie_reviews: CategorizedPlaintextCorpusReader = LazyCorpusLoader(
+    "movie_reviews",
+    CategorizedPlaintextCorpusReader,
+    r"(?!\.).*\.txt",
+    cat_pattern=r"(neg|pos)/.*",
+    encoding="ascii",
+)
+multext_east: MTECorpusReader = LazyCorpusLoader(
+    "mte_teip5", MTECorpusReader, r"(oana).*\.xml", encoding="utf-8"
+)
+names: WordListCorpusReader = LazyCorpusLoader(
+    "names", WordListCorpusReader, r"(?!\.).*\.txt", encoding="ascii"
+)
+nps_chat: NPSChatCorpusReader = LazyCorpusLoader(
+    "nps_chat", NPSChatCorpusReader, r"(?!README|\.).*\.xml", tagset="wsj"
+)
+opinion_lexicon: OpinionLexiconCorpusReader = LazyCorpusLoader(
+    "opinion_lexicon",
+    OpinionLexiconCorpusReader,
+    r"(\w+)\-words\.txt",
+    encoding="ISO-8859-2",
+)
+ppattach: PPAttachmentCorpusReader = LazyCorpusLoader(
+    "ppattach", PPAttachmentCorpusReader, ["training", "test", "devset"]
+)
+product_reviews_1: ReviewsCorpusReader = LazyCorpusLoader(
+    "product_reviews_1", ReviewsCorpusReader, r"^(?!Readme).*\.txt", encoding="utf8"
+)
+product_reviews_2: ReviewsCorpusReader = LazyCorpusLoader(
+    "product_reviews_2", ReviewsCorpusReader, r"^(?!Readme).*\.txt", encoding="utf8"
+)
+pros_cons: ProsConsCorpusReader = LazyCorpusLoader(
+    "pros_cons",
+    ProsConsCorpusReader,
+    r"Integrated(Cons|Pros)\.txt",
+    cat_pattern=r"Integrated(Cons|Pros)\.txt",
+    encoding="ISO-8859-2",
+)
+ptb: CategorizedBracketParseCorpusReader = (
+    LazyCorpusLoader(  # Penn Treebank v3: WSJ and Brown portions
+        "ptb",
+        CategorizedBracketParseCorpusReader,
+        r"(WSJ/\d\d/WSJ_\d\d|BROWN/C[A-Z]/C[A-Z])\d\d.MRG",
+        cat_file="allcats.txt",
+        tagset="wsj",
+    )
+)
+qc: StringCategoryCorpusReader = LazyCorpusLoader(
+    "qc", StringCategoryCorpusReader, ["train.txt", "test.txt"], encoding="ISO-8859-2"
+)
+reuters: CategorizedPlaintextCorpusReader = LazyCorpusLoader(
+    "reuters",
+    CategorizedPlaintextCorpusReader,
+    "(training|test).*",
+    cat_file="cats.txt",
+    encoding="ISO-8859-2",
+)
+rte: RTECorpusReader = LazyCorpusLoader("rte", RTECorpusReader, r"(?!\.).*\.xml")
+senseval: SensevalCorpusReader = LazyCorpusLoader(
+    "senseval", SensevalCorpusReader, r"(?!\.).*\.pos"
+)
+sentence_polarity: CategorizedSentencesCorpusReader = LazyCorpusLoader(
+    "sentence_polarity",
+    CategorizedSentencesCorpusReader,
+    r"rt-polarity\.(neg|pos)",
+    cat_pattern=r"rt-polarity\.(neg|pos)",
+    encoding="utf-8",
+)
+sentiwordnet: SentiWordNetCorpusReader = LazyCorpusLoader(
+    "sentiwordnet", SentiWordNetCorpusReader, "SentiWordNet_3.0.0.txt", encoding="utf-8"
+)
+shakespeare: XMLCorpusReader = LazyCorpusLoader(
+    "shakespeare", XMLCorpusReader, r"(?!\.).*\.xml"
+)
+sinica_treebank: SinicaTreebankCorpusReader = LazyCorpusLoader(
+    "sinica_treebank",
+    SinicaTreebankCorpusReader,
+    ["parsed"],
+    tagset="unknown",
+    encoding="utf-8",
+)
+state_union: PlaintextCorpusReader = LazyCorpusLoader(
+    "state_union", PlaintextCorpusReader, r"(?!\.).*\.txt", encoding="ISO-8859-2"
+)
+stopwords: WordListCorpusReader = LazyCorpusLoader(
+    "stopwords", WordListCorpusReader, r"(?!README|\.).*", encoding="utf8"
+)
+subjectivity: CategorizedSentencesCorpusReader = LazyCorpusLoader(
+    "subjectivity",
+    CategorizedSentencesCorpusReader,
+    r"(quote.tok.gt9|plot.tok.gt9)\.5000",
+    cat_map={"quote.tok.gt9.5000": ["subj"], "plot.tok.gt9.5000": ["obj"]},
+    encoding="latin-1",
+)
+swadesh: SwadeshCorpusReader = LazyCorpusLoader(
+    "swadesh", SwadeshCorpusReader, r"(?!README|\.).*", encoding="utf8"
+)
+swadesh110: PanlexSwadeshCorpusReader = LazyCorpusLoader(
+    "panlex_swadesh", PanlexSwadeshCorpusReader, r"swadesh110/.*\.txt", encoding="utf8"
+)
+swadesh207: PanlexSwadeshCorpusReader = LazyCorpusLoader(
+    "panlex_swadesh", PanlexSwadeshCorpusReader, r"swadesh207/.*\.txt", encoding="utf8"
+)
+switchboard: SwitchboardCorpusReader = LazyCorpusLoader(
+    "switchboard", SwitchboardCorpusReader, tagset="wsj"
+)
+timit: TimitCorpusReader = LazyCorpusLoader("timit", TimitCorpusReader)
+timit_tagged: TimitTaggedCorpusReader = LazyCorpusLoader(
+    "timit", TimitTaggedCorpusReader, r".+\.tags", tagset="wsj", encoding="ascii"
+)
+toolbox: ToolboxCorpusReader = LazyCorpusLoader(
+    "toolbox", ToolboxCorpusReader, r"(?!.*(README|\.)).*\.(dic|txt)"
+)
+treebank: BracketParseCorpusReader = LazyCorpusLoader(
+    "treebank/combined",
+    BracketParseCorpusReader,
+    r"wsj_.*\.mrg",
+    tagset="wsj",
+    encoding="ascii",
+)
+treebank_chunk: ChunkedCorpusReader = LazyCorpusLoader(
+    "treebank/tagged",
+    ChunkedCorpusReader,
+    r"wsj_.*\.pos",
+    sent_tokenizer=RegexpTokenizer(r"(?<=/\.)\s*(?![^\[]*\])", gaps=True),
+    para_block_reader=tagged_treebank_para_block_reader,
+    tagset="wsj",
+    encoding="ascii",
+)
+treebank_raw: PlaintextCorpusReader = LazyCorpusLoader(
+    "treebank/raw", PlaintextCorpusReader, r"wsj_.*", encoding="ISO-8859-2"
+)
+twitter_samples: TwitterCorpusReader = LazyCorpusLoader(
+    "twitter_samples", TwitterCorpusReader, r".*\.json"
+)
+udhr: UdhrCorpusReader = LazyCorpusLoader("udhr", UdhrCorpusReader)
+udhr2: PlaintextCorpusReader = LazyCorpusLoader(
+    "udhr2", PlaintextCorpusReader, r".*\.txt", encoding="utf8"
+)
+universal_treebanks: ConllCorpusReader = LazyCorpusLoader(
+    "universal_treebanks_v20",
+    ConllCorpusReader,
+    r".*\.conll",
+    columntypes=(
+        "ignore",
+        "words",
+        "ignore",
+        "ignore",
+        "pos",
+        "ignore",
+        "ignore",
+        "ignore",
+        "ignore",
+        "ignore",
+    ),
+)
+verbnet: VerbnetCorpusReader = LazyCorpusLoader(
+    "verbnet", VerbnetCorpusReader, r"(?!\.).*\.xml"
+)
+webtext: PlaintextCorpusReader = LazyCorpusLoader(
+    "webtext", PlaintextCorpusReader, r"(?!README|\.).*\.txt", encoding="ISO-8859-2"
+)
+wordnet: WordNetCorpusReader = LazyCorpusLoader(
+    "wordnet",
+    WordNetCorpusReader,
+    LazyCorpusLoader("omw-1.4", CorpusReader, r".*/wn-data-.*\.tab", encoding="utf8"),
+)
+wordnet31: WordNetCorpusReader = LazyCorpusLoader(
+    "wordnet31",
+    WordNetCorpusReader,
+    LazyCorpusLoader("omw-1.4", CorpusReader, r".*/wn-data-.*\.tab", encoding="utf8"),
+)
+wordnet2021: WordNetCorpusReader = LazyCorpusLoader(
+    "wordnet2021",
+    WordNetCorpusReader,
+    LazyCorpusLoader("omw-1.4", CorpusReader, r".*/wn-data-.*\.tab", encoding="utf8"),
+)
+wordnet_ic: WordNetICCorpusReader = LazyCorpusLoader(
+    "wordnet_ic", WordNetICCorpusReader, r".*\.dat"
+)
+words: WordListCorpusReader = LazyCorpusLoader(
+    "words", WordListCorpusReader, r"(?!README|\.).*", encoding="ascii"
+)
+
+# defined after treebank
+propbank: PropbankCorpusReader = LazyCorpusLoader(
+    "propbank",
+    PropbankCorpusReader,
+    "prop.txt",
+    r"frames/.*\.xml",
+    "verbs.txt",
+    lambda filename: re.sub(r"^wsj/\d\d/", "", filename),
+    treebank,
+)  # Must be defined *after* treebank corpus.
+nombank: NombankCorpusReader = LazyCorpusLoader(
+    "nombank.1.0",
+    NombankCorpusReader,
+    "nombank.1.0",
+    r"frames/.*\.xml",
+    "nombank.1.0.words",
+    lambda filename: re.sub(r"^wsj/\d\d/", "", filename),
+    treebank,
+)  # Must be defined *after* treebank corpus.
+propbank_ptb: PropbankCorpusReader = LazyCorpusLoader(
+    "propbank",
+    PropbankCorpusReader,
+    "prop.txt",
+    r"frames/.*\.xml",
+    "verbs.txt",
+    lambda filename: filename.upper(),
+    ptb,
+)  # Must be defined *after* ptb corpus.
+nombank_ptb: NombankCorpusReader = LazyCorpusLoader(
+    "nombank.1.0",
+    NombankCorpusReader,
+    "nombank.1.0",
+    r"frames/.*\.xml",
+    "nombank.1.0.words",
+    lambda filename: filename.upper(),
+    ptb,
+)  # Must be defined *after* ptb corpus.
+semcor: SemcorCorpusReader = LazyCorpusLoader(
+    "semcor", SemcorCorpusReader, r"brown./tagfiles/br-.*\.xml", wordnet
+)  # Must be defined *after* wordnet corpus.
+
+nonbreaking_prefixes: NonbreakingPrefixesCorpusReader = LazyCorpusLoader(
+    "nonbreaking_prefixes",
+    NonbreakingPrefixesCorpusReader,
+    r"(?!README|\.).*",
+    encoding="utf8",
+)
+perluniprops: UnicharsCorpusReader = LazyCorpusLoader(
+    "perluniprops",
+    UnicharsCorpusReader,
+    r"(?!README|\.).*",
+    nltk_data_subdir="misc",
+    encoding="utf8",
+)
+
+# mwa_ppdb = LazyCorpusLoader(
+#     'mwa_ppdb', MWAPPDBCorpusReader, r'(?!README|\.).*', nltk_data_subdir='misc', encoding='utf8')
+
+# See https://github.com/nltk/nltk/issues/1579
+# and https://github.com/nltk/nltk/issues/1716
+#
+# pl196x = LazyCorpusLoader(
+#     'pl196x', Pl196xCorpusReader, r'[a-z]-.*\.xml',
+#     cat_file='cats.txt', textid_file='textids.txt', encoding='utf8')
+#
+# ipipan = LazyCorpusLoader(
+#     'ipipan', IPIPANCorpusReader, r'(?!\.).*morph\.xml')
+#
+# nkjp = LazyCorpusLoader(
+#     'nkjp', NKJPCorpusReader, r'', encoding='utf8')
+#
+# panlex_lite = LazyCorpusLoader(
+#    'panlex_lite', PanLexLiteCorpusReader)
+#
+# ycoe = LazyCorpusLoader(
+#     'ycoe', YCOECorpusReader)
+#
+# corpus not available with NLTK; these lines caused help(nltk.corpus) to break
+# hebrew_treebank = LazyCorpusLoader(
+#    'hebrew_treebank', BracketParseCorpusReader, r'.*\.txt')
+
+# FIXME:  override any imported demo from various corpora, see https://github.com/nltk/nltk/issues/2116
+def demo():
+    # This is out-of-date:
+    abc.demo()
+    brown.demo()
+    #    chat80.demo()
+    cmudict.demo()
+    conll2000.demo()
+    conll2002.demo()
+    genesis.demo()
+    gutenberg.demo()
+    ieer.demo()
+    inaugural.demo()
+    indian.demo()
+    names.demo()
+    ppattach.demo()
+    senseval.demo()
+    shakespeare.demo()
+    sinica_treebank.demo()
+    state_union.demo()
+    stopwords.demo()
+    timit.demo()
+    toolbox.demo()
+    treebank.demo()
+    udhr.demo()
+    webtext.demo()
+    words.demo()
+
+
+#    ycoe.demo()
+
+if __name__ == "__main__":
+    # demo()
+    pass

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/europarl_raw.py

@@ -0,0 +1,56 @@
+# Natural Language Toolkit: Europarl Corpus Readers
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author:  Nitin Madnani <nmadnani@umiacs.umd.edu>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+
+from nltk.corpus.reader import *
+from nltk.corpus.util import LazyCorpusLoader
+
+# Create a new corpus reader instance for each European language
+danish: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/danish", EuroparlCorpusReader, r"ep-.*\.da", encoding="utf-8"
+)
+
+dutch: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/dutch", EuroparlCorpusReader, r"ep-.*\.nl", encoding="utf-8"
+)
+
+english: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/english", EuroparlCorpusReader, r"ep-.*\.en", encoding="utf-8"
+)
+
+finnish: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/finnish", EuroparlCorpusReader, r"ep-.*\.fi", encoding="utf-8"
+)
+
+french: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/french", EuroparlCorpusReader, r"ep-.*\.fr", encoding="utf-8"
+)
+
+german: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/german", EuroparlCorpusReader, r"ep-.*\.de", encoding="utf-8"
+)
+
+greek: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/greek", EuroparlCorpusReader, r"ep-.*\.el", encoding="utf-8"
+)
+
+italian: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/italian", EuroparlCorpusReader, r"ep-.*\.it", encoding="utf-8"
+)
+
+portuguese: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/portuguese", EuroparlCorpusReader, r"ep-.*\.pt", encoding="utf-8"
+)
+
+spanish: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/spanish", EuroparlCorpusReader, r"ep-.*\.es", encoding="utf-8"
+)
+
+swedish: EuroparlCorpusReader = LazyCorpusLoader(
+    "europarl_raw/swedish", EuroparlCorpusReader, r"ep-.*\.sv", encoding="utf-8"
+)

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/childes.py

@@ -0,0 +1,630 @@
+# CHILDES XML Corpus Reader
+
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Tomonori Nagano <tnagano@gc.cuny.edu>
+#         Alexis Dimitriadis <A.Dimitriadis@uu.nl>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the XML version of the CHILDES corpus.
+"""
+
+__docformat__ = "epytext en"
+
+import re
+from collections import defaultdict
+
+from nltk.corpus.reader.util import concat
+from nltk.corpus.reader.xmldocs import ElementTree, XMLCorpusReader
+from nltk.util import LazyConcatenation, LazyMap, flatten
+
+# to resolve the namespace issue
+NS = "http://www.talkbank.org/ns/talkbank"
+
+
+class CHILDESCorpusReader(XMLCorpusReader):
+    """
+    Corpus reader for the XML version of the CHILDES corpus.
+    The CHILDES corpus is available at ``https://childes.talkbank.org/``. The XML
+    version of CHILDES is located at ``https://childes.talkbank.org/data-xml/``.
+    Copy the needed parts of the CHILDES XML corpus into the NLTK data directory
+    (``nltk_data/corpora/CHILDES/``).
+
+    For access to the file text use the usual nltk functions,
+    ``words()``, ``sents()``, ``tagged_words()`` and ``tagged_sents()``.
+    """
+
+    def __init__(self, root, fileids, lazy=True):
+        XMLCorpusReader.__init__(self, root, fileids)
+        self._lazy = lazy
+
+    def words(
+        self,
+        fileids=None,
+        speaker="ALL",
+        stem=False,
+        relation=False,
+        strip_space=True,
+        replace=False,
+    ):
+        """
+        :return: the given file(s) as a list of words
+        :rtype: list(str)
+
+        :param speaker: If specified, select specific speaker(s) defined
+            in the corpus. Default is 'ALL' (all participants). Common choices
+            are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+            researchers)
+        :param stem: If true, then use word stems instead of word strings.
+        :param relation: If true, then return tuples of (stem, index,
+            dependent_index)
+        :param strip_space: If true, then strip trailing spaces from word
+            tokens. Otherwise, leave the spaces on the tokens.
+        :param replace: If true, then use the replaced (intended) word instead
+            of the original word (e.g., 'wat' will be replaced with 'watch')
+        """
+        sent = None
+        pos = False
+        if not self._lazy:
+            return [
+                self._get_words(
+                    fileid, speaker, sent, stem, relation, pos, strip_space, replace
+                )
+                for fileid in self.abspaths(fileids)
+            ]
+
+        get_words = lambda fileid: self._get_words(
+            fileid, speaker, sent, stem, relation, pos, strip_space, replace
+        )
+        return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
+
+    def tagged_words(
+        self,
+        fileids=None,
+        speaker="ALL",
+        stem=False,
+        relation=False,
+        strip_space=True,
+        replace=False,
+    ):
+        """
+        :return: the given file(s) as a list of tagged
+            words and punctuation symbols, encoded as tuples
+            ``(word,tag)``.
+        :rtype: list(tuple(str,str))
+
+        :param speaker: If specified, select specific speaker(s) defined
+            in the corpus. Default is 'ALL' (all participants). Common choices
+            are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+            researchers)
+        :param stem: If true, then use word stems instead of word strings.
+        :param relation: If true, then return tuples of (stem, index,
+            dependent_index)
+        :param strip_space: If true, then strip trailing spaces from word
+            tokens. Otherwise, leave the spaces on the tokens.
+        :param replace: If true, then use the replaced (intended) word instead
+            of the original word (e.g., 'wat' will be replaced with 'watch')
+        """
+        sent = None
+        pos = True
+        if not self._lazy:
+            return [
+                self._get_words(
+                    fileid, speaker, sent, stem, relation, pos, strip_space, replace
+                )
+                for fileid in self.abspaths(fileids)
+            ]
+
+        get_words = lambda fileid: self._get_words(
+            fileid, speaker, sent, stem, relation, pos, strip_space, replace
+        )
+        return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
+
+    def sents(
+        self,
+        fileids=None,
+        speaker="ALL",
+        stem=False,
+        relation=None,
+        strip_space=True,
+        replace=False,
+    ):
+        """
+        :return: the given file(s) as a list of sentences or utterances, each
+            encoded as a list of word strings.
+        :rtype: list(list(str))
+
+        :param speaker: If specified, select specific speaker(s) defined
+            in the corpus. Default is 'ALL' (all participants). Common choices
+            are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+            researchers)
+        :param stem: If true, then use word stems instead of word strings.
+        :param relation: If true, then return tuples of ``(str,pos,relation_list)``.
+            If there is manually-annotated relation info, it will return
+            tuples of ``(str,pos,test_relation_list,str,pos,gold_relation_list)``
+        :param strip_space: If true, then strip trailing spaces from word
+            tokens. Otherwise, leave the spaces on the tokens.
+        :param replace: If true, then use the replaced (intended) word instead
+            of the original word (e.g., 'wat' will be replaced with 'watch')
+        """
+        sent = True
+        pos = False
+        if not self._lazy:
+            return [
+                self._get_words(
+                    fileid, speaker, sent, stem, relation, pos, strip_space, replace
+                )
+                for fileid in self.abspaths(fileids)
+            ]
+
+        get_words = lambda fileid: self._get_words(
+            fileid, speaker, sent, stem, relation, pos, strip_space, replace
+        )
+        return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
+
+    def tagged_sents(
+        self,
+        fileids=None,
+        speaker="ALL",
+        stem=False,
+        relation=None,
+        strip_space=True,
+        replace=False,
+    ):
+        """
+        :return: the given file(s) as a list of
+            sentences, each encoded as a list of ``(word,tag)`` tuples.
+        :rtype: list(list(tuple(str,str)))
+
+        :param speaker: If specified, select specific speaker(s) defined
+            in the corpus. Default is 'ALL' (all participants). Common choices
+            are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+            researchers)
+        :param stem: If true, then use word stems instead of word strings.
+        :param relation: If true, then return tuples of ``(str,pos,relation_list)``.
+            If there is manually-annotated relation info, it will return
+            tuples of ``(str,pos,test_relation_list,str,pos,gold_relation_list)``
+        :param strip_space: If true, then strip trailing spaces from word
+            tokens. Otherwise, leave the spaces on the tokens.
+        :param replace: If true, then use the replaced (intended) word instead
+            of the original word (e.g., 'wat' will be replaced with 'watch')
+        """
+        sent = True
+        pos = True
+        if not self._lazy:
+            return [
+                self._get_words(
+                    fileid, speaker, sent, stem, relation, pos, strip_space, replace
+                )
+                for fileid in self.abspaths(fileids)
+            ]
+
+        get_words = lambda fileid: self._get_words(
+            fileid, speaker, sent, stem, relation, pos, strip_space, replace
+        )
+        return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
+
+    def corpus(self, fileids=None):
+        """
+        :return: the given file(s) as a dict of ``(corpus_property_key, value)``
+        :rtype: list(dict)
+        """
+        if not self._lazy:
+            return [self._get_corpus(fileid) for fileid in self.abspaths(fileids)]
+        return LazyMap(self._get_corpus, self.abspaths(fileids))
+
+    def _get_corpus(self, fileid):
+        results = dict()
+        xmldoc = ElementTree.parse(fileid).getroot()
+        for key, value in xmldoc.items():
+            results[key] = value
+        return results
+
+    def participants(self, fileids=None):
+        """
+        :return: the given file(s) as a dict of
+            ``(participant_property_key, value)``
+        :rtype: list(dict)
+        """
+        if not self._lazy:
+            return [self._get_participants(fileid) for fileid in self.abspaths(fileids)]
+        return LazyMap(self._get_participants, self.abspaths(fileids))
+
+    def _get_participants(self, fileid):
+        # multidimensional dicts
+        def dictOfDicts():
+            return defaultdict(dictOfDicts)
+
+        xmldoc = ElementTree.parse(fileid).getroot()
+        # getting participants' data
+        pat = dictOfDicts()
+        for participant in xmldoc.findall(
+            f".//{{{NS}}}Participants/{{{NS}}}participant"
+        ):
+            for (key, value) in participant.items():
+                pat[participant.get("id")][key] = value
+        return pat
+
+    def age(self, fileids=None, speaker="CHI", month=False):
+        """
+        :return: the given file(s) as string or int
+        :rtype: list or int
+
+        :param month: If true, return months instead of year-month-date
+        """
+        if not self._lazy:
+            return [
+                self._get_age(fileid, speaker, month)
+                for fileid in self.abspaths(fileids)
+            ]
+        get_age = lambda fileid: self._get_age(fileid, speaker, month)
+        return LazyMap(get_age, self.abspaths(fileids))
+
+    def _get_age(self, fileid, speaker, month):
+        xmldoc = ElementTree.parse(fileid).getroot()
+        for pat in xmldoc.findall(f".//{{{NS}}}Participants/{{{NS}}}participant"):
+            try:
+                if pat.get("id") == speaker:
+                    age = pat.get("age")
+                    if month:
+                        age = self.convert_age(age)
+                    return age
+            # some files don't have age data
+            except (TypeError, AttributeError) as e:
+                return None
+
+    def convert_age(self, age_year):
+        "Caclculate age in months from a string in CHILDES format"
+        m = re.match(r"P(\d+)Y(\d+)M?(\d?\d?)D?", age_year)
+        age_month = int(m.group(1)) * 12 + int(m.group(2))
+        try:
+            if int(m.group(3)) > 15:
+                age_month += 1
+        # some corpora don't have age information?
+        except ValueError as e:
+            pass
+        return age_month
+
+    def MLU(self, fileids=None, speaker="CHI"):
+        """
+        :return: the given file(s) as a floating number
+        :rtype: list(float)
+        """
+        if not self._lazy:
+            return [
+                self._getMLU(fileid, speaker=speaker)
+                for fileid in self.abspaths(fileids)
+            ]
+        get_MLU = lambda fileid: self._getMLU(fileid, speaker=speaker)
+        return LazyMap(get_MLU, self.abspaths(fileids))
+
+    def _getMLU(self, fileid, speaker):
+        sents = self._get_words(
+            fileid,
+            speaker=speaker,
+            sent=True,
+            stem=True,
+            relation=False,
+            pos=True,
+            strip_space=True,
+            replace=True,
+        )
+        results = []
+        lastSent = []
+        numFillers = 0
+        sentDiscount = 0
+        for sent in sents:
+            posList = [pos for (word, pos) in sent]
+            # if any part of the sentence is intelligible
+            if any(pos == "unk" for pos in posList):
+                continue
+            # if the sentence is null
+            elif sent == []:
+                continue
+            # if the sentence is the same as the last sent
+            elif sent == lastSent:
+                continue
+            else:
+                results.append([word for (word, pos) in sent])
+                # count number of fillers
+                if len({"co", None}.intersection(posList)) > 0:
+                    numFillers += posList.count("co")
+                    numFillers += posList.count(None)
+                    sentDiscount += 1
+            lastSent = sent
+        try:
+            thisWordList = flatten(results)
+            # count number of morphemes
+            # (e.g., 'read' = 1 morpheme but 'read-PAST' is 2 morphemes)
+            numWords = (
+                len(flatten([word.split("-") for word in thisWordList])) - numFillers
+            )
+            numSents = len(results) - sentDiscount
+            mlu = numWords / numSents
+        except ZeroDivisionError:
+            mlu = 0
+        # return {'mlu':mlu,'wordNum':numWords,'sentNum':numSents}
+        return mlu
+
+    def _get_words(
+        self, fileid, speaker, sent, stem, relation, pos, strip_space, replace
+    ):
+        if (
+            isinstance(speaker, str) and speaker != "ALL"
+        ):  # ensure we have a list of speakers
+            speaker = [speaker]
+        xmldoc = ElementTree.parse(fileid).getroot()
+        # processing each xml doc
+        results = []
+        for xmlsent in xmldoc.findall(".//{%s}u" % NS):
+            sents = []
+            # select speakers
+            if speaker == "ALL" or xmlsent.get("who") in speaker:
+                for xmlword in xmlsent.findall(".//{%s}w" % NS):
+                    infl = None
+                    suffixStem = None
+                    suffixTag = None
+                    # getting replaced words
+                    if replace and xmlsent.find(f".//{{{NS}}}w/{{{NS}}}replacement"):
+                        xmlword = xmlsent.find(
+                            f".//{{{NS}}}w/{{{NS}}}replacement/{{{NS}}}w"
+                        )
+                    elif replace and xmlsent.find(f".//{{{NS}}}w/{{{NS}}}wk"):
+                        xmlword = xmlsent.find(f".//{{{NS}}}w/{{{NS}}}wk")
+                    # get text
+                    if xmlword.text:
+                        word = xmlword.text
+                    else:
+                        word = ""
+                    # strip tailing space
+                    if strip_space:
+                        word = word.strip()
+                    # stem
+                    if relation or stem:
+                        try:
+                            xmlstem = xmlword.find(".//{%s}stem" % NS)
+                            word = xmlstem.text
+                        except AttributeError as e:
+                            pass
+                        # if there is an inflection
+                        try:
+                            xmlinfl = xmlword.find(
+                                f".//{{{NS}}}mor/{{{NS}}}mw/{{{NS}}}mk"
+                            )
+                            word += "-" + xmlinfl.text
+                        except:
+                            pass
+                        # if there is a suffix
+                        try:
+                            xmlsuffix = xmlword.find(
+                                ".//{%s}mor/{%s}mor-post/{%s}mw/{%s}stem"
+                                % (NS, NS, NS, NS)
+                            )
+                            suffixStem = xmlsuffix.text
+                        except AttributeError:
+                            suffixStem = ""
+                        if suffixStem:
+                            word += "~" + suffixStem
+                    # pos
+                    if relation or pos:
+                        try:
+                            xmlpos = xmlword.findall(".//{%s}c" % NS)
+                            xmlpos2 = xmlword.findall(".//{%s}s" % NS)
+                            if xmlpos2 != []:
+                                tag = xmlpos[0].text + ":" + xmlpos2[0].text
+                            else:
+                                tag = xmlpos[0].text
+                        except (AttributeError, IndexError) as e:
+                            tag = ""
+                        try:
+                            xmlsuffixpos = xmlword.findall(
+                                ".//{%s}mor/{%s}mor-post/{%s}mw/{%s}pos/{%s}c"
+                                % (NS, NS, NS, NS, NS)
+                            )
+                            xmlsuffixpos2 = xmlword.findall(
+                                ".//{%s}mor/{%s}mor-post/{%s}mw/{%s}pos/{%s}s"
+                                % (NS, NS, NS, NS, NS)
+                            )
+                            if xmlsuffixpos2:
+                                suffixTag = (
+                                    xmlsuffixpos[0].text + ":" + xmlsuffixpos2[0].text
+                                )
+                            else:
+                                suffixTag = xmlsuffixpos[0].text
+                        except:
+                            pass
+                        if suffixTag:
+                            tag += "~" + suffixTag
+                        word = (word, tag)
+                    # relational
+                    # the gold standard is stored in
+                    # <mor></mor><mor type="trn"><gra type="grt">
+                    if relation == True:
+                        for xmlstem_rel in xmlword.findall(
+                            f".//{{{NS}}}mor/{{{NS}}}gra"
+                        ):
+                            if not xmlstem_rel.get("type") == "grt":
+                                word = (
+                                    word[0],
+                                    word[1],
+                                    xmlstem_rel.get("index")
+                                    + "|"
+                                    + xmlstem_rel.get("head")
+                                    + "|"
+                                    + xmlstem_rel.get("relation"),
+                                )
+                            else:
+                                word = (
+                                    word[0],
+                                    word[1],
+                                    word[2],
+                                    word[0],
+                                    word[1],
+                                    xmlstem_rel.get("index")
+                                    + "|"
+                                    + xmlstem_rel.get("head")
+                                    + "|"
+                                    + xmlstem_rel.get("relation"),
+                                )
+                        try:
+                            for xmlpost_rel in xmlword.findall(
+                                f".//{{{NS}}}mor/{{{NS}}}mor-post/{{{NS}}}gra"
+                            ):
+                                if not xmlpost_rel.get("type") == "grt":
+                                    suffixStem = (
+                                        suffixStem[0],
+                                        suffixStem[1],
+                                        xmlpost_rel.get("index")
+                                        + "|"
+                                        + xmlpost_rel.get("head")
+                                        + "|"
+                                        + xmlpost_rel.get("relation"),
+                                    )
+                                else:
+                                    suffixStem = (
+                                        suffixStem[0],
+                                        suffixStem[1],
+                                        suffixStem[2],
+                                        suffixStem[0],
+                                        suffixStem[1],
+                                        xmlpost_rel.get("index")
+                                        + "|"
+                                        + xmlpost_rel.get("head")
+                                        + "|"
+                                        + xmlpost_rel.get("relation"),
+                                    )
+                        except:
+                            pass
+                    sents.append(word)
+                if sent or relation:
+                    results.append(sents)
+                else:
+                    results.extend(sents)
+        return LazyMap(lambda x: x, results)
+
+    # Ready-to-use browser opener
+
+    """
+    The base URL for viewing files on the childes website. This
+    shouldn't need to be changed, unless CHILDES changes the configuration
+    of their server or unless the user sets up their own corpus webserver.
+    """
+    childes_url_base = r"https://childes.talkbank.org/browser/index.php?url="
+
+    def webview_file(self, fileid, urlbase=None):
+        """Map a corpus file to its web version on the CHILDES website,
+        and open it in a web browser.
+
+        The complete URL to be used is:
+            childes.childes_url_base + urlbase + fileid.replace('.xml', '.cha')
+
+        If no urlbase is passed, we try to calculate it.  This
+        requires that the childes corpus was set up to mirror the
+        folder hierarchy under childes.psy.cmu.edu/data-xml/, e.g.:
+        nltk_data/corpora/childes/Eng-USA/Cornell/??? or
+        nltk_data/corpora/childes/Romance/Spanish/Aguirre/???
+
+        The function first looks (as a special case) if "Eng-USA" is
+        on the path consisting of <corpus root>+fileid; then if
+        "childes", possibly followed by "data-xml", appears. If neither
+        one is found, we use the unmodified fileid and hope for the best.
+        If this is not right, specify urlbase explicitly, e.g., if the
+        corpus root points to the Cornell folder, urlbase='Eng-USA/Cornell'.
+        """
+
+        import webbrowser
+
+        if urlbase:
+            path = urlbase + "/" + fileid
+        else:
+            full = self.root + "/" + fileid
+            full = re.sub(r"\\", "/", full)
+            if "/childes/" in full.lower():
+                # Discard /data-xml/ if present
+                path = re.findall(r"(?i)/childes(?:/data-xml)?/(.*)\.xml", full)[0]
+            elif "eng-usa" in full.lower():
+                path = "Eng-USA/" + re.findall(r"/(?i)Eng-USA/(.*)\.xml", full)[0]
+            else:
+                path = fileid
+
+        # Strip ".xml" and add ".cha", as necessary:
+        if path.endswith(".xml"):
+            path = path[:-4]
+
+        if not path.endswith(".cha"):
+            path = path + ".cha"
+
+        url = self.childes_url_base + path
+
+        webbrowser.open_new_tab(url)
+        print("Opening in browser:", url)
+        # Pausing is a good idea, but it's up to the user...
+        # raw_input("Hit Return to continue")
+
+
+def demo(corpus_root=None):
+    """
+    The CHILDES corpus should be manually downloaded and saved
+    to ``[NLTK_Data_Dir]/corpora/childes/``
+    """
+    if not corpus_root:
+        from nltk.data import find
+
+        corpus_root = find("corpora/childes/data-xml/Eng-USA/")
+
+    try:
+        childes = CHILDESCorpusReader(corpus_root, ".*.xml")
+        # describe all corpus
+        for file in childes.fileids()[:5]:
+            corpus = ""
+            corpus_id = ""
+            for (key, value) in childes.corpus(file)[0].items():
+                if key == "Corpus":
+                    corpus = value
+                if key == "Id":
+                    corpus_id = value
+            print("Reading", corpus, corpus_id, " .....")
+            print("words:", childes.words(file)[:7], "...")
+            print(
+                "words with replaced words:",
+                childes.words(file, replace=True)[:7],
+                " ...",
+            )
+            print("words with pos tags:", childes.tagged_words(file)[:7], " ...")
+            print("words (only MOT):", childes.words(file, speaker="MOT")[:7], "...")
+            print("words (only CHI):", childes.words(file, speaker="CHI")[:7], "...")
+            print("stemmed words:", childes.words(file, stem=True)[:7], " ...")
+            print(
+                "words with relations and pos-tag:",
+                childes.words(file, relation=True)[:5],
+                " ...",
+            )
+            print("sentence:", childes.sents(file)[:2], " ...")
+            for (participant, values) in childes.participants(file)[0].items():
+                for (key, value) in values.items():
+                    print("\tparticipant", participant, key, ":", value)
+            print("num of sent:", len(childes.sents(file)))
+            print("num of morphemes:", len(childes.words(file, stem=True)))
+            print("age:", childes.age(file))
+            print("age in month:", childes.age(file, month=True))
+            print("MLU:", childes.MLU(file))
+            print()
+
+    except LookupError as e:
+        print(
+            """The CHILDES corpus, or the parts you need, should be manually
+        downloaded from https://childes.talkbank.org/data-xml/ and saved at
+        [NLTK_Data_Dir]/corpora/childes/
+            Alternately, you can call the demo with the path to a portion of the CHILDES corpus, e.g.:
+        demo('/path/to/childes/data-xml/Eng-USA/")
+        """
+        )
+        # corpus_root_http = urllib2.urlopen('https://childes.talkbank.org/data-xml/Eng-USA/Bates.zip')
+        # corpus_root_http_bates = zipfile.ZipFile(cStringIO.StringIO(corpus_root_http.read()))
+        ##this fails
+        # childes = CHILDESCorpusReader(corpus_root_http_bates,corpus_root_http_bates.namelist())
+
+
+if __name__ == "__main__":
+    demo()

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/chunked.py

@@ -0,0 +1,273 @@
+# Natural Language Toolkit: Chunked Corpus Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora that contain chunked (and optionally tagged)
+documents.
+"""
+
+import codecs
+import os.path
+
+import nltk
+from nltk.chunk import tagstr2tree
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
+from nltk.corpus.reader.util import *
+from nltk.tokenize import *
+from nltk.tree import Tree
+
+
+class ChunkedCorpusReader(CorpusReader):
+    """
+    Reader for chunked (and optionally tagged) corpora.  Paragraphs
+    are split using a block reader.  They are then tokenized into
+    sentences using a sentence tokenizer.  Finally, these sentences
+    are parsed into chunk trees using a string-to-chunktree conversion
+    function.  Each of these steps can be performed using a default
+    function or a custom function.  By default, paragraphs are split
+    on blank lines; sentences are listed one per line; and sentences
+    are parsed into chunk trees using ``nltk.chunk.tagstr2tree``.
+    """
+
+    def __init__(
+        self,
+        root,
+        fileids,
+        extension="",
+        str2chunktree=tagstr2tree,
+        sent_tokenizer=RegexpTokenizer("\n", gaps=True),
+        para_block_reader=read_blankline_block,
+        encoding="utf8",
+        tagset=None,
+    ):
+        """
+        :param root: The root directory for this corpus.
+        :param fileids: A list or regexp specifying the fileids in this corpus.
+        """
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._cv_args = (str2chunktree, sent_tokenizer, para_block_reader, tagset)
+        """Arguments for corpus views generated by this corpus: a tuple
+        (str2chunktree, sent_tokenizer, para_block_tokenizer)"""
+
+    def words(self, fileids=None):
+        """
+        :return: the given file(s) as a list of words
+            and punctuation symbols.
+        :rtype: list(str)
+        """
+        return concat(
+            [
+                ChunkedCorpusView(f, enc, 0, 0, 0, 0, *self._cv_args)
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def sents(self, fileids=None):
+        """
+        :return: the given file(s) as a list of
+            sentences or utterances, each encoded as a list of word
+            strings.
+        :rtype: list(list(str))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(f, enc, 0, 1, 0, 0, *self._cv_args)
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def paras(self, fileids=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as lists of word strings.
+        :rtype: list(list(list(str)))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(f, enc, 0, 1, 1, 0, *self._cv_args)
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def tagged_words(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of tagged
+            words and punctuation symbols, encoded as tuples
+            ``(word,tag)``.
+        :rtype: list(tuple(str,str))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 0, 0, 0, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def tagged_sents(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            sentences, each encoded as a list of ``(word,tag)`` tuples.
+
+        :rtype: list(list(tuple(str,str)))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 1, 0, 0, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def tagged_paras(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as lists of ``(word,tag)`` tuples.
+        :rtype: list(list(list(tuple(str,str))))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 1, 1, 0, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def chunked_words(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of tagged
+            words and chunks.  Words are encoded as ``(word, tag)``
+            tuples (if the corpus has tags) or word strings (if the
+            corpus has no tags).  Chunks are encoded as depth-one
+            trees over ``(word,tag)`` tuples or word strings.
+        :rtype: list(tuple(str,str) and Tree)
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 0, 0, 1, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def chunked_sents(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            sentences, each encoded as a shallow Tree.  The leaves
+            of these trees are encoded as ``(word, tag)`` tuples (if
+            the corpus has tags) or word strings (if the corpus has no
+            tags).
+        :rtype: list(Tree)
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 1, 0, 1, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def chunked_paras(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as a shallow Tree.  The leaves of these
+            trees are encoded as ``(word, tag)`` tuples (if the corpus
+            has tags) or word strings (if the corpus has no tags).
+        :rtype: list(list(Tree))
+        """
+        return concat(
+            [
+                ChunkedCorpusView(
+                    f, enc, 1, 1, 1, 1, *self._cv_args, target_tagset=tagset
+                )
+                for (f, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def _read_block(self, stream):
+        return [tagstr2tree(t) for t in read_blankline_block(stream)]
+
+
+class ChunkedCorpusView(StreamBackedCorpusView):
+    def __init__(
+        self,
+        fileid,
+        encoding,
+        tagged,
+        group_by_sent,
+        group_by_para,
+        chunked,
+        str2chunktree,
+        sent_tokenizer,
+        para_block_reader,
+        source_tagset=None,
+        target_tagset=None,
+    ):
+        StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
+        self._tagged = tagged
+        self._group_by_sent = group_by_sent
+        self._group_by_para = group_by_para
+        self._chunked = chunked
+        self._str2chunktree = str2chunktree
+        self._sent_tokenizer = sent_tokenizer
+        self._para_block_reader = para_block_reader
+        self._source_tagset = source_tagset
+        self._target_tagset = target_tagset
+
+    def read_block(self, stream):
+        block = []
+        for para_str in self._para_block_reader(stream):
+            para = []
+            for sent_str in self._sent_tokenizer.tokenize(para_str):
+                sent = self._str2chunktree(
+                    sent_str,
+                    source_tagset=self._source_tagset,
+                    target_tagset=self._target_tagset,
+                )
+
+                # If requested, throw away the tags.
+                if not self._tagged:
+                    sent = self._untag(sent)
+
+                # If requested, throw away the chunks.
+                if not self._chunked:
+                    sent = sent.leaves()
+
+                # Add the sentence to `para`.
+                if self._group_by_sent:
+                    para.append(sent)
+                else:
+                    para.extend(sent)
+
+            # Add the paragraph to `block`.
+            if self._group_by_para:
+                block.append(para)
+            else:
+                block.extend(para)
+
+        # Return the block
+        return block
+
+    def _untag(self, tree):
+        for i, child in enumerate(tree):
+            if isinstance(child, Tree):
+                self._untag(child)
+            elif isinstance(child, tuple):
+                tree[i] = child[0]
+            else:
+                raise ValueError("expected child to be Tree or tuple")
+        return tree

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/cmudict.py

@@ -0,0 +1,88 @@
+# Natural Language Toolkit: Carnegie Mellon Pronouncing Dictionary Corpus Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The Carnegie Mellon Pronouncing Dictionary [cmudict.0.6]
+ftp://ftp.cs.cmu.edu/project/speech/dict/
+Copyright 1998 Carnegie Mellon University
+
+File Format: Each line consists of an uppercased word, a counter
+(for alternative pronunciations), and a transcription.  Vowels are
+marked for stress (1=primary, 2=secondary, 0=no stress).  E.g.:
+NATURAL 1 N AE1 CH ER0 AH0 L
+
+The dictionary contains 127069 entries.  Of these, 119400 words are assigned
+a unique pronunciation, 6830 words have two pronunciations, and 839 words have
+three or more pronunciations.  Many of these are fast-speech variants.
+
+Phonemes: There are 39 phonemes, as shown below:
+
+Phoneme Example Translation    Phoneme Example Translation
+------- ------- -----------    ------- ------- -----------
+AA      odd     AA D           AE      at      AE T
+AH      hut     HH AH T        AO      ought   AO T
+AW      cow     K AW           AY      hide    HH AY D
+B       be      B IY           CH      cheese  CH IY Z
+D       dee     D IY           DH      thee    DH IY
+EH      Ed      EH D           ER      hurt    HH ER T
+EY      ate     EY T           F       fee     F IY
+G       green   G R IY N       HH      he      HH IY
+IH      it      IH T           IY      eat     IY T
+JH      gee     JH IY          K       key     K IY
+L       lee     L IY           M       me      M IY
+N       knee    N IY           NG      ping    P IH NG
+OW      oat     OW T           OY      toy     T OY
+P       pee     P IY           R       read    R IY D
+S       sea     S IY           SH      she     SH IY
+T       tea     T IY           TH      theta   TH EY T AH
+UH      hood    HH UH D        UW      two     T UW
+V       vee     V IY           W       we      W IY
+Y       yield   Y IY L D       Z       zee     Z IY
+ZH      seizure S IY ZH ER
+"""
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.util import *
+from nltk.util import Index
+
+
+class CMUDictCorpusReader(CorpusReader):
+    def entries(self):
+        """
+        :return: the cmudict lexicon as a list of entries
+            containing (word, transcriptions) tuples.
+        """
+        return concat(
+            [
+                StreamBackedCorpusView(fileid, read_cmudict_block, encoding=enc)
+                for fileid, enc in self.abspaths(None, True)
+            ]
+        )
+
+    def words(self):
+        """
+        :return: a list of all words defined in the cmudict lexicon.
+        """
+        return [word.lower() for (word, _) in self.entries()]
+
+    def dict(self):
+        """
+        :return: the cmudict lexicon as a dictionary, whose keys are
+            lowercase words and whose values are lists of pronunciations.
+        """
+        return dict(Index(self.entries()))
+
+
+def read_cmudict_block(stream):
+    entries = []
+    while len(entries) < 100:  # Read 100 at a time.
+        line = stream.readline()
+        if line == "":
+            return entries  # end of file.
+        pieces = line.split()
+        entries.append((pieces[0].lower(), pieces[2:]))
+    return entries

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/comparative_sents.py

@@ -0,0 +1,309 @@
+# Natural Language Toolkit: Comparative Sentence Corpus Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Pierpaolo Pantone <24alsecondo@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+CorpusReader for the Comparative Sentence Dataset.
+
+- Comparative Sentence Dataset information -
+
+Annotated by: Nitin Jindal and Bing Liu, 2006.
+              Department of Computer Sicence
+              University of Illinois at Chicago
+
+Contact: Nitin Jindal, njindal@cs.uic.edu
+         Bing Liu, liub@cs.uic.edu (https://www.cs.uic.edu/~liub)
+
+Distributed with permission.
+
+Related papers:
+
+- Nitin Jindal and Bing Liu. "Identifying Comparative Sentences in Text Documents".
+   Proceedings of the ACM SIGIR International Conference on Information Retrieval
+   (SIGIR-06), 2006.
+
+- Nitin Jindal and Bing Liu. "Mining Comprative Sentences and Relations".
+   Proceedings of Twenty First National Conference on Artificial Intelligence
+   (AAAI-2006), 2006.
+
+- Murthy Ganapathibhotla and Bing Liu. "Mining Opinions in Comparative Sentences".
+    Proceedings of the 22nd International Conference on Computational Linguistics
+    (Coling-2008), Manchester, 18-22 August, 2008.
+"""
+import re
+
+from nltk.corpus.reader.api import *
+from nltk.tokenize import *
+
+# Regular expressions for dataset components
+STARS = re.compile(r"^\*+$")
+COMPARISON = re.compile(r"<cs-[1234]>")
+CLOSE_COMPARISON = re.compile(r"</cs-[1234]>")
+GRAD_COMPARISON = re.compile(r"<cs-[123]>")
+NON_GRAD_COMPARISON = re.compile(r"<cs-4>")
+ENTITIES_FEATS = re.compile(r"(\d)_((?:[\.\w\s/-](?!\d_))+)")
+KEYWORD = re.compile(r"\(([^\(]*)\)$")
+
+
+class Comparison:
+    """
+    A Comparison represents a comparative sentence and its constituents.
+    """
+
+    def __init__(
+        self,
+        text=None,
+        comp_type=None,
+        entity_1=None,
+        entity_2=None,
+        feature=None,
+        keyword=None,
+    ):
+        """
+        :param text: a string (optionally tokenized) containing a comparison.
+        :param comp_type: an integer defining the type of comparison expressed.
+            Values can be: 1 (Non-equal gradable), 2 (Equative), 3 (Superlative),
+            4 (Non-gradable).
+        :param entity_1: the first entity considered in the comparison relation.
+        :param entity_2: the second entity considered in the comparison relation.
+        :param feature: the feature considered in the comparison relation.
+        :param keyword: the word or phrase which is used for that comparative relation.
+        """
+        self.text = text
+        self.comp_type = comp_type
+        self.entity_1 = entity_1
+        self.entity_2 = entity_2
+        self.feature = feature
+        self.keyword = keyword
+
+    def __repr__(self):
+        return (
+            'Comparison(text="{}", comp_type={}, entity_1="{}", entity_2="{}", '
+            'feature="{}", keyword="{}")'
+        ).format(
+            self.text,
+            self.comp_type,
+            self.entity_1,
+            self.entity_2,
+            self.feature,
+            self.keyword,
+        )
+
+
+class ComparativeSentencesCorpusReader(CorpusReader):
+    """
+    Reader for the Comparative Sentence Dataset by Jindal and Liu (2006).
+
+        >>> from nltk.corpus import comparative_sentences
+        >>> comparison = comparative_sentences.comparisons()[0]
+        >>> comparison.text # doctest: +NORMALIZE_WHITESPACE
+        ['its', 'fast-forward', 'and', 'rewind', 'work', 'much', 'more', 'smoothly',
+        'and', 'consistently', 'than', 'those', 'of', 'other', 'models', 'i', "'ve",
+        'had', '.']
+        >>> comparison.entity_2
+        'models'
+        >>> (comparison.feature, comparison.keyword)
+        ('rewind', 'more')
+        >>> len(comparative_sentences.comparisons())
+        853
+    """
+
+    CorpusView = StreamBackedCorpusView
+
+    def __init__(
+        self,
+        root,
+        fileids,
+        word_tokenizer=WhitespaceTokenizer(),
+        sent_tokenizer=None,
+        encoding="utf8",
+    ):
+        """
+        :param root: The root directory for this corpus.
+        :param fileids: a list or regexp specifying the fileids in this corpus.
+        :param word_tokenizer: tokenizer for breaking sentences or paragraphs
+            into words. Default: `WhitespaceTokenizer`
+        :param sent_tokenizer: tokenizer for breaking paragraphs into sentences.
+        :param encoding: the encoding that should be used to read the corpus.
+        """
+
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._word_tokenizer = word_tokenizer
+        self._sent_tokenizer = sent_tokenizer
+        self._readme = "README.txt"
+
+    def comparisons(self, fileids=None):
+        """
+        Return all comparisons in the corpus.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            comparisons have to be returned.
+        :return: the given file(s) as a list of Comparison objects.
+        :rtype: list(Comparison)
+        """
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, str):
+            fileids = [fileids]
+        return concat(
+            [
+                self.CorpusView(path, self._read_comparison_block, encoding=enc)
+                for (path, enc, fileid) in self.abspaths(fileids, True, True)
+            ]
+        )
+
+    def keywords(self, fileids=None):
+        """
+        Return a set of all keywords used in the corpus.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            keywords have to be returned.
+        :return: the set of keywords and comparative phrases used in the corpus.
+        :rtype: set(str)
+        """
+        all_keywords = concat(
+            [
+                self.CorpusView(path, self._read_keyword_block, encoding=enc)
+                for (path, enc, fileid) in self.abspaths(fileids, True, True)
+            ]
+        )
+
+        keywords_set = {keyword.lower() for keyword in all_keywords if keyword}
+        return keywords_set
+
+    def keywords_readme(self):
+        """
+        Return the list of words and constituents considered as clues of a
+        comparison (from listOfkeywords.txt).
+        """
+        keywords = []
+        with self.open("listOfkeywords.txt") as fp:
+            raw_text = fp.read()
+        for line in raw_text.split("\n"):
+            if not line or line.startswith("//"):
+                continue
+            keywords.append(line.strip())
+        return keywords
+
+    def sents(self, fileids=None):
+        """
+        Return all sentences in the corpus.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            sentences have to be returned.
+        :return: all sentences of the corpus as lists of tokens (or as plain
+            strings, if no word tokenizer is specified).
+        :rtype: list(list(str)) or list(str)
+        """
+        return concat(
+            [
+                self.CorpusView(path, self._read_sent_block, encoding=enc)
+                for (path, enc, fileid) in self.abspaths(fileids, True, True)
+            ]
+        )
+
+    def words(self, fileids=None):
+        """
+        Return all words and punctuation symbols in the corpus.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            words have to be returned.
+        :return: the given file(s) as a list of words and punctuation symbols.
+        :rtype: list(str)
+        """
+        return concat(
+            [
+                self.CorpusView(path, self._read_word_block, encoding=enc)
+                for (path, enc, fileid) in self.abspaths(fileids, True, True)
+            ]
+        )
+
+    def _read_comparison_block(self, stream):
+        while True:
+            line = stream.readline()
+            if not line:
+                return []  # end of file.
+            comparison_tags = re.findall(COMPARISON, line)
+            if comparison_tags:
+                grad_comparisons = re.findall(GRAD_COMPARISON, line)
+                non_grad_comparisons = re.findall(NON_GRAD_COMPARISON, line)
+                # Advance to the next line (it contains the comparative sentence)
+                comparison_text = stream.readline().strip()
+                if self._word_tokenizer:
+                    comparison_text = self._word_tokenizer.tokenize(comparison_text)
+                # Skip the next line (it contains closing comparison tags)
+                stream.readline()
+                # If gradable comparisons are found, create Comparison instances
+                # and populate their fields
+                comparison_bundle = []
+                if grad_comparisons:
+                    # Each comparison tag has its own relations on a separate line
+                    for comp in grad_comparisons:
+                        comp_type = int(re.match(r"<cs-(\d)>", comp).group(1))
+                        comparison = Comparison(
+                            text=comparison_text, comp_type=comp_type
+                        )
+                        line = stream.readline()
+                        entities_feats = ENTITIES_FEATS.findall(line)
+                        if entities_feats:
+                            for (code, entity_feat) in entities_feats:
+                                if code == "1":
+                                    comparison.entity_1 = entity_feat.strip()
+                                elif code == "2":
+                                    comparison.entity_2 = entity_feat.strip()
+                                elif code == "3":
+                                    comparison.feature = entity_feat.strip()
+                        keyword = KEYWORD.findall(line)
+                        if keyword:
+                            comparison.keyword = keyword[0]
+                        comparison_bundle.append(comparison)
+                # If non-gradable comparisons are found, create a simple Comparison
+                # instance for each one
+                if non_grad_comparisons:
+                    for comp in non_grad_comparisons:
+                        # comp_type in this case should always be 4.
+                        comp_type = int(re.match(r"<cs-(\d)>", comp).group(1))
+                        comparison = Comparison(
+                            text=comparison_text, comp_type=comp_type
+                        )
+                        comparison_bundle.append(comparison)
+                # Flatten the list of comparisons before returning them
+                # return concat([comparison_bundle])
+                return comparison_bundle
+
+    def _read_keyword_block(self, stream):
+        keywords = []
+        for comparison in self._read_comparison_block(stream):
+            keywords.append(comparison.keyword)
+        return keywords
+
+    def _read_sent_block(self, stream):
+        while True:
+            line = stream.readline()
+            if re.match(STARS, line):
+                while True:
+                    line = stream.readline()
+                    if re.match(STARS, line):
+                        break
+                continue
+            if (
+                not re.findall(COMPARISON, line)
+                and not ENTITIES_FEATS.findall(line)
+                and not re.findall(CLOSE_COMPARISON, line)
+            ):
+                if self._sent_tokenizer:
+                    return [
+                        self._word_tokenizer.tokenize(sent)
+                        for sent in self._sent_tokenizer.tokenize(line)
+                    ]
+                else:
+                    return [self._word_tokenizer.tokenize(line)]
+
+    def _read_word_block(self, stream):
+        words = []
+        for sent in self._read_sent_block(stream):
+            words.extend(sent)
+        return words

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/conll.py

@@ -0,0 +1,579 @@
+# Natural Language Toolkit: CONLL Corpus Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read CoNLL-style chunk fileids.
+"""
+
+import textwrap
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.util import *
+from nltk.tag import map_tag
+from nltk.tree import Tree
+from nltk.util import LazyConcatenation, LazyMap
+
+
+class ConllCorpusReader(CorpusReader):
+    """
+    A corpus reader for CoNLL-style files.  These files consist of a
+    series of sentences, separated by blank lines.  Each sentence is
+    encoded using a table (or "grid") of values, where each line
+    corresponds to a single word, and each column corresponds to an
+    annotation type.  The set of columns used by CoNLL-style files can
+    vary from corpus to corpus; the ``ConllCorpusReader`` constructor
+    therefore takes an argument, ``columntypes``, which is used to
+    specify the columns that are used by a given corpus. By default
+    columns are split by consecutive whitespaces, with the
+    ``separator`` argument you can set a string to split by (e.g.
+    ``\'\t\'``).
+
+
+    @todo: Add support for reading from corpora where different
+        parallel files contain different columns.
+    @todo: Possibly add caching of the grid corpus view?  This would
+        allow the same grid view to be used by different data access
+        methods (eg words() and parsed_sents() could both share the
+        same grid corpus view object).
+    @todo: Better support for -DOCSTART-.  Currently, we just ignore
+        it, but it could be used to define methods that retrieve a
+        document at a time (eg parsed_documents()).
+    """
+
+    # /////////////////////////////////////////////////////////////////
+    # Column Types
+    # /////////////////////////////////////////////////////////////////
+
+    WORDS = "words"  #: column type for words
+    POS = "pos"  #: column type for part-of-speech tags
+    TREE = "tree"  #: column type for parse trees
+    CHUNK = "chunk"  #: column type for chunk structures
+    NE = "ne"  #: column type for named entities
+    SRL = "srl"  #: column type for semantic role labels
+    IGNORE = "ignore"  #: column type for column that should be ignored
+
+    #: A list of all column types supported by the conll corpus reader.
+    COLUMN_TYPES = (WORDS, POS, TREE, CHUNK, NE, SRL, IGNORE)
+
+    # /////////////////////////////////////////////////////////////////
+    # Constructor
+    # /////////////////////////////////////////////////////////////////
+
+    def __init__(
+        self,
+        root,
+        fileids,
+        columntypes,
+        chunk_types=None,
+        root_label="S",
+        pos_in_tree=False,
+        srl_includes_roleset=True,
+        encoding="utf8",
+        tree_class=Tree,
+        tagset=None,
+        separator=None,
+    ):
+        for columntype in columntypes:
+            if columntype not in self.COLUMN_TYPES:
+                raise ValueError("Bad column type %r" % columntype)
+        if isinstance(chunk_types, str):
+            chunk_types = [chunk_types]
+        self._chunk_types = chunk_types
+        self._colmap = {c: i for (i, c) in enumerate(columntypes)}
+        self._pos_in_tree = pos_in_tree
+        self._root_label = root_label  # for chunks
+        self._srl_includes_roleset = srl_includes_roleset
+        self._tree_class = tree_class
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._tagset = tagset
+        self.sep = separator
+
+    # /////////////////////////////////////////////////////////////////
+    # Data Access Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def words(self, fileids=None):
+        self._require(self.WORDS)
+        return LazyConcatenation(LazyMap(self._get_words, self._grids(fileids)))
+
+    def sents(self, fileids=None):
+        self._require(self.WORDS)
+        return LazyMap(self._get_words, self._grids(fileids))
+
+    def tagged_words(self, fileids=None, tagset=None):
+        self._require(self.WORDS, self.POS)
+
+        def get_tagged_words(grid):
+            return self._get_tagged_words(grid, tagset)
+
+        return LazyConcatenation(LazyMap(get_tagged_words, self._grids(fileids)))
+
+    def tagged_sents(self, fileids=None, tagset=None):
+        self._require(self.WORDS, self.POS)
+
+        def get_tagged_words(grid):
+            return self._get_tagged_words(grid, tagset)
+
+        return LazyMap(get_tagged_words, self._grids(fileids))
+
+    def chunked_words(self, fileids=None, chunk_types=None, tagset=None):
+        self._require(self.WORDS, self.POS, self.CHUNK)
+        if chunk_types is None:
+            chunk_types = self._chunk_types
+
+        def get_chunked_words(grid):  # capture chunk_types as local var
+            return self._get_chunked_words(grid, chunk_types, tagset)
+
+        return LazyConcatenation(LazyMap(get_chunked_words, self._grids(fileids)))
+
+    def chunked_sents(self, fileids=None, chunk_types=None, tagset=None):
+        self._require(self.WORDS, self.POS, self.CHUNK)
+        if chunk_types is None:
+            chunk_types = self._chunk_types
+
+        def get_chunked_words(grid):  # capture chunk_types as local var
+            return self._get_chunked_words(grid, chunk_types, tagset)
+
+        return LazyMap(get_chunked_words, self._grids(fileids))
+
+    def parsed_sents(self, fileids=None, pos_in_tree=None, tagset=None):
+        self._require(self.WORDS, self.POS, self.TREE)
+        if pos_in_tree is None:
+            pos_in_tree = self._pos_in_tree
+
+        def get_parsed_sent(grid):  # capture pos_in_tree as local var
+            return self._get_parsed_sent(grid, pos_in_tree, tagset)
+
+        return LazyMap(get_parsed_sent, self._grids(fileids))
+
+    def srl_spans(self, fileids=None):
+        self._require(self.SRL)
+        return LazyMap(self._get_srl_spans, self._grids(fileids))
+
+    def srl_instances(self, fileids=None, pos_in_tree=None, flatten=True):
+        self._require(self.WORDS, self.POS, self.TREE, self.SRL)
+        if pos_in_tree is None:
+            pos_in_tree = self._pos_in_tree
+
+        def get_srl_instances(grid):  # capture pos_in_tree as local var
+            return self._get_srl_instances(grid, pos_in_tree)
+
+        result = LazyMap(get_srl_instances, self._grids(fileids))
+        if flatten:
+            result = LazyConcatenation(result)
+        return result
+
+    def iob_words(self, fileids=None, tagset=None):
+        """
+        :return: a list of word/tag/IOB tuples
+        :rtype: list(tuple)
+        :param fileids: the list of fileids that make up this corpus
+        :type fileids: None or str or list
+        """
+        self._require(self.WORDS, self.POS, self.CHUNK)
+
+        def get_iob_words(grid):
+            return self._get_iob_words(grid, tagset)
+
+        return LazyConcatenation(LazyMap(get_iob_words, self._grids(fileids)))
+
+    def iob_sents(self, fileids=None, tagset=None):
+        """
+        :return: a list of lists of word/tag/IOB tuples
+        :rtype: list(list)
+        :param fileids: the list of fileids that make up this corpus
+        :type fileids: None or str or list
+        """
+        self._require(self.WORDS, self.POS, self.CHUNK)
+
+        def get_iob_words(grid):
+            return self._get_iob_words(grid, tagset)
+
+        return LazyMap(get_iob_words, self._grids(fileids))
+
+    # /////////////////////////////////////////////////////////////////
+    # Grid Reading
+    # /////////////////////////////////////////////////////////////////
+
+    def _grids(self, fileids=None):
+        # n.b.: we could cache the object returned here (keyed on
+        # fileids), which would let us reuse the same corpus view for
+        # different things (eg srl and parse trees).
+        return concat(
+            [
+                StreamBackedCorpusView(fileid, self._read_grid_block, encoding=enc)
+                for (fileid, enc) in self.abspaths(fileids, True)
+            ]
+        )
+
+    def _read_grid_block(self, stream):
+        grids = []
+        for block in read_blankline_block(stream):
+            block = block.strip()
+            if not block:
+                continue
+
+            grid = [line.split(self.sep) for line in block.split("\n")]
+
+            # If there's a docstart row, then discard. ([xx] eventually it
+            # would be good to actually use it)
+            if grid[0][self._colmap.get("words", 0)] == "-DOCSTART-":
+                del grid[0]
+
+            # Check that the grid is consistent.
+            for row in grid:
+                if len(row) != len(grid[0]):
+                    raise ValueError("Inconsistent number of columns:\n%s" % block)
+            grids.append(grid)
+        return grids
+
+    # /////////////////////////////////////////////////////////////////
+    # Transforms
+    # /////////////////////////////////////////////////////////////////
+    # given a grid, transform it into some representation (e.g.,
+    # a list of words or a parse tree).
+
+    def _get_words(self, grid):
+        return self._get_column(grid, self._colmap["words"])
+
+    def _get_tagged_words(self, grid, tagset=None):
+        pos_tags = self._get_column(grid, self._colmap["pos"])
+        if tagset and tagset != self._tagset:
+            pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+        return list(zip(self._get_column(grid, self._colmap["words"]), pos_tags))
+
+    def _get_iob_words(self, grid, tagset=None):
+        pos_tags = self._get_column(grid, self._colmap["pos"])
+        if tagset and tagset != self._tagset:
+            pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+        return list(
+            zip(
+                self._get_column(grid, self._colmap["words"]),
+                pos_tags,
+                self._get_column(grid, self._colmap["chunk"]),
+            )
+        )
+
+    def _get_chunked_words(self, grid, chunk_types, tagset=None):
+        # n.b.: this method is very similar to conllstr2tree.
+        words = self._get_column(grid, self._colmap["words"])
+        pos_tags = self._get_column(grid, self._colmap["pos"])
+        if tagset and tagset != self._tagset:
+            pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+        chunk_tags = self._get_column(grid, self._colmap["chunk"])
+
+        stack = [Tree(self._root_label, [])]
+
+        for (word, pos_tag, chunk_tag) in zip(words, pos_tags, chunk_tags):
+            if chunk_tag == "O":
+                state, chunk_type = "O", ""
+            else:
+                (state, chunk_type) = chunk_tag.split("-")
+            # If it's a chunk we don't care about, treat it as O.
+            if chunk_types is not None and chunk_type not in chunk_types:
+                state = "O"
+            # Treat a mismatching I like a B.
+            if state == "I" and chunk_type != stack[-1].label():
+                state = "B"
+            # For B or I: close any open chunks
+            if state in "BO" and len(stack) == 2:
+                stack.pop()
+            # For B: start a new chunk.
+            if state == "B":
+                new_chunk = Tree(chunk_type, [])
+                stack[-1].append(new_chunk)
+                stack.append(new_chunk)
+            # Add the word token.
+            stack[-1].append((word, pos_tag))
+
+        return stack[0]
+
+    def _get_parsed_sent(self, grid, pos_in_tree, tagset=None):
+        words = self._get_column(grid, self._colmap["words"])
+        pos_tags = self._get_column(grid, self._colmap["pos"])
+        if tagset and tagset != self._tagset:
+            pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+        parse_tags = self._get_column(grid, self._colmap["tree"])
+
+        treestr = ""
+        for (word, pos_tag, parse_tag) in zip(words, pos_tags, parse_tags):
+            if word == "(":
+                word = "-LRB-"
+            if word == ")":
+                word = "-RRB-"
+            if pos_tag == "(":
+                pos_tag = "-LRB-"
+            if pos_tag == ")":
+                pos_tag = "-RRB-"
+            (left, right) = parse_tag.split("*")
+            right = right.count(")") * ")"  # only keep ')'.
+            treestr += f"{left} ({pos_tag} {word}) {right}"
+        try:
+            tree = self._tree_class.fromstring(treestr)
+        except (ValueError, IndexError):
+            tree = self._tree_class.fromstring(f"({self._root_label} {treestr})")
+
+        if not pos_in_tree:
+            for subtree in tree.subtrees():
+                for i, child in enumerate(subtree):
+                    if (
+                        isinstance(child, Tree)
+                        and len(child) == 1
+                        and isinstance(child[0], str)
+                    ):
+                        subtree[i] = (child[0], child.label())
+
+        return tree
+
+    def _get_srl_spans(self, grid):
+        """
+        list of list of (start, end), tag) tuples
+        """
+        if self._srl_includes_roleset:
+            predicates = self._get_column(grid, self._colmap["srl"] + 1)
+            start_col = self._colmap["srl"] + 2
+        else:
+            predicates = self._get_column(grid, self._colmap["srl"])
+            start_col = self._colmap["srl"] + 1
+
+        # Count how many predicates there are.  This tells us how many
+        # columns to expect for SRL data.
+        num_preds = len([p for p in predicates if p != "-"])
+
+        spanlists = []
+        for i in range(num_preds):
+            col = self._get_column(grid, start_col + i)
+            spanlist = []
+            stack = []
+            for wordnum, srl_tag in enumerate(col):
+                (left, right) = srl_tag.split("*")
+                for tag in left.split("("):
+                    if tag:
+                        stack.append((tag, wordnum))
+                for i in range(right.count(")")):
+                    (tag, start) = stack.pop()
+                    spanlist.append(((start, wordnum + 1), tag))
+            spanlists.append(spanlist)
+
+        return spanlists
+
+    def _get_srl_instances(self, grid, pos_in_tree):
+        tree = self._get_parsed_sent(grid, pos_in_tree)
+        spanlists = self._get_srl_spans(grid)
+        if self._srl_includes_roleset:
+            predicates = self._get_column(grid, self._colmap["srl"] + 1)
+            rolesets = self._get_column(grid, self._colmap["srl"])
+        else:
+            predicates = self._get_column(grid, self._colmap["srl"])
+            rolesets = [None] * len(predicates)
+
+        instances = ConllSRLInstanceList(tree)
+        for wordnum, predicate in enumerate(predicates):
+            if predicate == "-":
+                continue
+            # Decide which spanlist to use.  Don't assume that they're
+            # sorted in the same order as the predicates (even though
+            # they usually are).
+            for spanlist in spanlists:
+                for (start, end), tag in spanlist:
+                    if wordnum in range(start, end) and tag in ("V", "C-V"):
+                        break
+                else:
+                    continue
+                break
+            else:
+                raise ValueError("No srl column found for %r" % predicate)
+            instances.append(
+                ConllSRLInstance(tree, wordnum, predicate, rolesets[wordnum], spanlist)
+            )
+
+        return instances
+
+    # /////////////////////////////////////////////////////////////////
+    # Helper Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def _require(self, *columntypes):
+        for columntype in columntypes:
+            if columntype not in self._colmap:
+                raise ValueError(
+                    "This corpus does not contain a %s " "column." % columntype
+                )
+
+    @staticmethod
+    def _get_column(grid, column_index):
+        return [grid[i][column_index] for i in range(len(grid))]
+
+
+class ConllSRLInstance:
+    """
+    An SRL instance from a CoNLL corpus, which identifies and
+    providing labels for the arguments of a single verb.
+    """
+
+    # [xx] add inst.core_arguments, inst.argm_arguments?
+
+    def __init__(self, tree, verb_head, verb_stem, roleset, tagged_spans):
+        self.verb = []
+        """A list of the word indices of the words that compose the
+           verb whose arguments are identified by this instance.
+           This will contain multiple word indices when multi-word
+           verbs are used (e.g. 'turn on')."""
+
+        self.verb_head = verb_head
+        """The word index of the head word of the verb whose arguments
+           are identified by this instance.  E.g., for a sentence that
+           uses the verb 'turn on,' ``verb_head`` will be the word index
+           of the word 'turn'."""
+
+        self.verb_stem = verb_stem
+
+        self.roleset = roleset
+
+        self.arguments = []
+        """A list of ``(argspan, argid)`` tuples, specifying the location
+           and type for each of the arguments identified by this
+           instance.  ``argspan`` is a tuple ``start, end``, indicating
+           that the argument consists of the ``words[start:end]``."""
+
+        self.tagged_spans = tagged_spans
+        """A list of ``(span, id)`` tuples, specifying the location and
+           type for each of the arguments, as well as the verb pieces,
+           that make up this instance."""
+
+        self.tree = tree
+        """The parse tree for the sentence containing this instance."""
+
+        self.words = tree.leaves()
+        """A list of the words in the sentence containing this
+           instance."""
+
+        # Fill in the self.verb and self.arguments values.
+        for (start, end), tag in tagged_spans:
+            if tag in ("V", "C-V"):
+                self.verb += list(range(start, end))
+            else:
+                self.arguments.append(((start, end), tag))
+
+    def __repr__(self):
+        # Originally, its:
+        ##plural = 's' if len(self.arguments) != 1 else ''
+        plural = "s" if len(self.arguments) != 1 else ""
+        return "<ConllSRLInstance for %r with %d argument%s>" % (
+            (self.verb_stem, len(self.arguments), plural)
+        )
+
+    def pprint(self):
+        verbstr = " ".join(self.words[i][0] for i in self.verb)
+        hdr = f"SRL for {verbstr!r} (stem={self.verb_stem!r}):\n"
+        s = ""
+        for i, word in enumerate(self.words):
+            if isinstance(word, tuple):
+                word = word[0]
+            for (start, end), argid in self.arguments:
+                if i == start:
+                    s += "[%s " % argid
+                if i == end:
+                    s += "] "
+            if i in self.verb:
+                word = "<<%s>>" % word
+            s += word + " "
+        return hdr + textwrap.fill(
+            s.replace(" ]", "]"), initial_indent="    ", subsequent_indent="    "
+        )
+
+
+class ConllSRLInstanceList(list):
+    """
+    Set of instances for a single sentence
+    """
+
+    def __init__(self, tree, instances=()):
+        self.tree = tree
+        list.__init__(self, instances)
+
+    def __str__(self):
+        return self.pprint()
+
+    def pprint(self, include_tree=False):
+        # Sanity check: trees should be the same
+        for inst in self:
+            if inst.tree != self.tree:
+                raise ValueError("Tree mismatch!")
+
+        # If desired, add trees:
+        if include_tree:
+            words = self.tree.leaves()
+            pos = [None] * len(words)
+            synt = ["*"] * len(words)
+            self._tree2conll(self.tree, 0, words, pos, synt)
+
+        s = ""
+        for i in range(len(words)):
+            # optional tree columns
+            if include_tree:
+                s += "%-20s " % words[i]
+                s += "%-8s " % pos[i]
+                s += "%15s*%-8s " % tuple(synt[i].split("*"))
+
+            # verb head column
+            for inst in self:
+                if i == inst.verb_head:
+                    s += "%-20s " % inst.verb_stem
+                    break
+            else:
+                s += "%-20s " % "-"
+            # Remaining columns: self
+            for inst in self:
+                argstr = "*"
+                for (start, end), argid in inst.tagged_spans:
+                    if i == start:
+                        argstr = f"({argid}{argstr}"
+                    if i == (end - 1):
+                        argstr += ")"
+                s += "%-12s " % argstr
+            s += "\n"
+        return s
+
+    def _tree2conll(self, tree, wordnum, words, pos, synt):
+        assert isinstance(tree, Tree)
+        if len(tree) == 1 and isinstance(tree[0], str):
+            pos[wordnum] = tree.label()
+            assert words[wordnum] == tree[0]
+            return wordnum + 1
+        elif len(tree) == 1 and isinstance(tree[0], tuple):
+            assert len(tree[0]) == 2
+            pos[wordnum], pos[wordnum] = tree[0]
+            return wordnum + 1
+        else:
+            synt[wordnum] = f"({tree.label()}{synt[wordnum]}"
+            for child in tree:
+                wordnum = self._tree2conll(child, wordnum, words, pos, synt)
+            synt[wordnum - 1] += ")"
+            return wordnum
+
+
+class ConllChunkCorpusReader(ConllCorpusReader):
+    """
+    A ConllCorpusReader whose data file contains three columns: words,
+    pos, and chunk.
+    """
+
+    def __init__(
+        self, root, fileids, chunk_types, encoding="utf8", tagset=None, separator=None
+    ):
+        ConllCorpusReader.__init__(
+            self,
+            root,
+            fileids,
+            ("words", "pos", "chunk"),
+            chunk_types=chunk_types,
+            encoding=encoding,
+            tagset=tagset,
+            separator=separator,
+        )

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/crubadan.py

@@ -0,0 +1,106 @@
+# Natural Language Toolkit: An Crubadan N-grams Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Avital Pekker <avital.pekker@utoronto.ca>
+#
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface for the n-gram statistics gathered from
+the corpora for each language using An Crubadan.
+
+There are multiple potential applications for the data but
+this reader was created with the goal of using it in the
+context of language identification.
+
+For details about An Crubadan, this data, and its potential uses, see:
+http://borel.slu.edu/crubadan/index.html
+"""
+
+import re
+from os import path
+
+from nltk.corpus.reader import CorpusReader
+from nltk.data import ZipFilePathPointer
+from nltk.probability import FreqDist
+
+
+class CrubadanCorpusReader(CorpusReader):
+    """
+    A corpus reader used to access language An Crubadan n-gram files.
+    """
+
+    _LANG_MAPPER_FILE = "table.txt"
+    _all_lang_freq = {}
+
+    def __init__(self, root, fileids, encoding="utf8", tagset=None):
+        super().__init__(root, fileids, encoding="utf8")
+        self._lang_mapping_data = []
+        self._load_lang_mapping_data()
+
+    def lang_freq(self, lang):
+        """Return n-gram FreqDist for a specific language
+        given ISO 639-3 language code"""
+
+        if lang not in self._all_lang_freq:
+            self._all_lang_freq[lang] = self._load_lang_ngrams(lang)
+
+        return self._all_lang_freq[lang]
+
+    def langs(self):
+        """Return a list of supported languages as ISO 639-3 codes"""
+        return [row[1] for row in self._lang_mapping_data]
+
+    def iso_to_crubadan(self, lang):
+        """Return internal Crubadan code based on ISO 639-3 code"""
+        for i in self._lang_mapping_data:
+            if i[1].lower() == lang.lower():
+                return i[0]
+
+    def crubadan_to_iso(self, lang):
+        """Return ISO 639-3 code given internal Crubadan code"""
+        for i in self._lang_mapping_data:
+            if i[0].lower() == lang.lower():
+                return i[1]
+
+    def _load_lang_mapping_data(self):
+        """Load language mappings between codes and description from table.txt"""
+        if isinstance(self.root, ZipFilePathPointer):
+            raise RuntimeError(
+                "Please install the 'crubadan' corpus first, use nltk.download()"
+            )
+
+        mapper_file = path.join(self.root, self._LANG_MAPPER_FILE)
+        if self._LANG_MAPPER_FILE not in self.fileids():
+            raise RuntimeError("Could not find language mapper file: " + mapper_file)
+
+        with open(mapper_file, encoding="utf-8") as raw:
+            strip_raw = raw.read().strip()
+
+            self._lang_mapping_data = [row.split("\t") for row in strip_raw.split("\n")]
+
+    def _load_lang_ngrams(self, lang):
+        """Load single n-gram language file given the ISO 639-3 language code
+        and return its FreqDist"""
+
+        if lang not in self.langs():
+            raise RuntimeError("Unsupported language.")
+
+        crubadan_code = self.iso_to_crubadan(lang)
+        ngram_file = path.join(self.root, crubadan_code + "-3grams.txt")
+
+        if not path.isfile(ngram_file):
+            raise RuntimeError("No N-gram file found for requested language.")
+
+        counts = FreqDist()
+        with open(ngram_file, encoding="utf-8") as f:
+            for line in f:
+                data = line.split(" ")
+
+                ngram = data[1].strip("\n")
+                freq = int(data[0])
+
+                counts[ngram] = freq
+
+        return counts

.eggs/nltk-3.8-py3.10.egg/nltk/corpus/reader/dependency.py

@@ -0,0 +1,115 @@
+# Natural Language Toolkit: Dependency Corpus Reader
+#
+# Copyright (C) 2001-2022 NLTK Project
+# Author: Kepa Sarasola <kepa.sarasola@ehu.es>
+#         Iker Manterola <returntothehangar@hotmail.com>
+#
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.util import *
+from nltk.parse import DependencyGraph
+from nltk.tokenize import *
+
+
+class DependencyCorpusReader(SyntaxCorpusReader):
+    def __init__(
+        self,
+        root,
+        fileids,
+        encoding="utf8",
+        word_tokenizer=TabTokenizer(),
+        sent_tokenizer=RegexpTokenizer("\n", gaps=True),
+        para_block_reader=read_blankline_block,
+    ):
+        SyntaxCorpusReader.__init__(self, root, fileids, encoding)
+
+    #########################################################
+
+    def words(self, fileids=None):
+        return concat(
+            [
+                DependencyCorpusView(fileid, False, False, False, encoding=enc)
+                for fileid, enc in self.abspaths(fileids, include_encoding=True)
+            ]
+        )
+
+    def tagged_words(self, fileids=None):
+        return concat(
+            [
+                DependencyCorpusView(fileid, True, False, False, encoding=enc)
+                for fileid, enc in self.abspaths(fileids, include_encoding=True)
+            ]
+        )
+
+    def sents(self, fileids=None):
+        return concat(
+            [
+                DependencyCorpusView(fileid, False, True, False, encoding=enc)
+                for fileid, enc in self.abspaths(fileids, include_encoding=True)
+            ]
+        )
+
+    def tagged_sents(self, fileids=None):
+        return concat(
+            [
+                DependencyCorpusView(fileid, True, True, False, encoding=enc)
+                for fileid, enc in self.abspaths(fileids, include_encoding=True)
+            ]
+        )
+
+    def parsed_sents(self, fileids=None):
+        sents = concat(
+            [
+                DependencyCorpusView(fileid, False, True, True, encoding=enc)
+                for fileid, enc in self.abspaths(fileids, include_encoding=True)
+            ]
+        )
+        return [DependencyGraph(sent) for sent in sents]
+
+
+class DependencyCorpusView(StreamBackedCorpusView):
+    _DOCSTART = "-DOCSTART- -DOCSTART- O\n"  # dokumentu hasiera definitzen da
+
+    def __init__(
+        self,
+        corpus_file,
+        tagged,
+        group_by_sent,
+        dependencies,
+        chunk_types=None,
+        encoding="utf8",
+    ):
+        self._tagged = tagged
+        self._dependencies = dependencies
+        self._group_by_sent = group_by_sent
+        self._chunk_types = chunk_types
+        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+    def read_block(self, stream):
+        # Read the next sentence.
+        sent = read_blankline_block(stream)[0].strip()
+        # Strip off the docstart marker, if present.
+        if sent.startswith(self._DOCSTART):
+            sent = sent[len(self._DOCSTART) :].lstrip()
+
+        # extract word and tag from any of the formats
+        if not self._dependencies:
+            lines = [line.split("\t") for line in sent.split("\n")]
+            if len(lines[0]) == 3 or len(lines[0]) == 4:
+                sent = [(line[0], line[1]) for line in lines]
+            elif len(lines[0]) == 10:
+                sent = [(line[1], line[4]) for line in lines]
+            else:
+                raise ValueError("Unexpected number of fields in dependency tree file")
+
+            # discard tags if they weren't requested
+            if not self._tagged:
+                sent = [word for (word, tag) in sent]
+
+        # Return the result.
+        if self._group_by_sent:
+            return [sent]
+        else:
+            return list(sent)