tech-chat rag first push

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+LMI[[:space:]]HUMAN[[:space:]]RESOURCE[[:space:]]POLICY.pdf filter=lfs diff=lfs merge=lfs -text
+LMI-HUMAN-RESOURCE-POLICY.pdf filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+WORKDIR /app
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY app.py .
+COPY "Admission_Requirement.pdf" .
+COPY "POSTGRADUATE_ADMISSIONS.pdf" .
+COPY "cleaned-dataset.jsonl" .
+RUN chmod 644 "Admission_Requirement.pdf" "POSTGRADUATE_ADMISSIONS.pdf" "cleaned-dataset.jsonl"
+COPY nltk_data /app/nltk_data
+RUN mkdir -p /app/cache /app/tmp
+RUN chmod -R 777 /app
+EXPOSE 7860
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,10 @@
+---
+title: Hr Policy Bot
+emoji: 💻
+colorFrom: blue
+colorTo: pink
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,196 @@
+import os
+import nltk
+import logging
+import json
+import numpy as np
+from sklearn.cluster import KMeans
+nltk.data.path.append("/app/nltk_data")
+
+os.environ["HF_HOME"] = "/app/cache"
+os.environ["XDG_CACHE_HOME"] = "/app/cache"
+os.environ["TMPDIR"] = "/app/tmp"
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+from langchain_core.output_parsers import StrOutputParser
+from langchain_community.retrievers import BM25Retriever
+from langchain.retrievers import EnsembleRetriever
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_core.documents import Document
+
+app = FastAPI(title="HR Policy Bot")
+
+class Question(BaseModel):
+    text: str
+
+def semantic_chunk_with_embeddings(documents, embeddings, max_chunk_size=1000, min_sentences=2, overlap_sentences=1):
+    """Chunk documents into semantically related groups using embeddings and clustering."""
+    all_chunks = []
+    for doc in documents:
+        sentences = nltk.sent_tokenize(doc.page_content)
+        if len(sentences) < min_sentences:
+            all_chunks.append(Document(page_content=" ".join(sentences), metadata=doc.metadata))
+            continue
+
+        # Generate embeddings for each sentence
+        sentence_embeddings = embeddings.embed_documents(sentences)
+        sentence_embeddings = np.array(sentence_embeddings)
+
+        # Cluster sentences using KMeans (dynamically determine num clusters)
+        num_clusters = max(1, min(len(sentences) // min_sentences, 10))  # Cap at 10 clusters
+        kmeans = KMeans(n_clusters=num_clusters, random_state=42).fit(sentence_embeddings)
+        labels = kmeans.labels_
+
+        # Group sentences by cluster
+        clusters = {}
+        for sentence, label in zip(sentences, labels):
+            if label not in clusters:
+                clusters[label] = []
+            clusters[label].append(sentence)
+
+        # Form chunks from clusters with overlap
+        for cluster_id, cluster_sentences in clusters.items():
+            current_chunk = ""
+            chunk_sentences = []
+            for i, sentence in enumerate(cluster_sentences):
+                if len(current_chunk) + len(sentence) < max_chunk_size:
+                    current_chunk += sentence + " "
+                    chunk_sentences.append(sentence)
+                else:
+                    all_chunks.append(Document(page_content=current_chunk.strip(), metadata=doc.metadata))
+                    # Add overlap
+                    overlap = " ".join(chunk_sentences[-overlap_sentences:]) + " "
+                    current_chunk = overlap + sentence + " "
+                    chunk_sentences = chunk_sentences[-overlap_sentences:] + [sentence]
+            if current_chunk:
+                all_chunks.append(Document(page_content=current_chunk.strip(), metadata=doc.metadata))
+
+    return all_chunks
+
+def load_rag_system():
+    logger.info("Loading Gemini model...")
+    try:
+        llm = ChatGoogleGenerativeAI(
+            model="gemini-2.0-flash",
+            google_api_key=os.getenv("GOOGLE_API_KEY"),
+            temperature=0.3,
+            top_p=0.9,
+            max_tokens=1024
+        )
+    except Exception as e:
+        logger.error(f"Gemini loading failed: {str(e)}")
+        raise
+
+    # Load embeddings for chunking and retrieval
+    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+    # Load PDF
+    logger.info("Loading PDF...")
+    pdf_paths = ["Admission_Requirement.pdf", "POSTGRADUATE_ADMISSIONS.pdf"]
+    pages = []
+    for path in pdf_paths:
+        if os.path.exists(path):
+            loader = PyPDFLoader(path)
+            pages.extend(loader.load())
+        else:
+            logger.warning(f"PDF not found: {path}")
+
+    pdf_docs = semantic_chunk_with_embeddings(pages, embeddings)
+
+    # Load JSONL
+    logger.info("Loading JSONL data...")
+    jsonl_paths = ["cleaned-dataset.jsonl"]
+    jsonl_docs = []
+    for path in jsonl_paths:
+        if os.path.exists(path):
+            with open(path, "r") as f:
+                for line in f:
+                    data = json.loads(line.strip())
+                    content = f"Instruction: {data['instruction']}\nResponse: {data['response']}"
+                    jsonl_docs.append(Document(page_content=content, metadata={"source": "jsonl", "instruction": data["instruction"]}))
+        else:
+            logger.warning(f"JSONL not found: {path}")
+
+    # Combine documents
+    all_docs = pdf_docs + jsonl_docs
+    unique_docs = {doc.page_content: doc for doc in all_docs}.values()
+    for i, doc in enumerate(unique_docs):
+        doc.metadata["doc_id"] = i
+
+    logger.info("Building vector store...")
+    vectorstore = FAISS.from_documents(list(unique_docs), embedding=embeddings)
+    faiss_retriever = vectorstore.as_retriever(search_kwargs={"k": 4})  # Increased to 4
+    bm25_retriever = BM25Retriever.from_documents(list(unique_docs))
+    bm25_retriever.k = 4  # Increased to 4
+    retriever = EnsembleRetriever(retrievers=[faiss_retriever, bm25_retriever], weights=[0.5, 0.5])
+
+    template = """
+    You are TechChat, an AI assistant created to provide accurate, concise, and helpful information about admissions to Kwame Nkrumah University of Science and Technology (KNUST). Your primary goal is to assist users with questions related to KNUST admissions, including application processes, requirements, deadlines, programs, and other relevant details.
+
+    ### Instructions:
+    1. **KNUST Admissions Questions**: Use the provided context to answer questions about KNUST admissions clearly and accurately. If the context is sufficient, tailor your response to the specific details provided.
+    2. **Limited Context**: If the context lacks enough information to fully answer a KNUST admissions question, provide a general but accurate response based on your knowledge of KNUST admissions, and invite the user to provide more details for a more specific answer.
+    3. **Off-Topic Questions**: If the question is unrelated to KNUST admissions, respond politely with: "I'm sorry, that question is outside my focus on KNUST admissions. Feel free to ask about KNUST application processes, requirements, or programs, and I'll be happy to help!"
+    4. **Tone and Style**: Maintain a friendly, professional, and approachable tone. Avoid overly technical jargon unless necessary, and ensure responses are easy to understand.
+    5. **No Assumptions**: Do not invent information. If you cannot answer due to missing or unclear information, acknowledge it and encourage the user to clarify.
+
+    ### Context:
+    {context}
+
+    ### Question:
+    {question}
+
+    ### Answer:
+    """
+    
+    prompt = PromptTemplate.from_template(template)
+    parser = StrOutputParser()
+    chain = LLMChain(llm=llm, prompt=prompt, output_parser=parser)
+
+    return retriever, vectorstore, chain
+
+logger.info("Initializing RAG system...")
+retriever, vectorstore, chain = load_rag_system()
+
+def rewrite_query(question):
+    if "dress" in question.lower() and "employees" in question.lower():
+        return "What is the dress code policy for employees regarding casual attire?"
+    return question
+
+@app.post("/ask")
+async def ask(question: Question):
+    print('ask route reached')
+    try:
+        logger.info(f"Received question: {question.text}")
+        rewritten_question = rewrite_query(question.text)
+        logger.info(f"Rewritten question: {rewritten_question}")
+        context_docs = retriever.invoke(rewritten_question)
+        logger.info(f"Retrieved {len(context_docs)} context documents")
+        max_similarity = max([vectorstore.similarity_search_with_score(rewritten_question, k=1)[0][1] for _ in context_docs], default=0)
+        if max_similarity < 0.25:  # Lowered threshold slightly
+            logger.info("Similarity too low, returning 'I don’t know'")
+            return {"answer": "I'm not sure about that, but I'd be happy to help if you provide more details!"}
+        context_text = "\n".join([doc.page_content for doc in context_docs])
+        logger.info("Generating response...")
+        response = chain.invoke({"context": context_text, "question": rewritten_question})
+        answer = response['text'].split("Answer:")[-1].strip()
+        logger.info(f"Generated answer: {answer}")
+        return {"answer": answer}
+    except Exception as e:
+        logger.error(f"Error processing question: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/")
+async def root():
+    print('HR Policy Bot is actively running!')
+    logger.info("Root endpoint accessed")
+    return {"message": "HR Policy Bot is running!"}
+

nltk_data/tokenizers/punkt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51c3078994aeaf650bfc8e028be4fb42b4a0d177d41c012b6a983979653660ec
+size 13905355

nltk_data/tokenizers/punkt/PY3/README

@@ -0,0 +1,98 @@
+Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
+
+Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
+been contributed by various people using NLTK for sentence boundary detection.
+
+For information about how to use these models, please confer the tokenization HOWTO:
+http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
+and chapter 3.8 of the NLTK book:
+http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
+
+There are pretrained tokenizers for the following languages:
+
+File                Language            Source                             Contents                Size of training corpus(in tokens)           Model contributed by
+=======================================================================================================================================================================
+czech.pickle        Czech               Multilingual Corpus 1 (ECI)        Lidove Noviny                   ~345,000                             Jan Strunk / Tibor Kiss
+                                                                           Literarni Noviny
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+danish.pickle       Danish              Avisdata CD-Rom Ver. 1.1. 1995     Berlingske Tidende              ~550,000                             Jan Strunk / Tibor Kiss
+                                        (Berlingske Avisdata, Copenhagen)  Weekend Avisen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+dutch.pickle        Dutch               Multilingual Corpus 1 (ECI)        De Limburger                    ~340,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+english.pickle      English             Penn Treebank (LDC)                Wall Street Journal             ~469,000                             Jan Strunk / Tibor Kiss
+                    (American)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+estonian.pickle     Estonian            University of Tartu, Estonia       Eesti Ekspress                  ~359,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+finnish.pickle      Finnish             Finnish Parole Corpus, Finnish     Books and major national        ~364,000                             Jan Strunk / Tibor Kiss
+                                        Text Bank (Suomen Kielen           newspapers
+                                        Tekstipankki)
+                                        Finnish Center for IT Science
+                                        (CSC)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+french.pickle       French              Multilingual Corpus 1 (ECI)        Le Monde                        ~370,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+german.pickle       German              Neue Zürcher Zeitung AG            Neue Zürcher Zeitung            ~847,000                             Jan Strunk / Tibor Kiss
+                    (Switzerland)       CD-ROM
+                    (Uses "ss"
+                     instead of "ß")
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+greek.pickle        Greek               Efstathios Stamatatos              To Vima (TO BHMA)               ~227,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+italian.pickle      Italian             Multilingual Corpus 1 (ECI)        La Stampa, Il Mattino           ~312,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+norwegian.pickle    Norwegian           Centre for Humanities              Bergens Tidende                 ~479,000                             Jan Strunk / Tibor Kiss
+                    (Bokmål and         Information Technologies,
+                     Nynorsk)           Bergen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+polish.pickle       Polish              Polish National Corpus             Literature, newspapers, etc.  ~1,000,000                             Krzysztof Langner
+                                        (http://www.nkjp.pl/)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+portuguese.pickle   Portuguese          CETENFolha Corpus                  Folha de São Paulo              ~321,000                             Jan Strunk / Tibor Kiss
+                    (Brazilian)         (Linguateca)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+slovene.pickle      Slovene             TRACTOR                            Delo                            ~354,000                             Jan Strunk / Tibor Kiss
+                                        Slovene Academy for Arts
+                                        and Sciences
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+spanish.pickle      Spanish             Multilingual Corpus 1 (ECI)        Sur                             ~353,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+swedish.pickle      Swedish             Multilingual Corpus 1 (ECI)        Dagens Nyheter                  ~339,000                             Jan Strunk / Tibor Kiss
+                                                                           (and some other texts)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+turkish.pickle      Turkish             METU Turkish Corpus                Milliyet                        ~333,000                             Jan Strunk / Tibor Kiss
+                                        (Türkçe Derlem Projesi)
+                                        University of Ankara
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
+Unicode using the codecs module.
+
+Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
+Computational Linguistics 32: 485-525.
+
+---- Training Code ----
+
+# import punkt
+import nltk.tokenize.punkt
+
+# Make a new Tokenizer
+tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
+
+# Read in training corpus (one example: Slovene)
+import codecs
+text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
+
+# Train tokenizer
+tokenizer.train(text)
+
+# Dump pickled tokenizer
+import pickle
+out = open("slovene.pickle","wb")
+pickle.dump(tokenizer, out)
+out.close()
+
+---------

nltk_data/tokenizers/punkt/PY3/czech.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b0734b6fbe8e8d7cac79f48d1dd9f853824e57c4e3594dadd74ba2c1d97f50
+size 1119050

nltk_data/tokenizers/punkt/PY3/danish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6189c7dd254e29e2bd406a7f6a4336297c8953214792466a790ea4444223ceb3
+size 1191710

nltk_data/tokenizers/punkt/PY3/dutch.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda0d6a13f02e8898daec7fe923da88e25abe081bcfa755c0e015075c215fe4c
+size 693759

nltk_data/tokenizers/punkt/PY3/english.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cad3758596392364e3be9803dbd7ebeda384b68937b488a01365f5551bb942c
+size 406697

nltk_data/tokenizers/punkt/PY3/estonian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b364f72538d17b146a98009ad239a8096ce6c0a8b02958c0bc776ecd0c58a25f
+size 1499502

nltk_data/tokenizers/punkt/PY3/finnish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a4b5ff5500ee851c456f9dd40d5fc0d8c1859c88eb3178de1317d26b7d22833
+size 1852226

nltk_data/tokenizers/punkt/PY3/french.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e3a4cd2971989b3cb9fd3433a6f15d17981e464db2be039364313b5de94f29
+size 553575

nltk_data/tokenizers/punkt/PY3/german.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddcbbe85e2042a019b1a6e37fd8c153286c38ba201fae0f5bfd9a3f74abae25c
+size 1463575

nltk_data/tokenizers/punkt/PY3/greek.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85dabc44ab90a5f208ef37ff6b4892ebe7e740f71fb4da47cfd95417ca3e22fd
+size 876006

nltk_data/tokenizers/punkt/PY3/italian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a94007b1e4ffdc4d1a190185ca5442c3dafeb17ab39d30329e84cd74a43947
+size 615089

nltk_data/tokenizers/punkt/PY3/malayalam.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f8cf58acbdb7f472ac40affc13663be42dafb47c15030c11ade0444c9e0e53d
+size 221207

nltk_data/tokenizers/punkt/PY3/norwegian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ff7a46d1438b311457d15d7763060b8d3270852c1850fd788c5cee194dc4a1d
+size 1181271

nltk_data/tokenizers/punkt/PY3/polish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:624900ae3ddfb4854a98c5d3b8b1c9bb719975f33fee61ce1441dab9f8a00718
+size 1738386

nltk_data/tokenizers/punkt/PY3/portuguese.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02a0b7b25c3c7471e1791b66a31bbb530afbb0160aee4fcecf0107652067b4a1
+size 611919

nltk_data/tokenizers/punkt/PY3/russian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549762f8190024d89b511472df21a3a135eee5d9233e63ac244db737c2c61d7e
+size 33020

nltk_data/tokenizers/punkt/PY3/slovene.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ef2cc0ed27d79b3aa635cbbc40ad811883a75a4b8a8be1ae406972870fd864
+size 734444

nltk_data/tokenizers/punkt/PY3/spanish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:164a50fadc5a49f8ec7426eae11d3111ee752b48a3ef373d47745011192a5984
+size 562337

nltk_data/tokenizers/punkt/PY3/swedish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0f7d538bfd5266633b09e842cd92e9e0ac10f1d923bf211e1497972ddc47318
+size 979681

nltk_data/tokenizers/punkt/PY3/turkish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae68ef5863728ac5332e87eb1f6bae772ff32a13a4caa2b01a5c68103e853c5b
+size 1017038

nltk_data/tokenizers/punkt/README

@@ -0,0 +1,98 @@
+Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
+
+Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
+been contributed by various people using NLTK for sentence boundary detection.
+
+For information about how to use these models, please confer the tokenization HOWTO:
+http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
+and chapter 3.8 of the NLTK book:
+http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
+
+There are pretrained tokenizers for the following languages:
+
+File                Language            Source                             Contents                Size of training corpus(in tokens)           Model contributed by
+=======================================================================================================================================================================
+czech.pickle        Czech               Multilingual Corpus 1 (ECI)        Lidove Noviny                   ~345,000                             Jan Strunk / Tibor Kiss
+                                                                           Literarni Noviny
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+danish.pickle       Danish              Avisdata CD-Rom Ver. 1.1. 1995     Berlingske Tidende              ~550,000                             Jan Strunk / Tibor Kiss
+                                        (Berlingske Avisdata, Copenhagen)  Weekend Avisen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+dutch.pickle        Dutch               Multilingual Corpus 1 (ECI)        De Limburger                    ~340,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+english.pickle      English             Penn Treebank (LDC)                Wall Street Journal             ~469,000                             Jan Strunk / Tibor Kiss
+                    (American)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+estonian.pickle     Estonian            University of Tartu, Estonia       Eesti Ekspress                  ~359,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+finnish.pickle      Finnish             Finnish Parole Corpus, Finnish     Books and major national        ~364,000                             Jan Strunk / Tibor Kiss
+                                        Text Bank (Suomen Kielen           newspapers
+                                        Tekstipankki)
+                                        Finnish Center for IT Science
+                                        (CSC)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+french.pickle       French              Multilingual Corpus 1 (ECI)        Le Monde                        ~370,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+german.pickle       German              Neue Zürcher Zeitung AG            Neue Zürcher Zeitung            ~847,000                             Jan Strunk / Tibor Kiss
+                    (Switzerland)       CD-ROM
+                    (Uses "ss"
+                     instead of "ß")
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+greek.pickle        Greek               Efstathios Stamatatos              To Vima (TO BHMA)               ~227,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+italian.pickle      Italian             Multilingual Corpus 1 (ECI)        La Stampa, Il Mattino           ~312,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+norwegian.pickle    Norwegian           Centre for Humanities              Bergens Tidende                 ~479,000                             Jan Strunk / Tibor Kiss
+                    (Bokmål and         Information Technologies,
+                     Nynorsk)           Bergen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+polish.pickle       Polish              Polish National Corpus             Literature, newspapers, etc.  ~1,000,000                             Krzysztof Langner
+                                        (http://www.nkjp.pl/)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+portuguese.pickle   Portuguese          CETENFolha Corpus                  Folha de São Paulo              ~321,000                             Jan Strunk / Tibor Kiss
+                    (Brazilian)         (Linguateca)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+slovene.pickle      Slovene             TRACTOR                            Delo                            ~354,000                             Jan Strunk / Tibor Kiss
+                                        Slovene Academy for Arts
+                                        and Sciences
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+spanish.pickle      Spanish             Multilingual Corpus 1 (ECI)        Sur                             ~353,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+swedish.pickle      Swedish             Multilingual Corpus 1 (ECI)        Dagens Nyheter                  ~339,000                             Jan Strunk / Tibor Kiss
+                                                                           (and some other texts)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+turkish.pickle      Turkish             METU Turkish Corpus                Milliyet                        ~333,000                             Jan Strunk / Tibor Kiss
+                                        (Türkçe Derlem Projesi)
+                                        University of Ankara
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
+Unicode using the codecs module.
+
+Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
+Computational Linguistics 32: 485-525.
+
+---- Training Code ----
+
+# import punkt
+import nltk.tokenize.punkt
+
+# Make a new Tokenizer
+tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
+
+# Read in training corpus (one example: Slovene)
+import codecs
+text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
+
+# Train tokenizer
+tokenizer.train(text)
+
+# Dump pickled tokenizer
+import pickle
+out = open("slovene.pickle","wb")
+pickle.dump(tokenizer, out)
+out.close()
+
+---------

nltk_data/tokenizers/punkt/czech.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba73d293c7d7953956bcf02f3695ec5c1f0d527f2a3c38097f5593394fa1690
+size 1265552

nltk_data/tokenizers/punkt/danish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea29760a0a9197f52ca59e78aeafc5a6f55d05258faf7db1709b2b9eb321ef20
+size 1264725

nltk_data/tokenizers/punkt/dutch.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a8e26b3d68c45c38e594d19e2d5677447bfdcaa636d3b1e7acfed0e9272d73c
+size 742624

nltk_data/tokenizers/punkt/english.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dda37972ae88998a6fd3e3ec002697a6bd362b32d050fda7d7ca5276873092aa
+size 433305

nltk_data/tokenizers/punkt/estonian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3867fee26a36bdb197c64362aa13ac683f5f33fa4d0d225a5d56707582a55a1d
+size 1596714

nltk_data/tokenizers/punkt/finnish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a9e17b3d5b4df76345d812b8a65b1da0767eda5086eadcc11e625eef0942835
+size 1951656

nltk_data/tokenizers/punkt/french.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de05f3d5647d3d2296626fb83f68428e4c6ad6e05a00ed4694c8bdc8f2f197ee
+size 583482

nltk_data/tokenizers/punkt/german.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eab497fa085413130c8fd0fb13b929128930afe2f6a26ea8715c95df7088e97c
+size 1526714

nltk_data/tokenizers/punkt/greek.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21752a6762fad5cfe46fb5c45fad9a85484a0e8e81c67e6af6fb973cfc27d67c
+size 1953106

nltk_data/tokenizers/punkt/italian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcb2717d7be5f26e860a92e05acf69b1123a5f4527cd7a269a9ab9e9e668c805
+size 658331

nltk_data/tokenizers/punkt/malayalam.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f8cf58acbdb7f472ac40affc13663be42dafb47c15030c11ade0444c9e0e53d
+size 221207

nltk_data/tokenizers/punkt/norwegian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a97f8f9a03a0338dd746bcc89a0ae0f54ae43b835fa37d83e279e1ca794faf
+size 1259779

nltk_data/tokenizers/punkt/polish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16127b6d10933427a3e90fb20e9be53e1fb371ff79a730c1030734ed80b90c92
+size 2042451

nltk_data/tokenizers/punkt/portuguese.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb01bf7c79a4eadc2178bbd209665139a0e4b38f2d1c44fef097de93955140e0
+size 649051

nltk_data/tokenizers/punkt/russian.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc984432fbe31f7000014f8047502476889169c60f09be5413ca09276b16c909
+size 33027

nltk_data/tokenizers/punkt/slovene.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dac650212b3787b39996c01bd2084115493e6f6ec390bab61f767525b08b8ea
+size 832867

nltk_data/tokenizers/punkt/spanish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:271dc6027c4aae056f72a9bfab5645cf67e198bf4f972895844e40f5989ccdc3
+size 597831

nltk_data/tokenizers/punkt/swedish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d50ebdad6caa87715f2e300b1217ec92c42de205a543cc4a56903bd2c9acfa
+size 1034496

nltk_data/tokenizers/punkt/turkish.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3ae47d76501d027698809d12e75292c9c392910488543342802f95db9765ccc
+size 1225013

nltk_data/tokenizers/punkt_tab.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106
+size 4319076

nltk_data/tokenizers/punkt_tab/README

@@ -0,0 +1,98 @@
+Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
+
+Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
+been contributed by various people using NLTK for sentence boundary detection.
+
+For information about how to use these models, please confer the tokenization HOWTO:
+http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
+and chapter 3.8 of the NLTK book:
+http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
+
+There are pretrained tokenizers for the following languages:
+
+File                Language            Source                             Contents                Size of training corpus(in tokens)           Model contributed by
+=======================================================================================================================================================================
+czech.pickle        Czech               Multilingual Corpus 1 (ECI)        Lidove Noviny                   ~345,000                             Jan Strunk / Tibor Kiss
+                                                                           Literarni Noviny
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+danish.pickle       Danish              Avisdata CD-Rom Ver. 1.1. 1995     Berlingske Tidende              ~550,000                             Jan Strunk / Tibor Kiss
+                                        (Berlingske Avisdata, Copenhagen)  Weekend Avisen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+dutch.pickle        Dutch               Multilingual Corpus 1 (ECI)        De Limburger                    ~340,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+english.pickle      English             Penn Treebank (LDC)                Wall Street Journal             ~469,000                             Jan Strunk / Tibor Kiss
+                    (American)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+estonian.pickle     Estonian            University of Tartu, Estonia       Eesti Ekspress                  ~359,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+finnish.pickle      Finnish             Finnish Parole Corpus, Finnish     Books and major national        ~364,000                             Jan Strunk / Tibor Kiss
+                                        Text Bank (Suomen Kielen           newspapers
+                                        Tekstipankki)
+                                        Finnish Center for IT Science
+                                        (CSC)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+french.pickle       French              Multilingual Corpus 1 (ECI)        Le Monde                        ~370,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+german.pickle       German              Neue Zürcher Zeitung AG            Neue Zürcher Zeitung            ~847,000                             Jan Strunk / Tibor Kiss
+                    (Switzerland)       CD-ROM
+                    (Uses "ss"
+                     instead of "ß")
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+greek.pickle        Greek               Efstathios Stamatatos              To Vima (TO BHMA)               ~227,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+italian.pickle      Italian             Multilingual Corpus 1 (ECI)        La Stampa, Il Mattino           ~312,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+norwegian.pickle    Norwegian           Centre for Humanities              Bergens Tidende                 ~479,000                             Jan Strunk / Tibor Kiss
+                    (Bokmål and         Information Technologies,
+                     Nynorsk)           Bergen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+polish.pickle       Polish              Polish National Corpus             Literature, newspapers, etc.  ~1,000,000                             Krzysztof Langner
+                                        (http://www.nkjp.pl/)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+portuguese.pickle   Portuguese          CETENFolha Corpus                  Folha de São Paulo              ~321,000                             Jan Strunk / Tibor Kiss
+                    (Brazilian)         (Linguateca)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+slovene.pickle      Slovene             TRACTOR                            Delo                            ~354,000                             Jan Strunk / Tibor Kiss
+                                        Slovene Academy for Arts
+                                        and Sciences
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+spanish.pickle      Spanish             Multilingual Corpus 1 (ECI)        Sur                             ~353,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+swedish.pickle      Swedish             Multilingual Corpus 1 (ECI)        Dagens Nyheter                  ~339,000                             Jan Strunk / Tibor Kiss
+                                                                           (and some other texts)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+turkish.pickle      Turkish             METU Turkish Corpus                Milliyet                        ~333,000                             Jan Strunk / Tibor Kiss
+                                        (Türkçe Derlem Projesi)
+                                        University of Ankara
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
+Unicode using the codecs module.
+
+Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
+Computational Linguistics 32: 485-525.
+
+---- Training Code ----
+
+# import punkt
+import nltk.tokenize.punkt
+
+# Make a new Tokenizer
+tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
+
+# Read in training corpus (one example: Slovene)
+import codecs
+text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
+
+# Train tokenizer
+tokenizer.train(text)
+
+# Dump pickled tokenizer
+import pickle
+out = open("slovene.pickle","wb")
+pickle.dump(tokenizer, out)
+out.close()
+
+---------

nltk_data/tokenizers/punkt_tab/czech/abbrev_types.txt

@@ -0,0 +1,118 @@
+t
+množ
+např
+j.h
+man
+ú
+jug
+dr
+bl
+ml
+okr
+st
+uh
+šp
+judr
+u.s.a
+p
+arg
+žitě
+st.celsia
+etc
+p.s
+t.r
+lok
+mil
+ict
+n
+tl
+min
+č
+d
+al
+ravenně
+mj
+nar
+plk
+s.p
+a.g
+roč
+b
+zdi
+r.s.c
+přek
+m
+gen
+csc
+mudr
+vic
+š
+sb
+resp
+tzn
+iv
+s.r.o
+mar
+w
+čs
+vi
+tzv
+ul
+pen
+zv
+str
+čp
+org
+rak
+sv
+pplk
+u.s
+prof
+c.k
+op
+g
+vii
+kr
+ing
+j.o
+drsc
+m3
+l
+tr
+ceo
+ch
+fuk
+vl
+viii
+líp
+hl.m
+t.zv
+phdr
+o.k
+tis
+doc
+kl
+ard
+čkd
+pok
+apod
+r
+př
+a.s
+j
+jr
+i.m
+e
+kupř
+f
+tř
+xvi
+mir
+atď
+vr
+r.i.v
+hl
+kv
+t.j
+y
+q.p.r

nltk_data/tokenizers/punkt_tab/czech/collocations.tab

@@ -0,0 +1,96 @@
+i	dejmala
+##number##	prosince
+h	steina
+##number##	listopadu
+a	dvořák
+v	klaus
+i	čnhl
+##number##	wladyslawowo
+##number##	letech
+a	jiráska
+a	dubček
+##number##	štrasburk
+##number##	juniorské
+##number##	století
+##number##	kola
+##number##	pád
+##number##	května
+##number##	týdne
+v	dlouhý
+k	design
+##number##	červenec
+i	ligy
+##number##	kolo
+z	svěrák
+##number##	mája
+##number##	šimková
+a	bělého
+a	bradáč
+##number##	ročníku
+##number##	dubna
+a	vivaldiho
+v	mečiara
+c	carrićre
+##number##	sjezd
+##number##	výroční
+##number##	kole
+##number##	narozenin
+k	maleevová
+i	čnfl
+##number##	pádě
+##number##	září
+##number##	výročí
+a	dvořáka
+h	g.
+##number##	ledna
+a	dvorský
+h	měsíc
+##number##	srpna
+##number##	tř.
+a	mozarta
+##number##	sudetoněmeckých
+o	sokolov
+k	škrach
+v	benda
+##number##	symfonie
+##number##	července
+x	šalda
+c	abrahama
+a	tichý
+##number##	místo
+k	bielecki
+v	havel
+##number##	etapu
+a	dubčeka
+i	liga
+##number##	světový
+v	klausem
+##number##	ženy
+##number##	létech
+##number##	minutě
+##number##	listopadem
+##number##	místě
+o	vlček
+k	peteraje
+i	sponzor
+##number##	června
+##number##	min.
+##number##	oprávněnou
+##number##	květnu
+##number##	aktu
+##number##	květnem
+##number##	října
+i	rynda
+##number##	února
+i	snfl
+a	mozart
+z	košler
+a	dvorskému
+v	marhoul
+v	mečiar
+##number##	ročník
+##number##	máje
+v	havla
+k	gott
+s	bacha
+##number##	ad