Initital Commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+src/chatbot_error.log
+src/chatbot.db
+src/mlb.joblib
+src/mlb.pkl
+src/my_model.h5
+src/rf_classifier.joblib
+src/templates.zip
+src/tfidf_vectorizer.joblib
+src/tokenizer.pkl
+src/.env
+.env

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.10
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the project files
+COPY . /app
+
+#Copy the environment file
+COPY .env /app/.env
+
+# Copy nltk_data to the appropriate location
+COPY src/nltk_data /root/nltk_data
+
+# Set the NLTK_DATA environment variable
+ENV NLTK_DATA=/root/nltk_data
+
+# Install dependencies
+RUN pip install --no-cache-dir -r src/requirements.txt
+
+# Expose port 2000
+EXPOSE 2000
+
+# Run the Flask app
+CMD ["python", "src/final_code.py"]

src/Classifier.py

@@ -0,0 +1,65 @@
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+# from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import train_test_split
+# from sklearn.preprocessing import LabelEncoder
+from sklearn.metrics import accuracy_score
+import numpy as np
+
+from sklearn.metrics import accuracy_score, hamming_loss, classification_report
+from sklearn.preprocessing import MultiLabelBinarizer
+from sklearn.svm import SVC
+from sklearn.multiclass import OneVsRestClassifier
+from joblib import dump
+from sklearn.ensemble import RandomForestClassifier
+from preprocessing import Preprocessing
+# class svm_classifier:
+#     # def __init__(self,path_to_data):
+#     #     self.data = path_to_data
+#     @staticmethod
+#     def run_classifier(novels_data):
+        # novels_data = pd.read_csv(self.data)
+novels_data = pd.read_csv('novels_data.csv')
+novels_data['title_author'] = novels_data['title'] + " by " + novels_data['author']
+novels_data['contents_preprocessed'] = novels_data['content_original'].apply(Preprocessing.preprocess_content)
+novels_data = novels_data.drop(['content_preprocessed'],axis=1)
+novels_data.to_csv('novels_preprocessed_data.csv', index=False)
+        # Initialize TF-IDF Vectorizer and Label Encoder
+tfidf_vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')
+        # label_encoder = LabelEncoder()
+print('tfidf_vectorization:',tfidf_vectorizer)
+        # Encode target variable and extract features
+        # y_encoded = label_encoder.fit_transform(novels_data['title_author'])
+X_tfidf = tfidf_vectorizer.fit_transform(novels_data['contents_preprocessed'])
+print('X_tdidf:',X_tfidf)
+
+        # Assuming 'topic' and 'author' are columns in your dataset
+y = novels_data[['title', 'author']]
+
+        # Use MultiLabelBinarizer for multi-label encoding
+mlb = MultiLabelBinarizer()
+y_encoded = mlb.fit_transform(y.values)
+
+# Initialize the Random Forest classifier
+rf_classifier = OneVsRestClassifier(RandomForestClassifier(n_estimators=100, random_state=42))
+
+# Split data into training and test sets
+X_train, X_test, y_train, y_test = train_test_split(X_tfidf, y_encoded, test_size=0.2, random_state=42)
+
+# Train the classifier
+rf_classifier.fit(X_train, y_train)
+
+# Predict and calculate accuracy
+y_pred_rf = rf_classifier.predict(X_test)
+accuracy_rf = accuracy_score(y_test, y_pred_rf)
+hamming = hamming_loss(y_test, y_pred_rf)
+report = classification_report(y_test, y_pred_rf, target_names=mlb.classes_)
+
+print("Random Forest Accuracy:", accuracy_rf)
+print("Hamming Loss: ", hamming)
+print("Classification Report:\n", report)
+
+# Save the classifier, vectorizer, and label binarizer
+dump(rf_classifier, 'rf_classifier.joblib')
+dump(tfidf_vectorizer, 'tfidf_vectorizer.joblib')
+dump(mlb, 'mlb.joblib')

src/Dockerfile

@@ -0,0 +1,19 @@
+# Use Python 3.10 as base image
+FROM python:3.10
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the project files
+COPY . /app
+
+COPY nltk_data /root/nltk_data
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose port 2000
+EXPOSE 2000
+
+# Run the Flask app
+CMD ["python", "final_code.py"]

src/Project_code.py

@@ -0,0 +1,285 @@
+import pandas as pd
+import numpy as np
+from flask import Flask, request, jsonify, render_template,session
+from flask_session import Session
+import time
+import sqlite3
+from RetreivingDocument import RetrievingDocument, DocumentProcessor
+from preprocessing import Preprocessing
+import traceback
+import requests
+from RAG import RAG
+from joblib import load
+import logging
+
+
+def is_title(label, titles_list):
+    """
+    Determine if a label represents a book title.
+
+    Args:
+    label (str): The label to be checked.
+    titles_list (list): A list of all possible titles.
+
+    Returns:
+    bool: True if the label is a title, False otherwise.
+    """
+    return label in titles_list
+
+def is_author(label, authors_list):
+    """
+    Determine if a label represents an author's name.
+
+    Args:
+    label (str): The label to be checked.
+    authors_list (list): A list of all possible authors.
+
+    Returns:
+    bool: True if the label is an author's name, False otherwise.
+    """
+    return label in authors_list
+def init_db():
+    conn = sqlite3.connect('chatbot.db')
+    c = conn.cursor()
+    c.execute('''
+        CREATE TABLE IF NOT EXISTS chat_logs (
+            id INTEGER PRIMARY KEY, 
+            user_query TEXT, 
+            bot_response TEXT, 
+            query_type TEXT, 
+            response_time REAL, 
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+    ''')
+    conn.commit()
+    conn.close()
+
+init_db()
+print('Flask App Starting')
+app = Flask(__name__)
+
+# Configure logging
+logging.basicConfig(filename='chatbot_error.log', level=logging.ERROR,
+                    format='%(asctime)s:%(levelname)s:%(message)s')
+
+# Load BlenderBot for chit-chat
+# model_name = 'facebook/blenderbot-400M-distill'
+# tokenizer = BlenderbotTokenizer.from_pretrained(model_name)
+# model = BlenderbotForConditionalGeneration.from_pretrained(model_name)
+
+# Load zero-shot classifier
+#classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
+
+# Load models and other resources outside of the route
+rf_classifier = load('rf_classifier.joblib')
+tfidf_vectorizer = load('tfidf_vectorizer.joblib')
+mlb = load('mlb.joblib')
+novels_data = pd.read_csv('novels_preprocessed_data.csv')
+doc_processor = DocumentProcessor(novels_data,tfidf_vectorizer)
+titles_list = novels_data['title'].unique().tolist()
+authors_list = novels_data['author'].unique().tolist()
+# print("Titles List Sample:", titles_list[:5])  # Print first 5 titles
+# print("Authors List Sample:", authors_list[:5])
+# print("Is Title Test:", is_title(titles_list[0], titles_list))
+# print("Is Author Test:", is_author(authors_list[0], authors_list))
+
+def query_blenderbot_via_api(prompt):
+    api_url = "https://api-inference.huggingface.co/models/facebook/blenderbot-400M-distill"
+    headers = {"Authorization": "Bearer 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"}
+
+    response = requests.post(api_url, headers=headers, json={"inputs": prompt})
+    if response.status_code == 200:
+        return response.json()['generated_text']
+    else:
+        raise Exception(f"API request failed with status code {response.status_code}")
+    
+def query_zero_shot_classification(prompt, candidate_labels):
+    api_url = "https://api-inference.huggingface.co/models/facebook/bart-large-mnli"
+    headers = {"Authorization": "Bearer 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"}
+
+    payload = {
+        "inputs": prompt,
+        "parameters": {"candidate_labels": candidate_labels}
+    }
+
+    response = requests.post(api_url, headers=headers, json=payload)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        raise Exception(f"API request failed with status code {response.status_code}")
+
+
+
+class NoLabelsException(Exception):
+    """Exception raised when no labels are returned by the classifier."""
+    pass
+
+@app.route('/chat', methods=['POST'])
+def chat():
+    start_time = time.time()  # Start time
+
+    try:    
+        user_input = request.json.get('query', '')
+        categories = ["Chit-chat", "Novel-related"] 
+        processed_input = Preprocessing.preprocess_query(user_input) 
+        # Call the zero-shot classification API
+        results = query_zero_shot_classification(user_input, categories)
+        # print(results)
+        novel_related_score = results['scores'][results['labels'].index("Novel-related")]
+        total_retrieved=0
+
+        if novel_related_score >= 0.4:
+            query_type = "Novel-related"            
+            selected_topics = request.json.get('topics', [])  # Expecting a list of topics
+            document_ids = novels_data['Document ID'].tolist()
+          #  processed_input = Preprocessing.preprocess_query(user_input)
+            # print("Processed Query:", processed_input)
+            tfidf_matrix = tfidf_vectorizer.transform(novels_data['contents_preprocessed'])
+            transformed_input = tfidf_vectorizer.transform([processed_input])
+            # print("Transformed Input Shape:", transformed_input.shape)
+            features_name = tfidf_vectorizer.get_feature_names_out()
+            inverted_index = RetrievingDocument.build_inverted_index(tfidf_matrix,features_name,document_ids)
+
+            if selected_topics:
+                top_k_per_topic = 5
+                total_limit = 5
+                relevant_documents, total_retrieved = retrieve_balanced_documents(
+                    selected_topics, doc_processor, processed_input, novels_data,
+                    tfidf_vectorizer, inverted_index, top_k_per_topic, total_limit)
+                # print(f"Total number of documents retrieved: {total_retrieved}")
+                # print('Response from Topics:',relevant_documents)
+                response = RAG.get_answer(relevant_documents, user_input)
+                # print('After RAG:',response)
+                # final_response = f"Response based on selected topics:{', '.join(selected_topics)}:\n\n{response}"
+                final_response = f"{response}"
+
+            else:
+                binary_predictions = rf_classifier.predict(tfidf_vectorizer.transform([processed_input]))
+                model_output = mlb.inverse_transform(binary_predictions)[0]
+                if not model_output:
+                    raise NoLabelsException("Classifier returned no labels.")
+                # print("Classifier Output:", model_output)
+                predicted_title = next((label for label in model_output if is_title(label, titles_list)), None)
+                predicted_author = next((label for label in model_output if is_author(label, authors_list)), None)
+                # print('Title:',predicted_title)
+                # print('Author: ',predicted_author)
+                top_documents, total_retrieved = RetrievingDocument.retrieve_documents_for_novel(
+                processed_input, doc_processor, predicted_title, predicted_author, 
+                novels_data, tfidf_vectorizer, inverted_index, top_k=5)
+                # print(f'Documents Retrieved: {top_documents}, Total Retrieved: {total_retrieved}')
+                response = RAG.get_answer(top_documents, user_input)
+                # print('After RAG: ',response)
+                # final_response = f"Title: {predicted_title}, Author: {predicted_author}. \n{response}"
+                final_response = f"{response}"
+        else:
+            query_type = "Chit-chat"
+            # Chit-chat response
+            # inputs = tokenizer([user_input], return_tensors='pt')
+            # reply_ids = model.generate(**inputs, max_length=1000, num_beams=5, temperature=0.7)
+            # final_response = tokenizer.decode(reply_ids[0], skip_special_tokens=True)
+            final_response = query_blenderbot_via_api(user_input)
+
+        end_time = time.time()
+        response_time = end_time - start_time 
+        # Database insertion now includes the total_retrieved column
+        conn = sqlite3.connect('chatbot.db')
+        c = conn.cursor()
+        c.execute('''
+            INSERT INTO chat_logs (user_query, bot_response, query_type, response_time, total_retrieved) 
+            VALUES (?, ?, ?, ?, ?)
+        ''', (user_input, final_response, query_type, response_time, total_retrieved))
+        conn.commit()
+        conn.close()      
+
+        return jsonify({'response': final_response})
+    except NoLabelsException as e:
+        log_error(e)
+        return jsonify({'response': 'No relevant labels found. Would you like to try another question or exit?', 'error': True}), 500
+    except Exception as e:
+        log_error(e)
+        return jsonify({'response': 'Something went wrong. Would you like to try another question or exit?', 'error': True}), 500
+@app.route('/get-topics', methods=['GET'])
+def get_topics():
+    topics = novels_data['title'].unique().tolist()# Example topics
+    return jsonify(topics)
+@app.route('/logs', methods=['GET'])
+def get_logs():
+    conn = sqlite3.connect('chatbot.db')
+    c = conn.cursor()
+    c.execute("SELECT * FROM chat_logs")
+    logs = c.fetchall()
+    conn.close()
+    return jsonify({'logs': logs})
+
+
+
+# def update_db():
+#     conn = sqlite3.connect('chatbot.db')
+#     c = conn.cursor()
+#     c.execute('ALTER TABLE chat_logs ADD COLUMN total_retrieved INTEGER DEFAULT 0')
+#     conn.commit()
+#     conn.close()
+
+# # Call this function once to update your database
+# update_db()
+
+
+def log_error(e):
+    logging.error(str(e))
+    traceback.print_exc()
+
+def retrieve_balanced_documents(selected_topics, doc_processor, processed_input, novels_data, tfidf_vectorizer, inverted_index, top_k_per_topic, total_limit):
+    topic_pools = {
+        topic: RetrievingDocument.retrieve_documents_for_topic(topic, doc_processor, processed_input, novels_data, tfidf_vectorizer, inverted_index, top_k=top_k_per_topic) 
+        for topic in selected_topics
+    }
+    
+    balanced_documents = []
+    total_retrieved_count = 0  # Initialize a counter for the total retrieved documents
+    
+    # Iterate over the topics to accumulate the documents and the total count
+    for topic, (documents, count) in topic_pools.items():
+        # print(f"Topic: {topic}, Documents Retrieved: {len(documents)}, Total Count: {count}")
+        total_retrieved_count += count  # Add the count from each topic
+        for doc in documents:
+            balanced_documents.append(doc)  # Add the document to the balanced documents
+            if len(balanced_documents) >= total_limit:
+                break
+        if len(balanced_documents) >= total_limit:
+            break
+
+    print(f"Total Retrieved Count across all topics: {total_retrieved_count}")
+    return balanced_documents, total_retrieved_count
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/visualization')
+def visualization():
+    return render_template('i.html')
+
+@app.route('/data')
+def data():
+    conn = sqlite3.connect('chatbot.db')
+    cur = conn.cursor()
+    
+    cur.execute("SELECT query_type, COUNT(*) as count FROM chat_logs GROUP BY query_type")
+    query_types_data = cur.fetchall()
+    
+    cur.execute("SELECT id, response_time FROM chat_logs ORDER BY id")
+    response_times_data = cur.fetchall()
+    
+    cur.execute("SELECT id, total_retrieved FROM chat_logs ORDER BY id")
+    total_retrieved_data = cur.fetchall()
+    
+    conn.close()    
+    query_types = [{'query_type': qt[0], 'count': qt[1]} for qt in query_types_data]
+    response_times = [{'query_id': rt[0], 'response_time': rt[1]} for rt in response_times_data]
+    total_retrieved = [{'query_id': tr[0], 'total_retrieved': tr[1]} for tr in total_retrieved_data]
+    
+    return jsonify({'query_types': query_types, 'response_times': response_times, 'total_retrieved': total_retrieved})
+
+if __name__ == '__main__':
+    app.run(debug=False)
+# response = qa_bot(user_input_query, rf_classifier, retrieval_index, top_k=3) inde

src/RAG.py

@@ -0,0 +1,48 @@
+import requests
+import dotenv
+import os
+dotenv.load_dotenv()
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+class RAG:
+    @staticmethod
+    def get_answer(context, query):
+        # Construct the prompt
+        prompt = f"{context}\nQuestion: {query}\nAnswer:"
+
+        # Set up the API parameters for the chat model endpoint
+        api_url = "https://api.openai.com/v1/chat/completions"  # Endpoint for chat models
+        headers = {
+            "Authorization": f"Bearer {OPENAI_API_KEY}"
+        }
+        data = {
+            "model": "gpt-3.5-turbo",  # Specify the model you are using
+            "messages": [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": prompt}
+            ],
+            "max_tokens": 150,
+            "temperature": 0.7,
+            "stop": ["\n", "Question:"]
+        }
+
+        # Make the API request
+        response = requests.post(api_url, headers=headers, json=data)
+
+        # Check for successful response
+        if response.status_code != 200:
+            print(f"Error: {response.status_code}, {response.text}")
+            return ""
+
+        # Parse the response
+        result = response.json()
+
+        # Extract the answer
+        try:
+            answer = result['choices'][0]['message']['content'].strip() if result.get('choices') else ""
+        except KeyError as e:
+            print(f"KeyError: {e}, Response: {response.text}")
+            answer = ""
+
+        return answer
+
+

src/RetreivingDocument.py

@@ -0,0 +1,142 @@
+from collections import defaultdict
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+class DocumentProcessor:
+
+    def __init__(self, novels_data, tfidf_vectorizer):
+        self.novels_data = novels_data
+        self.tfidf_vectorizer = tfidf_vectorizer
+        self.tfidf_vectors = {}  # Dictionary to store TF-IDF vectors
+        self.preprocess_documents()
+
+    def preprocess_documents(self):
+        # Compute TF-IDF matrix
+        tfidf_matrix = self.tfidf_vectorizer.fit_transform(self.novels_data['contents_preprocessed'])
+        # Store each document's TF-IDF vector in the dictionary
+        for doc_id, vector in zip(self.novels_data['Document ID'], tfidf_matrix):
+            self.tfidf_vectors[doc_id] = vector
+
+    def get_tfidf_vector(self, doc_id):
+        # Retrieve the precomputed TF-IDF vector for a given document
+        return self.tfidf_vectors.get(doc_id, None)
+
+# Initialize the processor with your novels data
+#document_processor = DocumentProcessor(novels_data)
+
+class RetrievingDocument:
+
+    @staticmethod
+    def build_inverted_index(tfidf_matrix, feature_names, document_ids):
+        inverted_index = defaultdict(list)
+        for doc_idx, doc in enumerate(tfidf_matrix):
+            doc_id = document_ids[doc_idx]  # Get the document ID for the current document
+            for word_idx in doc.indices:
+                word = feature_names[word_idx]
+                inverted_index[word].append(doc_id)  # Append the document ID instead of the index
+        return inverted_index
+
+    @staticmethod
+    def rerank_documents(filtered_data, top_indices, additional_criteria):
+        reranked_indices = sorted(top_indices, key=lambda idx: additional_criteria[idx], reverse=True)
+        return reranked_indices
+    @staticmethod
+    def calculate_content_score(document, query_terms):
+        score = 0
+        for term in query_terms:
+            score += document.count(term)
+        return score
+    @staticmethod
+    def retrieve_documents_for_novel(processed_query,doc_processor,predicted_topic, predicted_author, novels_data, tfidf_vectorizer, inverted_index, top_k=5):
+     #   predicted_topic, predicted_author = model_output
+        #print('Predicted_topic:',predicted_topic)
+       # print('Predicted_author:',predicted_author)
+        # Use inverted index for initial retrieval
+        query_terms = processed_query.split()
+      #  print('Query terms:',query_terms)
+        relevant_doc_ids = set()
+        for term in query_terms:
+            if term in inverted_index:
+                relevant_doc_ids.update(inverted_index[term])
+      #  print('Relevant Docs:',relevant_doc_ids)
+
+        if not relevant_doc_ids:
+            return []
+
+        # Filter the dataset based on relevant document IDs
+        filtered_data = novels_data[novels_data['Document ID'].isin(relevant_doc_ids)]
+
+        # Modify this line to use an OR condition
+        filtered_data = filtered_data[
+            (filtered_data['title'] == predicted_topic) | 
+            (filtered_data['author'] == predicted_author)
+        ]
+
+        if filtered_data.empty:
+            return []
+
+        # Calculate cosine similarities for the filtered data
+        query_tfidf = doc_processor.tfidf_vectorizer.transform([processed_query])
+        cos_similarities = []
+        for doc_id in filtered_data['Document ID']:
+            doc_tfidf_vector = doc_processor.get_tfidf_vector(doc_id)
+            if doc_tfidf_vector is not None:
+                similarity = cosine_similarity(query_tfidf, doc_tfidf_vector)
+                cos_similarities.append(similarity[0][0])
+            else:
+                cos_similarities.append(0)
+       # print('Cosine Similarity:',cos_similarities)
+       # print(filtered_data.columns)
+        # Get the top k indices with the highest similarity scores
+        filtered_data['content_score'] = [RetrievingDocument.calculate_content_score(doc, query_terms) for doc in filtered_data['contents_preprocessed']]
+        
+        # Combine cosine similarity with content score for re-ranking
+        filtered_data['combined_score'] = cos_similarities + filtered_data['content_score']
+        top_indices = np.argsort(filtered_data['combined_score'])[-top_k:][::-1]
+        top_documents = filtered_data.iloc[top_indices]['content_original']
+        # print('Top Documents:', top_documents)
+        return top_documents.tolist(), len(filtered_data)
+
+       # return top_documents.tolist()
+    @staticmethod
+    def retrieve_documents_for_topic(selected_topic,doc_processor, processed_query, novels_data, tfidf_vectorizer, inverted_index, top_k=5):
+    # Use inverted index for initial retrieval
+        query_terms = processed_query.split()
+        relevant_doc_ids = set()
+        for term in query_terms:
+            if term in inverted_index:
+                relevant_doc_ids.update(inverted_index[term])
+
+        if not relevant_doc_ids:
+            return []
+
+        # Filter the dataset based on relevant document IDs
+        filtered_data = novels_data[novels_data['Document ID'].isin(relevant_doc_ids)]
+
+        # Filter the data further based on the selected topic
+        filtered_data = filtered_data[filtered_data['title'] == selected_topic]
+
+        if filtered_data.empty:
+            return []
+
+        # Calculate cosine similarities for the filtered data
+        query_tfidf = doc_processor.tfidf_vectorizer.transform([processed_query])
+        cos_similarities = []
+        for doc_id in filtered_data['Document ID']:
+            doc_tfidf_vector = doc_processor.get_tfidf_vector(doc_id)
+            if doc_tfidf_vector is not None:
+                similarity = cosine_similarity(query_tfidf, doc_tfidf_vector)
+                cos_similarities.append(similarity[0][0])
+            else:
+                cos_similarities.append(0)
+       # print('Cosine Similarity:',cos_similarities)
+       # print(filtered_data.columns)
+        # Get the top k indices with the highest similarity scores
+        filtered_data['content_score'] = [RetrievingDocument.calculate_content_score(doc, query_terms) for doc in filtered_data['contents_preprocessed']]
+        
+        # Combine cosine similarity with content score for re-ranking
+        filtered_data['combined_score'] = cos_similarities + filtered_data['content_score']
+        top_indices = np.argsort(filtered_data['combined_score'])[-top_k:][::-1]
+        top_documents = filtered_data.iloc[top_indices]['content_original']
+
+        return top_documents.tolist(), len(filtered_data)

src/download_nltk.py

@@ -0,0 +1,4 @@
+import nltk
+nltk.download('punkt')
+nltk.download('stopwords')
+nltk.download('wordnet')

src/final_code.py

@@ -0,0 +1,316 @@
+import os
+import dotenv
+import pandas as pd
+import numpy as np
+from flask import Flask, request, jsonify, render_template, session
+from flask_session import Session
+import time
+import sqlite3
+from RetreivingDocument import RetrievingDocument, DocumentProcessor
+from preprocessing import Preprocessing
+import traceback
+import requests
+from RAG import RAG
+from joblib import load
+import logging
+
+# Load environment variables
+dotenv.load_dotenv()
+
+HUGGINGFACE_API_KEY = os.getenv("HUGGINGFACE_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+def is_title(label, titles_list):
+    """
+    Determine if a label represents a book title.
+
+    Args:
+    label (str): The label to be checked.
+    titles_list (list): A list of all possible titles.
+
+    Returns:
+    bool: True if the label is a title, False otherwise.
+    """
+    return label in titles_list
+
+def is_author(label, authors_list):
+    """
+    Determine if a label represents an author's name.
+
+    Args:
+    label (str): The label to be checked.
+    authors_list (list): A list of all possible authors.
+
+    Returns:
+    bool: True if the label is an author's name, False otherwise.
+    """
+    return label in authors_list
+def init_db():
+    conn = sqlite3.connect('chatbot.db')
+    c = conn.cursor()
+    c.execute('''
+        CREATE TABLE IF NOT EXISTS chat_logs (
+            id INTEGER PRIMARY KEY, 
+            user_query TEXT, 
+            bot_response TEXT, 
+            query_type TEXT, 
+            response_time REAL, 
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+    ''')
+    conn.commit()
+    conn.close()
+
+init_db()
+print('Flask App Starting')
+app = Flask(__name__)
+
+# Configure logging
+logging.basicConfig(filename='chatbot_error.log', level=logging.ERROR,
+                    format='%(asctime)s:%(levelname)s:%(message)s')
+
+# Load BlenderBot for chit-chat
+# model_name = 'facebook/blenderbot-400M-distill'
+# tokenizer = BlenderbotTokenizer.from_pretrained(model_name)
+# model = BlenderbotForConditionalGeneration.from_pretrained(model_name)
+
+# Load zero-shot classifier
+#classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
+
+# Load models and other resources outside of the route
+rf_classifier = load('rf_classifier.joblib')
+tfidf_vectorizer = load('tfidf_vectorizer.joblib')
+mlb = load('mlb.joblib')
+novels_data = pd.read_csv('novels_preprocessed_data.csv')
+doc_processor = DocumentProcessor(novels_data,tfidf_vectorizer)
+titles_list = novels_data['title'].unique().tolist()
+authors_list = novels_data['author'].unique().tolist()
+# print("Titles List Sample:", titles_list[:5])  # Print first 5 titles
+# print("Authors List Sample:", authors_list[:5])
+# print("Is Title Test:", is_title(titles_list[0], titles_list))
+# print("Is Author Test:", is_author(authors_list[0], authors_list))
+
+import time
+
+def query_blenderbot_via_api(prompt, max_retries=5, wait_time=30):
+    api_url = "https://api-inference.huggingface.co/models/facebook/blenderbot-400M-distill"
+    headers = {"Authorization": f"Bearer {HUGGINGFACE_API_KEY}"}
+
+    for attempt in range(max_retries):
+        response = requests.post(api_url, headers=headers, json={"inputs": prompt})
+        response_data = response.json()
+
+        # If the model is still loading, wait and retry
+        if response.status_code == 503 and isinstance(response_data, dict) and "error" in response_data and "loading" in response_data["error"]:
+            estimated_time = response_data.get("estimated_time", wait_time)
+            print(f"Model is still loading. Retrying in {estimated_time:.2f} seconds...")
+            time.sleep(estimated_time)
+        else:
+            break  # Exit loop if the request is successful or another error occurs
+
+    if response.status_code == 200:
+        # **Fix: Handle both dictionary and list responses**
+        if isinstance(response_data, list) and len(response_data) > 0:
+            return response_data[0].get('generated_text', 'Error: No response text found.')
+        elif isinstance(response_data, dict):
+            return response_data.get('generated_text', 'Error: No response text found.')
+        else:
+            return "Error: Unexpected API response format."
+    else:
+        raise Exception(f"API request failed with status code {response.status_code}: {response_data}")
+
+
+    
+def query_zero_shot_classification(prompt, candidate_labels):
+    api_url = "https://api-inference.huggingface.co/models/facebook/bart-large-mnli"
+    headers = {"Authorization": f"Bearer {HUGGINGFACE_API_KEY}"}
+
+    payload = {
+        "inputs": prompt,
+        "parameters": {"candidate_labels": candidate_labels}
+    }
+
+    response = requests.post(api_url, headers=headers, json=payload)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        raise Exception(f"API request failed with status code {response.status_code}")
+
+
+
+class NoLabelsException(Exception):
+    """Exception raised when no labels are returned by the classifier."""
+    pass
+
+@app.route('/chat', methods=['POST'])
+def chat():
+    start_time = time.time()  # Start time
+
+    try:    
+        user_input = request.json.get('query', '')
+        categories = ["Chit-chat", "Novel-related"] 
+        processed_input = Preprocessing.preprocess_query(user_input) 
+        # Call the zero-shot classification API
+        results = query_zero_shot_classification(user_input, categories)
+        # print(results)
+        novel_related_score = results['scores'][results['labels'].index("Novel-related")]
+        total_retrieved=0
+
+        if novel_related_score >= 0.4:
+            query_type = "Novel-related"            
+            selected_topics = request.json.get('topics', [])  # Expecting a list of topics
+            document_ids = novels_data['Document ID'].tolist()
+          #  processed_input = Preprocessing.preprocess_query(user_input)
+            # print("Processed Query:", processed_input)
+            tfidf_matrix = tfidf_vectorizer.transform(novels_data['contents_preprocessed'])
+            transformed_input = tfidf_vectorizer.transform([processed_input])
+            # print("Transformed Input Shape:", transformed_input.shape)
+            features_name = tfidf_vectorizer.get_feature_names_out()
+            inverted_index = RetrievingDocument.build_inverted_index(tfidf_matrix,features_name,document_ids)
+
+            if selected_topics:
+                top_k_per_topic = 5
+                total_limit = 5
+                relevant_documents, total_retrieved = retrieve_balanced_documents(
+                    selected_topics, doc_processor, processed_input, novels_data,
+                    tfidf_vectorizer, inverted_index, top_k_per_topic, total_limit)
+                # print(f"Total number of documents retrieved: {total_retrieved}")
+                # print('Response from Topics:',relevant_documents)
+                response = RAG.get_answer(relevant_documents, user_input)
+                print('Response:',response)
+                # print('After RAG:',response)
+                # final_response = f"Response based on selected topics:{', '.join(selected_topics)}:\n\n{response}"
+                final_response = f"{response}"
+
+            else:
+                binary_predictions = rf_classifier.predict(tfidf_vectorizer.transform([processed_input]))
+                model_output = mlb.inverse_transform(binary_predictions)[0]
+                if not model_output:
+                    raise NoLabelsException("Classifier returned no labels.")
+                # print("Classifier Output:", model_output)
+                predicted_title = next((label for label in model_output if is_title(label, titles_list)), None)
+                predicted_author = next((label for label in model_output if is_author(label, authors_list)), None)
+                # print('Title:',predicted_title)
+                # print('Author: ',predicted_author)
+                top_documents, total_retrieved = RetrievingDocument.retrieve_documents_for_novel(
+                processed_input, doc_processor, predicted_title, predicted_author, 
+                novels_data, tfidf_vectorizer, inverted_index, top_k=5)
+                # print(f'Documents Retrieved: {top_documents}, Total Retrieved: {total_retrieved}')
+                response = RAG.get_answer(top_documents, user_input)
+                # print('After RAG: ',response)
+                # final_response = f"Title: {predicted_title}, Author: {predicted_author}. \n{response}"
+                final_response = f"{response}"
+        else:
+            query_type = "Chit-chat"
+            # Chit-chat response
+            # inputs = tokenizer([user_input], return_tensors='pt')
+            # reply_ids = model.generate(**inputs, max_length=1000, num_beams=5, temperature=0.7)
+            # final_response = tokenizer.decode(reply_ids[0], skip_special_tokens=True)
+            final_response = query_blenderbot_via_api(user_input)
+
+        end_time = time.time()
+        response_time = end_time - start_time 
+        # Database insertion now includes the total_retrieved column
+        conn = sqlite3.connect('chatbot.db')
+        c = conn.cursor()
+        c.execute('''
+            INSERT INTO chat_logs (user_query, bot_response, query_type, response_time, total_retrieved) 
+            VALUES (?, ?, ?, ?, ?)
+        ''', (user_input, final_response, query_type, response_time, total_retrieved))
+        conn.commit()
+        conn.close()      
+
+        return jsonify({'response': final_response})
+    except NoLabelsException as e:
+        log_error(e)
+        return jsonify({'response': 'No relevant labels found. Would you like to try another question or exit?', 'error': True}), 500
+    except Exception as e:
+        log_error(e)
+        return jsonify({'response': 'Something went wrong. Would you like to try another question or exit?', 'error': True}), 500
+@app.route('/get-topics', methods=['GET'])
+def get_topics():
+    topics = novels_data['title'].unique().tolist()# Example topics
+    return jsonify(topics)
+@app.route('/logs', methods=['GET'])
+def get_logs():
+    conn = sqlite3.connect('chatbot.db')
+    c = conn.cursor()
+    c.execute("SELECT * FROM chat_logs")
+    logs = c.fetchall()
+    conn.close()
+    return jsonify({'logs': logs})
+
+
+
+# def update_db():
+#     conn = sqlite3.connect('chatbot.db')
+#     c = conn.cursor()
+#     c.execute('ALTER TABLE chat_logs ADD COLUMN total_retrieved INTEGER DEFAULT 0')
+#     conn.commit()
+#     conn.close()
+
+# # Call this function once to update your database
+# update_db()
+
+
+def log_error(e):
+    logging.error(str(e))
+    traceback.print_exc()
+
+def retrieve_balanced_documents(selected_topics, doc_processor, processed_input, novels_data, tfidf_vectorizer, inverted_index, top_k_per_topic, total_limit):
+    topic_pools = {
+        topic: RetrievingDocument.retrieve_documents_for_topic(topic, doc_processor, processed_input, novels_data, tfidf_vectorizer, inverted_index, top_k=top_k_per_topic) 
+        for topic in selected_topics
+    }
+    
+    balanced_documents = []
+    total_retrieved_count = 0  # Initialize a counter for the total retrieved documents
+    print(f"Selected Topics: {selected_topics}")
+    print(f"Topic Pools Retrieved: {topic_pools}")
+    
+    # Iterate over the topics to accumulate the documents and the total count
+    for topic, (documents, count) in topic_pools.items():
+        # print(f"Topic: {topic}, Documents Retrieved: {len(documents)}, Total Count: {count}")
+        total_retrieved_count += count  # Add the count from each topic
+        for doc in documents:
+            balanced_documents.append(doc)  # Add the document to the balanced documents
+            if len(balanced_documents) >= total_limit:
+                break
+        if len(balanced_documents) >= total_limit:
+            break
+
+    print(f"Total Retrieved Count across all topics: {total_retrieved_count}")
+    return balanced_documents, total_retrieved_count
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/visualization')
+def visualization():
+    return render_template('i.html')
+
+@app.route('/data')
+def data():
+    conn = sqlite3.connect('chatbot.db')
+    cur = conn.cursor()
+    
+    cur.execute("SELECT query_type, COUNT(*) as count FROM chat_logs GROUP BY query_type")
+    query_types_data = cur.fetchall()
+    
+    cur.execute("SELECT id, response_time FROM chat_logs ORDER BY id")
+    response_times_data = cur.fetchall()
+    
+    cur.execute("SELECT id, total_retrieved FROM chat_logs ORDER BY id")
+    total_retrieved_data = cur.fetchall()
+    
+    conn.close()    
+    query_types = [{'query_type': qt[0], 'count': qt[1]} for qt in query_types_data]
+    response_times = [{'query_id': rt[0], 'response_time': rt[1]} for rt in response_times_data]
+    total_retrieved = [{'query_id': tr[0], 'total_retrieved': tr[1]} for tr in total_retrieved_data]
+    
+    return jsonify({'query_types': query_types, 'response_times': response_times, 'total_retrieved': total_retrieved})
+port = int(os.environ.get("PORT", 2000))
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=port, debug=False)  
+

src/mlb.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f612247686caf8d821b6a5984c6682da1109fdfa6f4adbdd0d06d4e5a1ced3
+size 1053

src/nltk_data/corpora/stopwords.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15c94179887425ca1bedc265608cab9f27d650211f709bb929e320990a4b01d1
+size 34276

src/nltk_data/corpora/stopwords/README

@@ -0,0 +1,32 @@
+Stopwords Corpus
+
+This corpus contains lists of stop words for several languages.  These
+are high-frequency grammatical words which are usually ignored in text
+retrieval applications.
+
+They were obtained from:
+http://anoncvs.postgresql.org/cvsweb.cgi/pgsql/src/backend/snowball/stopwords/
+
+The stop words for the Romanian language were obtained from:
+http://arlc.ro/resources/
+
+The English list has been augmented
+https://github.com/nltk/nltk_data/issues/22
+
+The German list has been corrected
+https://github.com/nltk/nltk_data/pull/49
+
+A Kazakh list has been added
+https://github.com/nltk/nltk_data/pull/52
+
+A Nepali list has been added
+https://github.com/nltk/nltk_data/pull/83
+
+An Azerbaijani list has been added
+https://github.com/nltk/nltk_data/pull/100
+
+A Greek list has been added
+https://github.com/nltk/nltk_data/pull/103
+
+An Indonesian list has been added
+https://github.com/nltk/nltk_data/pull/112

src/nltk_data/corpora/stopwords/arabic

@@ -0,0 +1,754 @@
+إذ
+إذا
+إذما
+إذن
+أف
+أقل
+أكثر
+ألا
+إلا
+التي
+الذي
+الذين
+اللاتي
+اللائي
+اللتان
+اللتيا
+اللتين
+اللذان
+اللذين
+اللواتي
+إلى
+إليك
+إليكم
+إليكما
+إليكن
+أم
+أما
+أما
+إما
+أن
+إن
+إنا
+أنا
+أنت
+أنتم
+أنتما
+أنتن
+إنما
+إنه
+أنى
+أنى
+آه
+آها
+أو
+أولاء
+أولئك
+أوه
+آي
+أي
+أيها
+إي
+أين
+أين
+أينما
+إيه
+بخ
+بس
+بعد
+بعض
+بك
+بكم
+بكم
+بكما
+بكن
+بل
+بلى
+بما
+بماذا
+بمن
+بنا
+به
+بها
+بهم
+بهما
+بهن
+بي
+بين
+بيد
+تلك
+تلكم
+تلكما
+ته
+تي
+تين
+تينك
+ثم
+ثمة
+حاشا
+حبذا
+حتى
+حيث
+حيثما
+حين
+خلا
+دون
+ذا
+ذات
+ذاك
+ذان
+ذانك
+ذلك
+ذلكم
+ذلكما
+ذلكن
+ذه
+ذو
+ذوا
+ذواتا
+ذواتي
+ذي
+ذين
+ذينك
+ريث
+سوف
+سوى
+شتان
+عدا
+عسى
+عل
+على
+عليك
+عليه
+عما
+عن
+عند
+غير
+فإذا
+فإن
+فلا
+فمن
+في
+فيم
+فيما
+فيه
+فيها
+قد
+كأن
+كأنما
+كأي
+كأين
+كذا
+كذلك
+كل
+كلا
+كلاهما
+كلتا
+كلما
+كليكما
+كليهما
+كم
+كم
+كما
+كي
+كيت
+كيف
+كيفما
+لا
+لاسيما
+لدى
+لست
+لستم
+لستما
+لستن
+لسن
+لسنا
+لعل
+لك
+لكم
+لكما
+لكن
+لكنما
+لكي
+لكيلا
+لم
+لما
+لن
+لنا
+له
+لها
+لهم
+لهما
+لهن
+لو
+لولا
+لوما
+لي
+لئن
+ليت
+ليس
+ليسا
+ليست
+ليستا
+ليسوا
+ما
+ماذا
+متى
+مذ
+مع
+مما
+ممن
+من
+منه
+منها
+منذ
+مه
+مهما
+نحن
+نحو
+نعم
+ها
+هاتان
+هاته
+هاتي
+هاتين
+هاك
+هاهنا
+هذا
+هذان
+هذه
+هذي
+هذين
+هكذا
+هل
+هلا
+هم
+هما
+هن
+هنا
+هناك
+هنالك
+هو
+هؤلاء
+هي
+هيا
+هيت
+هيهات
+والذي
+والذين
+وإذ
+وإذا
+وإن
+ولا
+ولكن
+ولو
+وما
+ومن
+وهو
+يا
+أبٌ
+أخٌ
+حمٌ
+فو
+أنتِ
+يناير
+فبراير
+مارس
+أبريل
+مايو
+يونيو
+يوليو
+أغسطس
+سبتمبر
+أكتوبر
+نوفمبر
+ديسمبر
+جانفي
+فيفري
+مارس
+أفريل
+ماي
+جوان
+جويلية
+أوت
+كانون
+شباط
+آذار
+نيسان
+أيار
+حزيران
+تموز
+آب
+أيلول
+تشرين
+دولار
+دينار
+ريال
+درهم
+ليرة
+جنيه
+قرش
+مليم
+فلس
+هللة
+سنتيم
+يورو
+ين
+يوان
+شيكل
+واحد
+اثنان
+ثلاثة
+أربعة
+خمسة
+ستة
+سبعة
+ثمانية
+تسعة
+عشرة
+أحد
+اثنا
+اثني
+إحدى
+ثلاث
+أربع
+خمس
+ست
+سبع
+ثماني
+تسع
+عشر
+ثمان
+سبت
+أحد
+اثنين
+ثلاثاء
+أربعاء
+خميس
+جمعة
+أول
+ثان
+ثاني
+ثالث
+رابع
+خامس
+سادس
+سابع
+ثامن
+تاسع
+عاشر
+حادي
+أ
+ب
+ت
+ث
+ج
+ح
+خ
+د
+ذ
+ر
+ز
+س
+ش
+ص
+ض
+ط
+ظ
+ع
+غ
+ف
+ق
+ك
+ل
+م
+ن
+ه
+و
+ي
+ء
+ى
+آ
+ؤ
+ئ
+أ
+ة
+ألف
+باء
+تاء
+ثاء
+جيم
+حاء
+خاء
+دال
+ذال
+راء
+زاي
+سين
+شين
+صاد
+ضاد
+طاء
+ظاء
+عين
+غين
+فاء
+قاف
+كاف
+لام
+ميم
+نون
+هاء
+واو
+ياء
+همزة
+ي
+نا
+ك
+كن
+ه
+إياه
+إياها
+إياهما
+إياهم
+إياهن
+إياك
+إياكما
+إياكم
+إياك
+إياكن
+إياي
+إيانا
+أولالك
+تانِ
+تانِك
+تِه
+تِي
+تَيْنِ
+ثمّ
+ثمّة
+ذانِ
+ذِه
+ذِي
+ذَيْنِ
+هَؤلاء
+هَاتانِ
+هَاتِه
+هَاتِي
+هَاتَيْنِ
+هَذا
+هَذانِ
+هَذِه
+هَذِي
+هَذَيْنِ
+الألى
+الألاء
+أل
+أنّى
+أيّ
+ّأيّان
+أنّى
+أيّ
+ّأيّان
+ذيت
+كأيّ
+كأيّن
+بضع
+فلان
+وا
+آمينَ
+آهِ
+آهٍ
+آهاً
+أُفٍّ
+أُفٍّ
+أفٍّ
+أمامك
+أمامكَ
+أوّهْ
+إلَيْكَ
+إلَيْكَ
+إليكَ
+إليكنّ
+إيهٍ
+بخٍ
+بسّ
+بَسْ
+بطآن
+بَلْهَ
+حاي
+حَذارِ
+حيَّ
+حيَّ
+دونك
+رويدك
+سرعان
+شتانَ
+شَتَّانَ
+صهْ
+صهٍ
+طاق
+طَق
+عَدَسْ
+كِخ
+مكانَك
+مكانَك
+مكانَك
+مكانكم
+مكانكما
+مكانكنّ
+نَخْ
+هاكَ
+هَجْ
+هلم
+هيّا
+هَيْهات
+وا
+واهاً
+وراءَك
+وُشْكَانَ
+وَيْ
+يفعلان
+تفعلان
+يفعلون
+تفعلون
+تفعلين
+اتخذ
+ألفى
+تخذ
+ترك
+تعلَّم
+جعل
+حجا
+حبيب
+خال
+حسب
+خال
+درى
+رأى
+زعم
+صبر
+ظنَّ
+عدَّ
+علم
+غادر
+ذهب
+وجد
+ورد
+وهب
+أسكن
+أطعم
+أعطى
+رزق
+زود
+سقى
+كسا
+أخبر
+أرى
+أعلم
+أنبأ
+حدَث
+خبَّر
+نبَّا
+أفعل به
+ما أفعله
+بئس
+ساء
+طالما
+قلما
+لات
+لكنَّ
+ءَ
+أجل
+إذاً
+أمّا
+إمّا
+إنَّ
+أنًّ
+أى
+إى
+أيا
+ب
+ثمَّ
+جلل
+جير
+رُبَّ
+س
+علًّ
+ف
+كأنّ
+كلَّا
+كى
+ل
+لات
+لعلَّ
+لكنَّ
+لكنَّ
+م
+نَّ
+هلّا
+وا
+أل
+إلّا
+ت
+ك
+لمّا
+ن
+ه
+و
+ا
+ي
+تجاه
+تلقاء
+جميع
+حسب
+سبحان
+شبه
+لعمر
+مثل
+معاذ
+أبو
+أخو
+حمو
+فو
+مئة
+مئتان
+ثلاثمئة
+أربعمئة
+خمسمئة
+ستمئة
+سبعمئة
+ثمنمئة
+تسعمئة
+مائة
+ثلاثمائة
+أربعمائة
+خمسمائة
+ستمائة
+سبعمائة
+ثمانمئة
+تسعمائة
+عشرون
+ثلاثون
+اربعون
+خمسون
+ستون
+سبعون
+ثمانون
+تسعون
+عشرين
+ثلاثين
+اربعين
+خمسين
+ستين
+سبعين
+ثمانين
+تسعين
+بضع
+نيف
+أجمع
+جميع
+عامة
+عين
+نفس
+لا سيما
+أصلا
+أهلا
+أيضا
+بؤسا
+بعدا
+بغتة
+تعسا
+حقا
+حمدا
+خلافا
+خاصة
+دواليك
+سحقا
+سرا
+سمعا
+صبرا
+صدقا
+صراحة
+طرا
+عجبا
+عيانا
+غالبا
+فرادى
+فضلا
+قاطبة
+كثيرا
+لبيك
+معاذ
+أبدا
+إزاء
+أصلا
+الآن
+أمد
+أمس
+آنفا
+آناء
+أنّى
+أول
+أيّان
+تارة
+ثمّ
+ثمّة
+حقا
+صباح
+مساء
+ضحوة
+عوض
+غدا
+غداة
+قطّ
+كلّما
+لدن
+لمّا
+مرّة
+قبل
+خلف
+أمام
+فوق
+تحت
+يمين
+شمال
+ارتدّ
+استحال
+أصبح
+أضحى
+آض
+أمسى
+انقلب
+بات
+تبدّل
+تحوّل
+حار
+رجع
+راح
+صار
+ظلّ
+عاد
+غدا
+كان
+ما انفك
+ما برح
+مادام
+مازال
+مافتئ
+ابتدأ
+أخذ
+اخلولق
+أقبل
+انبرى
+أنشأ
+أوشك
+جعل
+حرى
+شرع
+طفق
+علق
+قام
+كرب
+كاد
+هبّ

src/nltk_data/corpora/stopwords/azerbaijani

@@ -0,0 +1,165 @@
+a
+ad
+altı
+altmış
+amma
+arasında
+artıq
+ay
+az
+bax
+belə
+bəli
+bəlkə
+beş
+bəy
+bəzən
+bəzi
+bilər
+bir
+biraz
+biri
+birşey
+biz
+bizim
+bizlər
+bu
+buna
+bundan
+bunların
+bunu
+bunun
+buradan
+bütün
+ci
+cı
+çox
+cu
+cü
+çünki
+da
+daha
+də
+dedi
+dək
+dən
+dəqiqə
+deyil
+dir
+doqquz
+doqsan
+dörd
+düz
+ə
+edən
+edir
+əgər
+əlbəttə
+elə
+əlli
+ən
+əslində
+et
+etdi
+etmə
+etmək
+faiz
+gilə
+görə
+ha
+haqqında
+harada
+hə
+heç
+həm
+həmin
+həmişə
+hər
+ı
+idi
+iki
+il
+ildə
+ilə
+ilk
+in
+indi
+isə
+istifadə
+iyirmi
+ki
+kim
+kimə
+kimi
+lakin
+lap
+məhz
+mən
+mənə
+mirşey
+nə
+nəhayət
+niyə
+o
+obirisi
+of
+olan
+olar
+olaraq
+oldu
+olduğu
+olmadı
+olmaz
+olmuşdur
+olsun
+olur
+on
+ona
+ondan
+onlar
+onlardan
+onların 
+onsuzda
+onu
+onun
+oradan
+otuz
+öz
+özü
+qarşı
+qədər
+qırx
+saat
+sadəcə
+saniyə
+səhv
+səkkiz
+səksən
+sən
+sənə
+sənin
+siz
+sizin
+sizlər
+sonra
+təəssüf
+ü
+üç
+üçün
+var
+və
+xan
+xanım
+xeyr
+ya
+yalnız
+yaxşı
+yeddi
+yenə
+yəni
+yetmiş
+yox
+yoxdur
+yoxsa
+yüz
+zaman

src/nltk_data/corpora/stopwords/basque

@@ -0,0 +1,326 @@
+ahala
+aitzitik
+al
+ala 
+alabadere
+alabaina
+alabaina
+aldiz 
+alta
+amaitu
+amaitzeko
+anitz
+antzina
+arabera
+arabera
+arabera
+argi
+arratsaldero
+arte
+artean
+asko
+aspaldiko
+aurrera
+aurrera
+azkenez
+azkenik
+azkenik
+ba
+bada
+bada 
+bada 
+bada 
+badarik
+badarik
+badarik 
+badere
+bai
+baina
+baina
+baina 
+baino
+baino
+baino
+baino
+baita
+baizik 
+baldin
+baldin
+barren
+bat
+batean
+batean
+batean
+batean
+batek
+baten
+batera
+batez
+bati
+batzuei
+batzuek
+batzuetan
+batzuk
+bazen
+bederen
+bederik
+beharrez
+behiala
+behin
+behin
+behin
+behin
+behinik
+behinola
+behintzat
+bera
+beraiek
+beranduago
+berau
+berauek
+beraz
+beraz 
+bere
+berean
+berebat
+berehala
+berori
+beroriek
+berriro
+berriz
+bertzalde
+bertzenaz
+bestalde
+beste
+bestela
+besterik
+bezain
+bezala
+bide
+bien
+bigarrenez
+bigarrenik
+bitartean
+bitartean
+bizkitartean
+bukaeran
+bukatzeko 
+da
+dago
+dago
+dela
+dela
+dela
+delarik
+den
+dena
+dena
+dezadan
+dira
+ditu
+du
+dute
+edo
+edo 
+edota 
+egin
+egin
+egun
+egun
+egunean
+emateko
+era
+erdi
+ere
+ere
+ere
+ere
+ere 
+esan
+esan
+esanak
+esandakoaren
+eta
+eta
+eta
+eta
+eta 
+eta 
+eurak
+ez
+ez 
+ez 
+eze 
+ezen
+ezer
+ezezik
+ezik
+ezpabere 
+ezpada 
+ezpere
+ezperen 
+ezta
+funtsean
+gabe
+gain
+gainera 
+gainera 	
+gainerontzean 
+gaur
+gero
+gero
+gero
+geroago
+gisa
+gu
+gutxi
+guzti
+guztia
+guztiz 
+haatik
+haiei
+haiek
+haietan
+hain
+hainbeste
+hainbestez 
+hala
+hala
+hala
+halaber
+halako
+halatan
+han
+handik
+hango
+hara
+hargatik
+hari
+hark
+hartan
+hartan
+hasi
+hasi
+hasiera
+hasieran
+hasteaz
+hasteko
+hasteko
+hau
+hau
+hau
+hau
+hau
+hau
+hauei
+hauek
+hauetan
+hemen
+hemendik
+hemengo
+hi
+hona
+honebestez
+honek
+honela
+honela
+honela
+honen
+honen
+honetan
+honetaz
+honi
+hor
+hori
+hori
+hori
+horiei
+horiek
+horietan
+horko
+horra 	
+horratik
+horregatik
+horregatik
+horrek
+horrela
+horrela
+horrela
+horren
+horrenbestez
+horretan
+horri
+hortaz
+hortaz
+hortik
+hura
+ikusi
+ikusi
+izan
+izan
+izan
+jarraituz
+kariaz
+kasuaz
+kontuan
+laburbilduz
+laburki
+laster
+laster
+lehen
+lehen
+lehen
+lehen
+lehenengo
+lehenengo
+lehenik
+lehen-lehenik
+litzateke
+medio
+mendean
+mundura
+nahiz
+ni
+noiz
+nola
+non
+nondik
+nongo
+nor
+nora
+on
+ondoren
+ondorio
+ondorioz
+ondorioz
+orain
+ordea 
+orduan
+orduan
+orduan 
+orduko
+ordura
+orobat
+ostean
+ostera
+osterantzean 
+pentsatuz
+ustez
+ze
+zein
+zein 
+zen
+zen
+zenbait
+zenbat
+zer
+zeren
+zergatik
+zergatik
+ziren
+zituen
+zu
+zuek
+zuen
+zuten
+zuzen

src/nltk_data/corpora/stopwords/bengali

@@ -0,0 +1,398 @@
+অতএব
+অথচ
+অথবা
+অনুযায়ী
+অনেক
+অনেকে
+অনেকেই
+অন্তত
+অন্য
+অবধি
+অবশ্য
+অর্থাত
+আই
+আগামী
+আগে
+আগেই
+আছে
+আজ
+আদ্যভাগে
+আপনার
+আপনি
+আবার
+আমরা
+আমাকে
+আমাদের
+আমার
+আমি
+আর
+আরও
+ই
+ইত্যাদি
+ইহা
+উচিত
+উত্তর
+উনি
+উপর
+উপরে
+এ
+এঁদের
+এঁরা
+এই
+একই
+একটি
+একবার
+একে
+এক্
+এখন
+এখনও
+এখানে
+এখানেই
+এটা
+এটাই
+এটি
+এত
+এতটাই
+এতে
+এদের
+এব
+এবং
+এবার
+এমন
+এমনকী
+এমনি
+এর
+এরা
+এল
+এস
+এসে
+ঐ
+ও
+ওঁদের
+ওঁর
+ওঁরা
+ওই
+ওকে
+ওখানে
+ওদের
+ওর
+ওরা
+কখনও
+কত
+কবে
+কমনে
+কয়েক
+কয়েকটি
+করছে
+করছেন
+করতে
+করবে
+করবেন
+করলে
+করলেন
+করা
+করাই
+করায়
+করার
+করি
+করিতে
+করিয়া
+করিয়ে
+করে
+করেই
+করেছিলেন
+করেছে
+করেছেন
+করেন
+কাউকে
+কাছ
+কাছে
+কাজ
+কাজে
+কারও
+কারণ
+কি
+কিংবা
+কিছু
+কিছুই
+কিন্তু
+কী
+কে
+কেউ
+কেউই
+কেখা
+কেন
+কোটি
+কোন
+কোনও
+কোনো
+ক্ষেত্রে
+কয়েক
+খুব
+গিয়ে
+গিয়েছে
+গিয়ে
+গুলি
+গেছে
+গেল
+গেলে
+গোটা
+চলে
+চান
+চায়
+চার
+চালু
+চেয়ে
+চেষ্টা
+ছাড়া
+ছাড়াও
+ছিল
+ছিলেন
+জন
+জনকে
+জনের
+জন্য
+জন্যওজে
+জানতে
+জানা
+জানানো
+জানায়
+জানিয়ে
+জানিয়েছে
+জে
+জ্নজন
+টি
+ঠিক
+তখন
+তত
+তথা
+তবু
+তবে
+তা
+তাঁকে
+তাঁদের
+তাঁর
+তাঁরা
+তাঁাহারা
+তাই
+তাও
+তাকে
+তাতে
+তাদের
+তার
+তারপর
+তারা
+তারৈ
+তাহলে
+তাহা
+তাহাতে
+তাহার
+তিনঐ
+তিনি
+তিনিও
+তুমি
+তুলে
+তেমন
+তো
+তোমার
+থাকবে
+থাকবেন
+থাকা
+থাকায়
+থাকে
+থাকেন
+থেকে
+থেকেই
+থেকেও
+দিকে
+দিতে
+দিন
+দিয়ে
+দিয়েছে
+দিয়েছেন
+দিলেন
+দু
+দুই
+দুটি
+দুটো
+দেওয়া
+দেওয়ার
+দেওয়া
+দেখতে
+দেখা
+দেখে
+দেন
+দেয়
+দ্বারা
+ধরা
+ধরে
+ধামার
+নতুন
+নয়
+না
+নাই
+নাকি
+নাগাদ
+নানা
+নিজে
+নিজেই
+নিজেদের
+নিজের
+নিতে
+নিয়ে
+নিয়ে
+নেই
+নেওয়া
+নেওয়ার
+নেওয়া
+নয়
+পক্ষে
+পর
+পরে
+পরেই
+পরেও
+পর্যন্ত
+পাওয়া
+পাচ
+পারি
+পারে
+পারেন
+পি
+পেয়ে
+পেয়্র্
+প্রতি
+প্রথম
+প্রভৃতি
+প্রযন্ত
+প্রাথমিক
+প্রায়
+প্রায়
+ফলে
+ফিরে
+ফের
+বক্তব্য
+বদলে
+বন
+বরং
+বলতে
+বলল
+বললেন
+বলা
+বলে
+বলেছেন
+বলেন
+বসে
+বহু
+বা
+বাদে
+বার
+বি
+বিনা
+বিভিন্ন
+বিশেষ
+বিষয়টি
+বেশ
+বেশি
+ব্যবহার
+ব্যাপারে
+ভাবে
+ভাবেই
+মতো
+মতোই
+মধ্যভাগে
+মধ্যে
+মধ্যেই
+মধ্যেও
+মনে
+মাত্র
+মাধ্যমে
+মোট
+মোটেই
+যখন
+যত
+যতটা
+যথেষ্ট
+যদি
+যদিও
+যা
+যাঁর
+যাঁরা
+যাওয়া
+যাওয়ার
+যাওয়া
+যাকে
+যাচ্ছে
+যাতে
+যাদের
+যান
+যাবে
+যায়
+যার
+যারা
+যিনি
+যে
+যেখানে
+যেতে
+যেন
+যেমন
+র
+রকম
+রয়েছে
+রাখা
+রেখে
+লক্ষ
+শুধু
+শুরু
+সঙ্গে
+সঙ্গেও
+সব
+সবার
+স��স্ত
+সম্প্রতি
+সহ
+সহিত
+সাধারণ
+সামনে
+সি
+সুতরাং
+সে
+সেই
+সেখান
+সেখানে
+সেটা
+সেটাই
+সেটাও
+সেটি
+স্পষ্ট
+স্বয়ং
+হইতে
+হইবে
+হইয়া
+হওয়া
+হওয়ায়
+হওয়ার
+হচ্ছে
+হত
+হতে
+হতেই
+হন
+হবে
+হবেন
+হয়
+হয়তো
+হয়নি
+হয়ে
+হয়েই
+হয়েছিল
+হয়েছে
+হয়েছেন
+হল
+হলে
+হলেই
+হলেও
+হলো
+হাজার
+হিসাবে
+হৈলে
+হোক
+হয়

src/nltk_data/corpora/stopwords/catalan

@@ -0,0 +1,278 @@
+a
+abans
+ací
+ah
+així
+això
+al
+aleshores
+algun
+alguna
+algunes
+alguns
+alhora
+allà
+allí
+allò
+als
+altra
+altre
+altres
+amb
+ambdues
+ambdós
+anar
+ans
+apa
+aquell
+aquella
+aquelles
+aquells
+aquest
+aquesta
+aquestes
+aquests
+aquí
+baix
+bastant
+bé
+cada
+cadascuna
+cadascunes
+cadascuns
+cadascú
+com
+consegueixo
+conseguim
+conseguir
+consigueix
+consigueixen
+consigueixes
+contra
+d'un
+d'una
+d'unes
+d'uns
+dalt
+de
+del
+dels
+des
+des de
+després
+dins
+dintre
+donat
+doncs
+durant
+e
+eh
+el
+elles
+ells
+els
+em
+en
+encara
+ens
+entre
+era
+erem
+eren
+eres
+es
+esta
+estan
+estat
+estava
+estaven
+estem
+esteu
+estic
+està
+estàvem
+estàveu
+et
+etc
+ets
+fa
+faig
+fan
+fas
+fem
+fer
+feu
+fi
+fins
+fora
+gairebé
+ha
+han
+has
+haver
+havia
+he
+hem
+heu
+hi
+ho
+i
+igual
+iguals
+inclòs
+ja
+jo
+l'hi
+la
+les
+li
+li'n
+llarg
+llavors
+m'he
+ma
+mal
+malgrat
+mateix
+mateixa
+mateixes
+mateixos
+me
+mentre
+meu
+meus
+meva
+meves
+mode
+molt
+molta
+moltes
+molts
+mon
+mons
+més
+n'he
+n'hi
+ne
+ni
+no
+nogensmenys
+només
+nosaltres
+nostra
+nostre
+nostres
+o
+oh
+oi
+on
+pas
+pel
+pels
+per
+per que
+perquè
+però
+poc
+poca
+pocs
+podem
+poden
+poder
+podeu
+poques
+potser
+primer
+propi
+puc
+qual
+quals
+quan
+quant
+que
+quelcom
+qui
+quin
+quina
+quines
+quins
+què
+s'ha
+s'han
+sa
+sabem
+saben
+saber
+sabeu
+sap
+saps
+semblant
+semblants
+sense
+ser
+ses
+seu
+seus
+seva
+seves
+si
+sobre
+sobretot
+soc
+solament
+sols
+som
+son
+sons
+sota
+sou
+sóc
+són
+t'ha
+t'han
+t'he
+ta
+tal
+també
+tampoc
+tan
+tant
+tanta
+tantes
+te
+tene
+tenim
+tenir
+teniu
+teu
+teus
+teva
+teves
+tinc
+ton
+tons
+tot
+tota
+totes
+tots
+un
+una
+unes
+uns
+us
+va
+vaig
+vam
+van
+vas
+veu
+vosaltres
+vostra
+vostre
+vostres
+érem
+éreu
+és
+éssent
+últim
+ús

src/nltk_data/corpora/stopwords/chinese

@@ -0,0 +1,841 @@
+一
+一下
+一些
+一切
+一则
+一天
+一定
+一方面
+一旦
+一时
+一来
+一样
+一次
+一片
+一直
+一致
+一般
+一起
+一边
+一面
+万一
+上下
+上升
+上去
+上来
+上述
+上面
+下列
+下去
+下来
+下面
+不一
+不久
+不仅
+不会
+不但
+不光
+不单
+不变
+不只
+不可
+不同
+不够
+不如
+不得
+不怕
+不惟
+不成
+不拘
+不敢
+不断
+不是
+不比
+不然
+不特
+不独
+不管
+不能
+不要
+不论
+不足
+不过
+不问
+与
+与其
+与否
+与此同时
+专门
+且
+两者
+严格
+严重
+个
+个人
+个别
+中小
+中间
+丰富
+临
+为
+为主
+为了
+为什么
+为什麽
+为何
+为着
+主张
+主要
+举行
+乃
+乃至
+么
+之
+之一
+之前
+之后
+之後
+之所以
+之类
+乌乎
+乎
+乘
+也
+也好
+也是
+也罢
+了
+了解
+争取
+于
+于是
+于是乎
+云云
+互相
+产生
+人们
+人家
+什么
+什么样
+什麽
+今后
+今天
+今年
+今後
+仍然
+从
+从事
+从而
+他
+他人
+他们
+他的
+代替
+以
+以上
+以下
+以为
+以便
+以免
+以前
+以及
+以后
+以外
+以後
+以来
+以至
+以至于
+以致
+们
+任
+任何
+任凭
+任务
+企图
+伟大
+似乎
+似的
+但
+但是
+何
+何况
+何处
+何时
+作为
+你
+你们
+你的
+使得
+使用
+例如
+依
+依照
+依靠
+促进
+保持
+俺
+俺们
+倘
+倘使
+倘或
+倘然
+倘若
+假使
+假如
+假若
+做到
+像
+允许
+充分
+先后
+先後
+先生
+全部
+全面
+兮
+共同
+关于
+其
+其一
+其中
+其二
+其他
+其余
+其它
+其实
+其次
+具体
+具体地说
+具体说来
+具有
+再者
+再说
+冒
+冲
+决定
+况且
+准备
+几
+几乎
+几时
+凭
+凭借
+出去
+出来
+出现
+分别
+则
+别
+别的
+别说
+到
+前后
+前者
+前进
+前面
+加之
+加以
+加入
+加强
+十分
+即
+即令
+即使
+即便
+即或
+即若
+却不
+原来
+又
+及
+及其
+及时
+及至
+双方
+反之
+反应
+反映
+反过来
+反过来说
+取得
+受到
+变成
+另
+另一方面
+另外
+只是
+只有
+只要
+只限
+叫
+叫做
+召开
+叮咚
+可
+可以
+可是
+可能
+可见
+各
+各个
+各人
+各位
+各地
+各种
+各级
+各自
+合理
+同
+同一
+同时
+同样
+后来
+后面
+向
+向着
+吓
+吗
+否则
+吧
+吧哒
+吱
+呀
+呃
+呕
+呗
+呜
+呜呼
+呢
+周围
+呵
+呸
+呼哧
+咋
+和
+咚
+咦
+咱
+咱们
+咳
+哇
+哈
+哈哈
+哉
+哎
+哎呀
+哎哟
+哗
+哟
+哦
+哩
+哪
+哪个
+哪些
+哪儿
+哪天
+哪年
+哪怕
+哪样
+哪边
+哪里
+哼
+哼唷
+唉
+啊
+啐
+啥
+啦
+啪达
+喂
+喏
+喔唷
+嗡嗡
+嗬
+嗯
+嗳
+嘎
+嘎登
+嘘
+嘛
+嘻
+嘿
+因
+因为
+因此
+因而
+固然
+在
+在下
+地
+坚决
+坚持
+基本
+处理
+复杂
+多
+多少
+多数
+多次
+大力
+大多数
+大大
+大家
+大批
+大约
+大量
+失去
+她
+她们
+她的
+好的
+好象
+如
+如上所述
+如下
+如何
+如其
+如果
+如此
+如若
+存在
+宁
+宁可
+宁愿
+宁肯
+它
+它们
+它们的
+它的
+安全
+完全
+完成
+实现
+实际
+宣布
+容易
+密切
+对
+对于
+对应
+将
+少数
+尔后
+尚且
+尤其
+就
+就是
+就是说
+尽
+尽管
+属于
+岂但
+左右
+巨大
+巩固
+己
+已经
+帮助
+常常
+并
+并不
+并不是
+并且
+并没有
+广大
+广泛
+应当
+应用
+应该
+开外
+开始
+开展
+引起
+强烈
+强调
+归
+当
+当前
+当时
+当然
+当着
+形成
+彻底
+彼
+彼此
+往
+往往
+待
+後来
+後面
+得
+得出
+得到
+心里
+必然
+必要
+必须
+怎
+怎么
+怎么办
+怎么样
+怎样
+怎麽
+总之
+总是
+总的来看
+总的来说
+总的说来
+总结
+总而言之
+恰恰相反
+您
+意思
+愿意
+慢说
+成为
+我
+我们
+我的
+或
+或是
+或者
+战斗
+所
+所以
+所有
+所谓
+打
+扩大
+把
+抑或
+拿
+按
+按照
+换句话说
+换言之
+据
+掌握
+接着
+接著
+故
+故此
+整个
+方便
+方面
+旁人
+无宁
+无法
+无论
+既
+既是
+既然
+时候
+明显
+明确
+是
+是否
+是的
+显然
+显著
+普通
+普遍
+更加
+曾经
+替
+最后
+最大
+最好
+最後
+最近
+最高
+有
+有些
+有关
+有利
+有力
+有所
+有效
+有时
+有点
+有的
+有着
+有著
+望
+朝
+朝着
+本
+本着
+来
+来着
+极了
+构成
+果然
+果真
+某
+某个
+某些
+根据
+根本
+欢迎
+正在
+正如
+正常
+此
+此外
+此时
+此间
+毋宁
+每
+每个
+每天
+每年
+每当
+比
+比如
+比方
+比较
+毫不
+没有
+沿
+沿着
+注意
+深入
+清楚
+满足
+漫说
+焉
+然则
+然后
+然後
+然而
+照
+照着
+特别是
+特殊
+特点
+现代
+现在
+甚么
+甚而
+甚至
+用
+由
+由于
+由此可见
+的
+的话
+目前
+直到
+直接
+相似
+相信
+相反
+相同
+相对
+相对而言
+相应
+相当
+相等
+省得
+看出
+看到
+看来
+看看
+看见
+真是
+真正
+着
+着呢
+矣
+知道
+确定
+离
+积极
+移动
+突出
+突然
+立即
+第
+等
+等等
+管
+紧接着
+纵
+纵令
+纵使
+纵然
+练习
+组成
+经
+经常
+经过
+结合
+结果
+给
+绝对
+继续
+继而
+维持
+综上所述
+罢了
+考虑
+者
+而
+而且
+而况
+而外
+而已
+而是
+而言
+联系
+能
+能否
+能够
+腾
+自
+自个儿
+自从
+自各儿
+自家
+自己
+自身
+至
+至于
+良好
+若
+若是
+若非
+范围
+莫若
+获得
+虽
+虽则
+虽然
+虽说
+行为
+行动
+表明
+表示
+被
+要
+要不
+要不是
+要不然
+要么
+要是
+要求
+规定
+觉得
+认为
+认真
+认识
+让
+许多
+论
+设使
+设若
+该
+说明
+诸位
+谁
+谁知
+赶
+起
+起来
+起见
+趁
+趁着
+越是
+跟
+转动
+转变
+转贴
+较
+较之
+边
+达到
+迅速
+过
+过去
+过来
+运用
+还是
+还有
+这
+这个
+这么
+这么些
+这么样
+这么点儿
+这些
+这会儿
+这儿
+这就是说
+这时
+这样
+这点
+这种
+这边
+这里
+这麽
+进入
+进步
+进而
+进行
+连
+连同
+适应
+适当
+适用
+逐步
+逐渐
+通常
+通过
+造成
+遇到
+遭到
+避免
+那
+那个
+那么
+那么些
+那么样
+那些
+那会儿
+那儿
+那时
+那样
+那边
+那里
+那麽
+部分
+鄙人
+采取
+里面
+重大
+重新
+重要
+鉴于
+问题
+防止
+阿
+附近
+限制
+除
+除了
+除此之外
+除非
+随
+随着
+随著
+集中
+需要
+非但
+非常
+非徒
+靠
+顺
+顺着
+首先
+高兴
+是不是

src/nltk_data/corpora/stopwords/danish

@@ -0,0 +1,94 @@
+og
+i
+jeg
+det
+at
+en
+den
+til
+er
+som
+på
+de
+med
+han
+af
+for
+ikke
+der
+var
+mig
+sig
+men
+et
+har
+om
+vi
+min
+havde
+ham
+hun
+nu
+over
+da
+fra
+du
+ud
+sin
+dem
+os
+op
+man
+hans
+hvor
+eller
+hvad
+skal
+selv
+her
+alle
+vil
+blev
+kunne
+ind
+når
+være
+dog
+noget
+ville
+jo
+deres
+efter
+ned
+skulle
+denne
+end
+dette
+mit
+også
+under
+have
+dig
+anden
+hende
+mine
+alt
+meget
+sit
+sine
+vor
+mod
+disse
+hvis
+din
+nogle
+hos
+blive
+mange
+ad
+bliver
+hendes
+været
+thi
+jer
+sådan

src/nltk_data/corpora/stopwords/dutch

@@ -0,0 +1,101 @@
+de
+en
+van
+ik
+te
+dat
+die
+in
+een
+hij
+het
+niet
+zijn
+is
+was
+op
+aan
+met
+als
+voor
+had
+er
+maar
+om
+hem
+dan
+zou
+of
+wat
+mijn
+men
+dit
+zo
+door
+over
+ze
+zich
+bij
+ook
+tot
+je
+mij
+uit
+der
+daar
+haar
+naar
+heb
+hoe
+heeft
+hebben
+deze
+u
+want
+nog
+zal
+me
+zij
+nu
+ge
+geen
+omdat
+iets
+worden
+toch
+al
+waren
+veel
+meer
+doen
+toen
+moet
+ben
+zonder
+kan
+hun
+dus
+alles
+onder
+ja
+eens
+hier
+wie
+werd
+altijd
+doch
+wordt
+wezen
+kunnen
+ons
+zelf
+tegen
+na
+reeds
+wil
+kon
+niets
+uw
+iemand
+geweest
+andere

src/nltk_data/corpora/stopwords/english

@@ -0,0 +1,179 @@
+i
+me
+my
+myself
+we
+our
+ours
+ourselves
+you
+you're
+you've
+you'll
+you'd
+your
+yours
+yourself
+yourselves
+he
+him
+his
+himself
+she
+she's
+her
+hers
+herself
+it
+it's
+its
+itself
+they
+them
+their
+theirs
+themselves
+what
+which
+who
+whom
+this
+that
+that'll
+these
+those
+am
+is
+are
+was
+were
+be
+been
+being
+have
+has
+had
+having
+do
+does
+did
+doing
+a
+an
+the
+and
+but
+if
+or
+because
+as
+until
+while
+of
+at
+by
+for
+with
+about
+against
+between
+into
+through
+during
+before
+after
+above
+below
+to
+from
+up
+down
+in
+out
+on
+off
+over
+under
+again
+further
+then
+once
+here
+there
+when
+where
+why
+how
+all
+any
+both
+each
+few
+more
+most
+other
+some
+such
+no
+nor
+not
+only
+own
+same
+so
+than
+too
+very
+s
+t
+can
+will
+just
+don
+don't
+should
+should've
+now
+d
+ll
+m
+o
+re
+ve
+y
+ain
+aren
+aren't
+couldn
+couldn't
+didn
+didn't
+doesn
+doesn't
+hadn
+hadn't
+hasn
+hasn't
+haven
+haven't
+isn
+isn't
+ma
+mightn
+mightn't
+mustn
+mustn't
+needn
+needn't
+shan
+shan't
+shouldn
+shouldn't
+wasn
+wasn't
+weren
+weren't
+won
+won't
+wouldn
+wouldn't

src/nltk_data/corpora/stopwords/finnish

@@ -0,0 +1,235 @@
+olla
+olen
+olet
+on
+olemme
+olette
+ovat
+ole
+oli
+olisi
+olisit
+olisin
+olisimme
+olisitte
+olisivat
+olit
+olin
+olimme
+olitte
+olivat
+ollut
+olleet
+en
+et
+ei
+emme
+ette
+eivät
+minä
+minun
+minut
+minua
+minussa
+minusta
+minuun
+minulla
+minulta
+minulle
+sinä
+sinun
+sinut
+sinua
+sinussa
+sinusta
+sinuun
+sinulla
+sinulta
+sinulle
+hän
+hänen
+hänet
+häntä
+hänessä
+hänestä
+häneen
+hänellä
+häneltä
+hänelle
+me
+meidän
+meidät
+meitä
+meissä
+meistä
+meihin
+meillä
+meiltä
+meille
+te
+teidän
+teidät
+teitä
+teissä
+teistä
+teihin
+teillä
+teiltä
+teille
+he
+heidän
+heidät
+heitä
+heissä
+heistä
+heihin
+heillä
+heiltä
+heille
+tämä
+tämän
+tätä
+tässä
+tästä
+tähän
+tallä
+tältä
+tälle
+tänä
+täksi
+tuo
+tuon
+tuotä
+tuossa
+tuosta
+tuohon
+tuolla
+tuolta
+tuolle
+tuona
+tuoksi
+se
+sen
+sitä
+siinä
+siitä
+siihen
+sillä
+siltä
+sille
+sinä
+siksi
+nämä
+näiden
+näitä
+näissä
+näistä
+näihin
+näillä
+näiltä
+näille
+näinä
+näiksi
+nuo
+noiden
+noita
+noissa
+noista
+noihin
+noilla
+noilta
+noille
+noina
+noiksi
+ne
+niiden
+niitä
+niissä
+niistä
+niihin
+niillä
+niiltä
+niille
+niinä
+niiksi
+kuka
+kenen
+kenet
+ketä
+kenessä
+kenestä
+keneen
+kenellä
+keneltä
+kenelle
+kenenä
+keneksi
+ketkä
+keiden
+ketkä
+keitä
+keissä
+keistä
+keihin
+keillä
+keiltä
+keille
+keinä
+keiksi
+mikä
+minkä
+minkä
+mitä
+missä
+mistä
+mihin
+millä
+miltä
+mille
+minä
+miksi
+mitkä
+joka
+jonka
+jota
+jossa
+josta
+johon
+jolla
+jolta
+jolle
+jona
+joksi
+jotka
+joiden
+joita
+joissa
+joista
+joihin
+joilla
+joilta
+joille
+joina
+joiksi
+että
+ja
+jos
+koska
+kuin
+mutta
+niin
+sekä
+sillä
+tai
+vaan
+vai
+vaikka
+kanssa
+mukaan
+noin
+poikki
+yli
+kun
+niin
+nyt
+itse

src/nltk_data/corpora/stopwords/french

@@ -0,0 +1,157 @@
+au
+aux
+avec
+ce
+ces
+dans
+de
+des
+du
+elle
+en
+et
+eux
+il
+ils
+je
+la
+le
+les
+leur
+lui
+ma
+mais
+me
+même
+mes
+moi
+mon
+ne
+nos
+notre
+nous
+on
+ou
+par
+pas
+pour
+qu
+que
+qui
+sa
+se
+ses
+son
+sur
+ta
+te
+tes
+toi
+ton
+tu
+un
+une
+vos
+votre
+vous
+c
+d
+j
+l
+à
+m
+n
+s
+t
+y
+été
+étée
+étées
+étés
+étant
+étante
+étants
+étantes
+suis
+es
+est
+sommes
+êtes
+sont
+serai
+seras
+sera
+serons
+serez
+seront
+serais
+serait
+serions
+seriez
+seraient
+étais
+était
+étions
+étiez
+étaient
+fus
+fut
+fûmes
+fûtes
+furent
+sois
+soit
+soyons
+soyez
+soient
+fusse
+fusses
+fût
+fussions
+fussiez
+fussent
+ayant
+ayante
+ayantes
+ayants
+eu
+eue
+eues
+eus
+ai
+as
+avons
+avez
+ont
+aurai
+auras
+aura
+aurons
+aurez
+auront
+aurais
+aurait
+aurions
+auriez
+auraient
+avais
+avait
+avions
+aviez
+avaient
+eut
+eûmes
+eûtes
+eurent
+aie
+aies
+ait
+ayons
+ayez
+aient
+eusse
+eusses
+eût
+eussions
+eussiez
+eussent

src/nltk_data/corpora/stopwords/german

@@ -0,0 +1,232 @@
+aber
+alle
+allem
+allen
+aller
+alles
+als
+also
+am
+an
+ander
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+auch
+auf
+aus
+bei
+bin
+bis
+bist
+da
+damit
+dann
+der
+den
+des
+dem
+die
+das
+dass
+daß
+derselbe
+derselben
+denselben
+desselben
+demselben
+dieselbe
+dieselben
+dasselbe
+dazu
+dein
+deine
+deinem
+deinen
+deiner
+deines
+denn
+derer
+dessen
+dich
+dir
+du
+dies
+diese
+diesem
+diesen
+dieser
+dieses
+doch
+dort
+durch
+ein
+eine
+einem
+einen
+einer
+eines
+einig
+einige
+einigem
+einigen
+einiger
+einiges
+einmal
+er
+ihn
+ihm
+es
+etwas
+euer
+eure
+eurem
+euren
+eurer
+eures
+für
+gegen
+gewesen
+hab
+habe
+haben
+hat
+hatte
+hatten
+hier
+hin
+hinter
+ich
+mich
+mir
+ihr
+ihre
+ihrem
+ihren
+ihrer
+ihres
+euch
+im
+in
+indem
+ins
+ist
+jede
+jedem
+jeden
+jeder
+jedes
+jene
+jenem
+jenen
+jener
+jenes
+jetzt
+kann
+kein
+keine
+keinem
+keinen
+keiner
+keines
+können
+könnte
+machen
+man
+manche
+manchem
+manchen
+mancher
+manches
+mein
+meine
+meinem
+meinen
+meiner
+meines
+mit
+muss
+musste
+nach
+nicht
+nichts
+noch
+nun
+nur
+ob
+oder
+ohne
+sehr
+sein
+seine
+seinem
+seinen
+seiner
+seines
+selbst
+sich
+sie
+ihnen
+sind
+so
+solche
+solchem
+solchen
+solcher
+solches
+soll
+sollte
+sondern
+sonst
+über
+um
+und
+uns
+unsere
+unserem
+unseren
+unser
+unseres
+unter
+viel
+vom
+von
+vor
+während
+war
+waren
+warst
+was
+weg
+weil
+weiter
+welche
+welchem
+welchen
+welcher
+welches
+wenn
+werde
+werden
+wie
+wieder
+will
+wir
+wird
+wirst
+wo
+wollen
+wollte
+würde
+würden
+zu
+zum
+zur
+zwar
+zwischen

src/nltk_data/corpora/stopwords/greek

@@ -0,0 +1,265 @@
+αλλα
+αν
+αντι
+απο
+αυτα
+αυτεσ
+αυτη
+αυτο
+αυτοι
+αυτοσ
+αυτουσ
+αυτων
+αἱ
+αἳ
+αἵ
+αὐτόσ
+αὐτὸς
+αὖ
+γάρ
+γα
+γα^
+γε
+για
+γοῦν
+γὰρ
+δ'
+δέ
+δή
+δαί
+δαίσ
+δαὶ
+δαὶς
+δε
+δεν
+δι'
+διά
+διὰ
+δὲ
+δὴ
+δ’
+εαν
+ειμαι
+ειμαστε
+ειναι
+εισαι
+ειστε
+εκεινα
+εκεινεσ
+εκεινη
+εκεινο
+εκεινοι
+εκεινοσ
+εκεινουσ
+εκεινων
+ενω
+επ
+επι
+εἰ
+εἰμί
+εἰμὶ
+εἰς
+εἰσ
+εἴ
+εἴμι
+εἴτε
+η
+θα
+ισωσ
+κ
+καί
+καίτοι
+καθ
+και
+κατ
+κατά
+κατα
+κατὰ
+καὶ
+κι
+κἀν
+κἂν
+μέν
+μή
+μήτε
+μα
+με
+μεθ
+μετ
+μετά
+μετα
+μετὰ
+μη
+μην
+μἐν
+μὲν
+μὴ
+μὴν
+να
+ο
+οι
+ομωσ
+οπωσ
+οσο
+οτι
+οἱ
+οἳ
+οἷς
+οὐ
+οὐδ
+οὐδέ
+οὐδείσ
+οὐδεὶς
+οὐδὲ
+οὐδὲν
+οὐκ
+οὐχ
+οὐχὶ
+οὓς
+οὔτε
+οὕτω
+οὕτως
+οὕτωσ
+οὖν
+οὗ
+οὗτος
+οὗτοσ
+παρ
+παρά
+παρα
+παρὰ
+περί
+περὶ
+ποια
+ποιεσ
+ποιο
+ποιοι
+ποιοσ
+ποιουσ
+ποιων
+ποτε
+που
+ποῦ
+προ
+προσ
+πρόσ
+πρὸ
+πρὸς
+πως
+πωσ
+σε
+στη
+στην
+στο
+στον
+σόσ
+σύ
+σύν
+σὸς
+σὺ
+σὺν
+τά
+τήν
+τί
+τίς
+τίσ
+τα
+ταῖς
+τε
+την
+τησ
+τι
+τινα
+τις
+τισ
+το
+τοί
+τοι
+τοιοῦτος
+τοιοῦτοσ
+τον
+τοτε
+του
+τούσ
+τοὺς
+τοῖς
+τοῦ
+των
+τό
+τόν
+τότε
+τὰ
+τὰς
+τὴν
+τὸ
+τὸν
+τῆς
+τῆσ
+τῇ
+τῶν
+τῷ
+ωσ
+ἀλλ'
+ἀλλά
+ἀλλὰ
+ἀλλ’
+ἀπ
+ἀπό
+ἀπὸ
+ἀφ
+ἂν
+ἃ
+ἄλλος
+ἄλλοσ
+ἄν
+ἄρα
+ἅμα
+ἐάν
+ἐγώ
+ἐγὼ
+ἐκ
+ἐμόσ
+ἐμὸς
+ἐν
+ἐξ
+ἐπί
+ἐπεὶ
+ἐπὶ
+ἐστι
+ἐφ
+ἐὰν
+ἑαυτοῦ
+ἔτι
+ἡ
+ἢ
+ἣ
+ἤ
+ἥ
+ἧς
+ἵνα
+ὁ
+ὃ
+ὃν
+ὃς
+ὅ
+ὅδε
+ὅθεν
+ὅπερ
+ὅς
+ὅσ
+ὅστις
+ὅστισ
+ὅτε
+ὅτι
+ὑμόσ
+ὑπ
+ὑπέρ
+ὑπό
+ὑπὲρ
+ὑπὸ
+ὡς
+ὡσ
+ὥς
+ὥστε
+ὦ
+ᾧ

src/nltk_data/corpora/stopwords/hebrew

@@ -0,0 +1,221 @@
+אני
+את
+אתה
+אנחנו
+אתן
+אתם
+הם
+הן
+היא
+הוא
+שלי
+שלו
+שלך
+שלה
+שלנו
+שלכם
+שלכן
+שלהם
+שלהן
+לי
+לו
+לה
+לנו
+לכם
+לכן
+להם
+להן
+אותה
+אותו
+זה
+זאת
+אלה
+אלו
+תחת
+מתחת
+מעל
+בין
+עם
+עד
+נגר
+על
+אל
+מול
+של
+אצל
+כמו
+אחר
+אותו
+בלי
+לפני
+אחרי
+מאחורי
+עלי
+עליו
+עליה
+עליך
+עלינו
+עליכם
+לעיכן
+עליהם
+עליהן
+כל
+כולם
+כולן
+כך
+ככה
+כזה
+זה
+זות
+אותי
+אותה
+אותם
+אותך
+אותו
+אותן
+אותנו
+ואת
+את
+אתכם
+אתכן
+איתי
+איתו
+איתך
+איתה
+איתם
+איתן
+איתנו
+איתכם
+איתכן
+יהיה
+תהיה
+היתי
+היתה
+היה
+להיות
+עצמי
+עצמו
+עצמה
+עצמם
+עצמן
+עצמנו
+עצמהם
+עצמהן
+מי
+מה
+איפה
+היכן
+במקום שבו
+אם
+לאן
+למקום שבו
+מקום בו
+איזה
+מהיכן
+איך
+כיצד
+באיזו מידה
+מתי
+בשעה ש
+כאשר
+כש
+למרות
+לפני
+אחרי
+מאיזו סיבה
+הסיבה שבגללה
+למה
+מדוע
+לאיזו תכלית
+כי
+יש
+אין
+אך
+מנין
+מאין
+מאיפה
+יכל
+יכלה
+יכלו
+יכול
+יכולה
+יכולים
+יכולות
+יוכלו
+יוכל
+מסוגל
+לא
+רק
+אולי
+אין
+לאו
+אי
+כלל
+נגד
+אם
+עם
+אל
+אלה
+אלו
+אף
+על
+מעל
+מתחת
+מצד
+בשביל
+לבין
+באמצע
+בתוך
+דרך
+מבעד
+באמצעות
+למעלה
+למטה
+מחוץ
+מן
+לעבר
+מכאן
+כאן
+הנה
+הרי
+פה
+שם
+אך
+ברם
+שוב
+אבל
+מבלי
+בלי
+מלבד
+רק
+בגלל
+מכיוון
+עד
+אשר
+ואילו
+למרות
+אס
+כמו
+כפי
+אז
+אחרי
+כן
+לכן
+לפיכך
+מאד
+עז
+מעט
+מעטים
+במידה
+שוב
+יותר
+מדי
+גם
+כן
+נו
+אחר
+אחרת
+אחרים
+אחרות
+אשר
+או

src/nltk_data/corpora/stopwords/hinglish

@@ -0,0 +1,1036 @@
+a
+aadi
+aaj
+aap
+aapne
+aata
+aati
+aaya
+aaye
+ab
+abbe
+abbey
+abe
+abhi
+able
+about
+above
+accha
+according
+accordingly
+acha
+achcha
+across
+actually
+after
+afterwards
+again
+against
+agar
+ain
+aint
+ain't
+aisa
+aise
+aisi
+alag
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+andar
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+ap
+apan
+apart
+apna
+apnaa
+apne
+apni
+appear
+are
+aren
+arent
+aren't
+around
+arre
+as
+aside
+ask
+asking
+at
+aur
+avum
+aya
+aye
+baad
+baar
+bad
+bahut
+bana
+banae
+banai
+banao
+banaya
+banaye
+banayi
+banda
+bande
+bandi
+bane
+bani
+bas
+bata
+batao
+bc
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+below
+beside
+besides
+best
+better
+between
+beyond
+bhai
+bheetar
+bhi
+bhitar
+bht
+bilkul
+bohot
+bol
+bola
+bole
+boli
+bolo
+bolta
+bolte
+bolti
+both
+brief
+bro
+btw
+but
+by
+came
+can
+cannot
+cant
+can't
+cause
+causes
+certain
+certainly
+chahiye
+chaiye
+chal
+chalega
+chhaiye
+clearly
+c'mon
+com
+come
+comes
+could
+couldn
+couldnt
+couldn't
+d
+de
+dede
+dega
+degi
+dekh
+dekha
+dekhe
+dekhi
+dekho
+denge
+dhang
+di
+did
+didn
+didnt
+didn't
+dijiye
+diya
+diyaa
+diye
+diyo
+do
+does
+doesn
+doesnt
+doesn't
+doing
+done
+dono
+dont
+don't
+doosra
+doosre
+down
+downwards
+dude
+dunga
+dungi
+during
+dusra
+dusre
+dusri
+dvaara
+dvara
+dwaara
+dwara
+each
+edu
+eg
+eight
+either
+ek
+else
+elsewhere
+enough
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+far
+few
+fifth
+fir
+first
+five
+followed
+following
+follows
+for
+forth
+four
+from
+further
+furthermore
+gaya
+gaye
+gayi
+get
+gets
+getting
+ghar
+given
+gives
+go
+goes
+going
+gone
+good
+got
+gotten
+greetings
+haan
+had
+hadd
+hadn
+hadnt
+hadn't
+hai
+hain
+hamara
+hamare
+hamari
+hamne
+han
+happens
+har
+hardly
+has
+hasn
+hasnt
+hasn't
+have
+haven
+havent
+haven't
+having
+he
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hm
+hmm
+ho
+hoga
+hoge
+hogi
+hona
+honaa
+hone
+honge
+hongi
+honi
+hopefully
+hota
+hotaa
+hote
+hoti
+how
+howbeit
+however
+hoyenge
+hoyengi
+hu
+hua
+hue
+huh
+hui
+hum
+humein
+humne
+hun
+huye
+huyi
+i
+i'd
+idk
+ie
+if
+i'll
+i'm
+imo
+in
+inasmuch
+inc
+inhe
+inhi
+inho
+inka
+inkaa
+inke
+inki
+inn
+inner
+inse
+insofar
+into
+inward
+is
+ise
+isi
+iska
+iskaa
+iske
+iski
+isme
+isn
+isne
+isnt
+isn't
+iss
+isse
+issi
+isski
+it
+it'd
+it'll
+itna
+itne
+itni
+itno
+its
+it's
+itself
+ityaadi
+ityadi
+i've
+ja
+jaa
+jab
+jabh
+jaha
+jahaan
+jahan
+jaisa
+jaise
+jaisi
+jata
+jayega
+jidhar
+jin
+jinhe
+jinhi
+jinho
+jinhone
+jinka
+jinke
+jinki
+jinn
+jis
+jise
+jiska
+jiske
+jiski
+jisme
+jiss
+jisse
+jitna
+jitne
+jitni
+jo
+just
+jyaada
+jyada
+k
+ka
+kaafi
+kab
+kabhi
+kafi
+kaha
+kahaa
+kahaan
+kahan
+kahi
+kahin
+kahte
+kaisa
+kaise
+kaisi
+kal
+kam
+kar
+kara
+kare
+karega
+karegi
+karen
+karenge
+kari
+karke
+karna
+karne
+karni
+karo
+karta
+karte
+karti
+karu
+karun
+karunga
+karungi
+kaun
+kaunsa
+kayi
+kch
+ke
+keep
+keeps
+keh
+kehte
+kept
+khud
+ki
+kin
+kine
+kinhe
+kinho
+kinka
+kinke
+kinki
+kinko
+kinn
+kino
+kis
+kise
+kisi
+kiska
+kiske
+kiski
+kisko
+kisliye
+kisne
+kitna
+kitne
+kitni
+kitno
+kiya
+kiye
+know
+known
+knows
+ko
+koi
+kon
+konsa
+koyi
+krna
+krne
+kuch
+kuchch
+kuchh
+kul
+kull
+kya
+kyaa
+kyu
+kyuki
+kyun
+kyunki
+lagta
+lagte
+lagti
+last
+lately
+later
+le
+least
+lekar
+lekin
+less
+lest
+let
+let's
+li
+like
+liked
+likely
+little
+liya
+liye
+ll
+lo
+log
+logon
+lol
+look
+looking
+looks
+ltd
+lunga
+m
+maan
+maana
+maane
+maani
+maano
+magar
+mai
+main
+maine
+mainly
+mana
+mane
+mani
+mano
+many
+mat
+may
+maybe
+me
+mean
+meanwhile
+mein
+mera
+mere
+merely
+meri
+might
+mightn
+mightnt
+mightn't
+mil
+mjhe
+more
+moreover
+most
+mostly
+much
+mujhe
+must
+mustn
+mustnt
+mustn't
+my
+myself
+na
+naa
+naah
+nahi
+nahin
+nai
+name
+namely
+nd
+ne
+near
+nearly
+necessary
+neeche
+need
+needn
+neednt
+needn't
+needs
+neither
+never
+nevertheless
+new
+next
+nhi
+nine
+no
+nobody
+non
+none
+noone
+nope
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+par
+pata
+pe
+pehla
+pehle
+pehli
+people
+per
+perhaps
+phla
+phle
+phli
+placed
+please
+plus
+poora
+poori
+provides
+pura
+puri
+q
+que
+quite
+raha
+rahaa
+rahe
+rahi
+rakh
+rakha
+rakhe
+rakhen
+rakhi
+rakho
+rather
+re
+really
+reasonably
+regarding
+regardless
+regards
+rehte
+rha
+rhaa
+rhe
+rhi
+ri
+right
+s
+sa
+saara
+saare
+saath
+sab
+sabhi
+sabse
+sahi
+said
+sakta
+saktaa
+sakte
+sakti
+same
+sang
+sara
+sath
+saw
+say
+saying
+says
+se
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan
+shant
+shan't
+she
+she's
+should
+shouldn
+shouldnt
+shouldn't
+should've
+si
+since
+six
+so
+soch
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+still
+sub
+such
+sup
+sure
+t
+tab
+tabh
+tak
+take
+taken
+tarah
+teen
+teeno
+teesra
+teesre
+teesri
+tell
+tends
+tera
+tere
+teri
+th
+tha
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that's
+the
+theek
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+theres
+there's
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+thi
+thik
+thing
+think
+thinking
+third
+this
+tho
+thoda
+thodi
+thorough
+thoroughly
+those
+though
+thought
+three
+through
+throughout
+thru
+thus
+tjhe
+to
+together
+toh
+too
+took
+toward
+towards
+tried
+tries
+true
+truly
+try
+trying
+tu
+tujhe
+tum
+tumhara
+tumhare
+tumhari
+tune
+twice
+two
+um
+umm
+un
+under
+unhe
+unhi
+unho
+unhone
+unka
+unkaa
+unke
+unki
+unko
+unless
+unlikely
+unn
+unse
+until
+unto
+up
+upar
+upon
+us
+use
+used
+useful
+uses
+usi
+using
+uska
+uske
+usne
+uss
+usse
+ussi
+usually
+vaala
+vaale
+vaali
+vahaan
+vahan
+vahi
+vahin
+vaisa
+vaise
+vaisi
+vala
+vale
+vali
+various
+ve
+very
+via
+viz
+vo
+waala
+waale
+waali
+wagaira
+wagairah
+wagerah
+waha
+wahaan
+wahan
+wahi
+wahin
+waisa
+waise
+waisi
+wala
+wale
+wali
+want
+wants
+was
+wasn
+wasnt
+wasn't
+way
+we
+we'd
+well
+we'll
+went
+were
+we're
+weren
+werent
+weren't
+we've
+what
+whatever
+what's
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+while
+who
+whoever
+whole
+whom
+who's
+whose
+why
+will
+willing
+with
+within
+without
+wo
+woh
+wohi
+won
+wont
+won't
+would
+wouldn
+wouldnt
+wouldn't
+y
+ya
+yadi
+yah
+yaha
+yahaan
+yahan
+yahi
+yahin
+ye
+yeah
+yeh
+yehi
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+yup

src/nltk_data/corpora/stopwords/hungarian

@@ -0,0 +1,199 @@
+a
+ahogy
+ahol
+aki
+akik
+akkor
+alatt
+által
+általában
+amely
+amelyek
+amelyekben
+amelyeket
+amelyet
+amelynek
+ami
+amit
+amolyan
+amíg
+amikor
+át
+abban
+ahhoz
+annak
+arra
+arról
+az
+azok
+azon
+azt
+azzal
+azért
+aztán
+azután
+azonban
+bár
+be
+belül
+benne
+cikk
+cikkek
+cikkeket
+csak
+de
+e
+eddig
+egész
+egy
+egyes
+egyetlen
+egyéb
+egyik
+egyre
+ekkor
+el
+elég
+ellen
+elõ
+elõször
+elõtt
+elsõ
+én
+éppen
+ebben
+ehhez
+emilyen
+ennek
+erre
+ez
+ezt
+ezek
+ezen
+ezzel
+ezért
+és
+fel
+felé
+hanem
+hiszen
+hogy
+hogyan
+igen
+így
+illetve
+ill.
+ill
+ilyen
+ilyenkor
+ison
+ismét
+itt
+jó
+jól
+jobban
+kell
+kellett
+keresztül
+keressünk
+ki
+kívül
+között
+közül
+legalább
+lehet
+lehetett
+legyen
+lenne
+lenni
+lesz
+lett
+maga
+magát
+majd
+majd
+már
+más
+másik
+meg
+még
+mellett
+mert
+mely
+melyek
+mi
+mit
+míg
+miért
+milyen
+mikor
+minden
+mindent
+mindenki
+mindig
+mint
+mintha
+mivel
+most
+nagy
+nagyobb
+nagyon
+ne
+néha
+nekem
+neki
+nem
+néhány
+nélkül
+nincs
+olyan
+ott
+össze
+õ
+õk
+õket
+pedig
+persze
+rá
+s
+saját
+sem
+semmi
+sok
+sokat
+sokkal
+számára
+szemben
+szerint
+szinte
+talán
+tehát
+teljes
+tovább
+továbbá
+több
+úgy
+ugyanis
+új
+újabb
+újra
+után
+utána
+utolsó
+vagy
+vagyis
+valaki
+valami
+valamint
+való
+vagyok
+van
+vannak
+volt
+voltam
+voltak
+voltunk
+vissza
+vele
+viszont
+volna

src/nltk_data/corpora/stopwords/indonesian

@@ -0,0 +1,758 @@
+ada
+adalah
+adanya
+adapun
+agak
+agaknya
+agar
+akan
+akankah
+akhir
+akhiri
+akhirnya
+aku
+akulah
+amat
+amatlah
+anda
+andalah
+antar
+antara
+antaranya
+apa
+apaan
+apabila
+apakah
+apalagi
+apatah
+artinya
+asal
+asalkan
+atas
+atau
+ataukah
+ataupun
+awal
+awalnya
+bagai
+bagaikan
+bagaimana
+bagaimanakah
+bagaimanapun
+bagi
+bagian
+bahkan
+bahwa
+bahwasanya
+baik
+bakal
+bakalan
+balik
+banyak
+bapak
+baru
+bawah
+beberapa
+begini
+beginian
+beginikah
+beginilah
+begitu
+begitukah
+begitulah
+begitupun
+bekerja
+belakang
+belakangan
+belum
+belumlah
+benar
+benarkah
+benarlah
+berada
+berakhir
+berakhirlah
+berakhirnya
+berapa
+berapakah
+berapalah
+berapapun
+berarti
+berawal
+berbagai
+berdatangan
+beri
+berikan
+berikut
+berikutnya
+berjumlah
+berkali-kali
+berkata
+berkehendak
+berkeinginan
+berkenaan
+berlainan
+berlalu
+berlangsung
+berlebihan
+bermacam
+bermacam-macam
+bermaksud
+bermula
+bersama
+bersama-sama
+bersiap
+bersiap-siap
+bertanya
+bertanya-tanya
+berturut
+berturut-turut
+bertutur
+berujar
+berupa
+besar
+betul
+betulkah
+biasa
+biasanya
+bila
+bilakah
+bisa
+bisakah
+boleh
+bolehkah
+bolehlah
+buat
+bukan
+bukankah
+bukanlah
+bukannya
+bulan
+bung
+cara
+caranya
+cukup
+cukupkah
+cukuplah
+cuma
+dahulu
+dalam
+dan
+dapat
+dari
+daripada
+datang
+dekat
+demi
+demikian
+demikianlah
+dengan
+depan
+di
+dia
+diakhiri
+diakhirinya
+dialah
+diantara
+diantaranya
+diberi
+diberikan
+diberikannya
+dibuat
+dibuatnya
+didapat
+didatangkan
+digunakan
+diibaratkan
+diibaratkannya
+diingat
+diingatkan
+diinginkan
+dijawab
+dijelaskan
+dijelaskannya
+dikarenakan
+dikatakan
+dikatakannya
+dikerjakan
+diketahui
+diketahuinya
+dikira
+dilakukan
+dilalui
+dilihat
+dimaksud
+dimaksudkan
+dimaksudkannya
+dimaksudnya
+diminta
+dimintai
+dimisalkan
+dimulai
+dimulailah
+dimulainya
+dimungkinkan
+dini
+dipastikan
+diperbuat
+diperbuatnya
+dipergunakan
+diperkirakan
+diperlihatkan
+diperlukan
+diperlukannya
+dipersoalkan
+dipertanyakan
+dipunyai
+diri
+dirinya
+disampaikan
+disebut
+disebutkan
+disebutkannya
+disini
+disinilah
+ditambahkan
+ditandaskan
+ditanya
+ditanyai
+ditanyakan
+ditegaskan
+ditujukan
+ditunjuk
+ditunjuki
+ditunjukkan
+ditunjukkannya
+ditunjuknya
+dituturkan
+dituturkannya
+diucapkan
+diucapkannya
+diungkapkan
+dong
+dua
+dulu
+empat
+enggak
+enggaknya
+entah
+entahlah
+guna
+gunakan
+hal
+hampir
+hanya
+hanyalah
+hari
+harus
+haruslah
+harusnya
+hendak
+hendaklah
+hendaknya
+hingga
+ia
+ialah
+ibarat
+ibaratkan
+ibaratnya
+ibu
+ikut
+ingat
+ingat-ingat
+ingin
+inginkah
+inginkan
+ini
+inikah
+inilah
+itu
+itukah
+itulah
+jadi
+jadilah
+jadinya
+jangan
+jangankan
+janganlah
+jauh
+jawab
+jawaban
+jawabnya
+jelas
+jelaskan
+jelaslah
+jelasnya
+jika
+jikalau
+juga
+jumlah
+jumlahnya
+justru
+kala
+kalau
+kalaulah
+kalaupun
+kalian
+kami
+kamilah
+kamu
+kamulah
+kan
+kapan
+kapankah
+kapanpun
+karena
+karenanya
+kasus
+kata
+katakan
+katakanlah
+katanya
+ke
+keadaan
+kebetulan
+kecil
+kedua
+keduanya
+keinginan
+kelamaan
+kelihatan
+kelihatannya
+kelima
+keluar
+kembali
+kemudian
+kemungkinan
+kemungkinannya
+kenapa
+kepada
+kepadanya
+kesampaian
+keseluruhan
+keseluruhannya
+keterlaluan
+ketika
+khususnya
+kini
+kinilah
+kira
+kira-kira
+kiranya
+kita
+kitalah
+kok
+kurang
+lagi
+lagian
+lah
+lain
+lainnya
+lalu
+lama
+lamanya
+lanjut
+lanjutnya
+lebih
+lewat
+lima
+luar
+macam
+maka
+makanya
+makin
+malah
+malahan
+mampu
+mampukah
+mana
+manakala
+manalagi
+masa
+masalah
+masalahnya
+masih
+masihkah
+masing
+masing-masing
+mau
+maupun
+melainkan
+melakukan
+melalui
+melihat
+melihatnya
+memang
+memastikan
+memberi
+memberikan
+membuat
+memerlukan
+memihak
+meminta
+memintakan
+memisalkan
+memperbuat
+mempergunakan
+memperkirakan
+memperlihatkan
+mempersiapkan
+mempersoalkan
+mempertanyakan
+mempunyai
+memulai
+memungkinkan
+menaiki
+menambahkan
+menandaskan
+menanti
+menanti-nanti
+menantikan
+menanya
+menanyai
+menanyakan
+mendapat
+mendapatkan
+mendatang
+mendatangi
+mendatangkan
+menegaskan
+mengakhiri
+mengapa
+mengatakan
+mengatakannya
+mengenai
+mengerjakan
+mengetahui
+menggunakan
+menghendaki
+mengibaratkan
+mengibaratkannya
+mengingat
+mengingatkan
+menginginkan
+mengira
+mengucapkan
+mengucapkannya
+mengungkapkan
+menjadi
+menjawab
+menjelaskan
+menuju
+menunjuk
+menunjuki
+menunjukkan
+menunjuknya
+menurut
+menuturkan
+menyampaikan
+menyangkut
+menyatakan
+menyebutkan
+menyeluruh
+menyiapkan
+merasa
+mereka
+merekalah
+merupakan
+meski
+meskipun
+meyakini
+meyakinkan
+minta
+mirip
+misal
+misalkan
+misalnya
+mula
+mulai
+mulailah
+mulanya
+mungkin
+mungkinkah
+nah
+naik
+namun
+nanti
+nantinya
+nyaris
+nyatanya
+oleh
+olehnya
+pada
+padahal
+padanya
+pak
+paling
+panjang
+pantas
+para
+pasti
+pastilah
+penting
+pentingnya
+per
+percuma
+perlu
+perlukah
+perlunya
+pernah
+persoalan
+pertama
+pertama-tama
+pertanyaan
+pertanyakan
+pihak
+pihaknya
+pukul
+pula
+pun
+punya
+rasa
+rasanya
+rata
+rupanya
+saat
+saatnya
+saja
+sajalah
+saling
+sama
+sama-sama
+sambil
+sampai
+sampai-sampai
+sampaikan
+sana
+sangat
+sangatlah
+satu
+saya
+sayalah
+se
+sebab
+sebabnya
+sebagai
+sebagaimana
+sebagainya
+sebagian
+sebaik
+sebaik-baiknya
+sebaiknya
+sebaliknya
+sebanyak
+sebegini
+sebegitu
+sebelum
+sebelumnya
+sebenarnya
+seberapa
+sebesar
+sebetulnya
+sebisanya
+sebuah
+sebut
+sebutlah
+sebutnya
+secara
+secukupnya
+sedang
+sedangkan
+sedemikian
+sedikit
+sedikitnya
+seenaknya
+segala
+segalanya
+segera
+seharusnya
+sehingga
+seingat
+sejak
+sejauh
+sejenak
+sejumlah
+sekadar
+sekadarnya
+sekali
+sekali-kali
+sekalian
+sekaligus
+sekalipun
+sekarang
+sekarang
+sekecil
+seketika
+sekiranya
+sekitar
+sekitarnya
+sekurang-kurangnya
+sekurangnya
+sela
+selain
+selaku
+selalu
+selama
+selama-lamanya
+selamanya
+selanjutnya
+seluruh
+seluruhnya
+semacam
+semakin
+semampu
+semampunya
+semasa
+semasih
+semata
+semata-mata
+semaunya
+sementara
+semisal
+semisalnya
+sempat
+semua
+semuanya
+semula
+sendiri
+sendirian
+sendirinya
+seolah
+seolah-olah
+seorang
+sepanjang
+sepantasnya
+sepantasnyalah
+seperlunya
+seperti
+sepertinya
+sepihak
+sering
+seringnya
+serta
+serupa
+sesaat
+sesama
+sesampai
+sesegera
+sesekali
+seseorang
+sesuatu
+sesuatunya
+sesudah
+sesudahnya
+setelah
+setempat
+setengah
+seterusnya
+setiap
+setiba
+setibanya
+setidak-tidaknya
+setidaknya
+setinggi
+seusai
+sewaktu
+siap
+siapa
+siapakah
+siapapun
+sini
+sinilah
+soal
+soalnya
+suatu
+sudah
+sudahkah
+sudahlah
+supaya
+tadi
+tadinya
+tahu
+tahun
+tak
+tambah
+tambahnya
+tampak
+tampaknya
+tandas
+tandasnya
+tanpa
+tanya
+tanyakan
+tanyanya
+tapi
+tegas
+tegasnya
+telah
+tempat
+tengah
+tentang
+tentu
+tentulah
+tentunya
+tepat
+terakhir
+terasa
+terbanyak
+terdahulu
+terdapat
+terdiri
+terhadap
+terhadapnya
+teringat
+teringat-ingat
+terjadi
+terjadilah
+terjadinya
+terkira
+terlalu
+terlebih
+terlihat
+termasuk
+ternyata
+tersampaikan
+tersebut
+tersebutlah
+tertentu
+tertuju
+terus
+terutama
+tetap
+tetapi
+tiap
+tiba
+tiba-tiba
+tidak
+tidakkah
+tidaklah
+tiga
+tinggi
+toh
+tunjuk
+turut
+tutur
+tuturnya
+ucap
+ucapnya
+ujar
+ujarnya
+umum
+umumnya
+ungkap
+ungkapnya
+untuk
+usah
+usai
+waduh
+wah
+wahai
+waktu
+waktunya
+walau
+walaupun
+wong
+yaitu
+yakin
+yakni
+yang

src/nltk_data/corpora/stopwords/italian

@@ -0,0 +1,279 @@
+ad
+al
+allo
+ai
+agli
+all
+agl
+alla
+alle
+con
+col
+coi
+da
+dal
+dallo
+dai
+dagli
+dall
+dagl
+dalla
+dalle
+di
+del
+dello
+dei
+degli
+dell
+degl
+della
+delle
+in
+nel
+nello
+nei
+negli
+nell
+negl
+nella
+nelle
+su
+sul
+sullo
+sui
+sugli
+sull
+sugl
+sulla
+sulle
+per
+tra
+contro
+io
+tu
+lui
+lei
+noi
+voi
+loro
+mio
+mia
+miei
+mie
+tuo
+tua
+tuoi
+tue
+suo
+sua
+suoi
+sue
+nostro
+nostra
+nostri
+nostre
+vostro
+vostra
+vostri
+vostre
+mi
+ti
+ci
+vi
+lo
+la
+li
+le
+gli
+ne
+il
+un
+uno
+una
+ma
+ed
+se
+perché
+anche
+come
+dov
+dove
+che
+chi
+cui
+non
+più
+quale
+quanto
+quanti
+quanta
+quante
+quello
+quelli
+quella
+quelle
+questo
+questi
+questa
+queste
+si
+tutto
+tutti
+a
+c
+e
+i
+l
+o
+ho
+hai
+ha
+abbiamo
+avete
+hanno
+abbia
+abbiate
+abbiano
+avrò
+avrai
+avrà
+avremo
+avrete
+avranno
+avrei
+avresti
+avrebbe
+avremmo
+avreste
+avrebbero
+avevo
+avevi
+aveva
+avevamo
+avevate
+avevano
+ebbi
+avesti
+ebbe
+avemmo
+aveste
+ebbero
+avessi
+avesse
+avessimo
+avessero
+avendo
+avuto
+avuta
+avuti
+avute
+sono
+sei
+è
+siamo
+siete
+sia
+siate
+siano
+sarò
+sarai
+sarà
+saremo
+sarete
+saranno
+sarei
+saresti
+sarebbe
+saremmo
+sareste
+sarebbero
+ero
+eri
+era
+eravamo
+eravate
+erano
+fui
+fosti
+fu
+fummo
+foste
+furono
+fossi
+fosse
+fossimo
+fossero
+essendo
+faccio
+fai
+facciamo
+fanno
+faccia
+facciate
+facciano
+farò
+farai
+farà
+faremo
+farete
+faranno
+farei
+faresti
+farebbe
+faremmo
+fareste
+farebbero
+facevo
+facevi
+faceva
+facevamo
+facevate
+facevano
+feci
+facesti
+fece
+facemmo
+faceste
+fecero
+facessi
+facesse
+facessimo
+facessero
+facendo
+sto
+stai
+sta
+stiamo
+stanno
+stia
+stiate
+stiano
+starò
+starai
+starà
+staremo
+starete
+staranno
+starei
+staresti
+starebbe
+staremmo
+stareste
+starebbero
+stavo
+stavi
+stava
+stavamo
+stavate
+stavano
+stetti
+stesti
+stette
+stemmo
+steste
+stettero
+stessi
+stesse
+stessimo
+stessero
+stando

src/nltk_data/corpora/stopwords/kazakh

@@ -0,0 +1,380 @@
+ах
+ох
+эх
+ай
+эй
+ой
+тағы
+тағыда
+әрине
+жоқ
+сондай
+осындай
+осылай
+солай
+мұндай
+бұндай
+мен
+сен
+ол
+біз
+біздер
+олар
+сіз
+сіздер
+маған
+оған
+саған
+біздің
+сіздің
+оның
+бізге
+сізге
+оларға
+біздерге
+сіздерге
+оларға
+менімен
+сенімен
+онымен
+бізбен
+сізбен
+олармен
+біздермен
+сіздермен
+менің
+сенің
+біздің
+сіздің
+оның
+біздердің
+сіздердің
+олардың
+маған
+саған
+оған
+менен
+сенен
+одан
+бізден
+сізден
+олардан
+біздерден
+сіздерден
+олардан
+айтпақшы
+сонымен
+сондықтан
+бұл
+осы
+сол
+анау
+мынау
+сонау
+осынау
+ана
+мына
+сона
+әні
+міне
+өй
+үйт
+бүйт
+біреу
+кейбіреу
+кейбір
+қайсыбір
+әрбір
+бірнеше
+бірдеме
+бірнеше
+әркім
+әрне
+әрқайсы
+әрқалай
+әлдекім
+әлдене
+әлдеқайдан
+әлденеше
+әлдеқалай
+әлдеқашан
+алдақашан
+еш
+ешкім
+ешбір
+ештеме
+дәнеңе
+ешқашан
+ешқандай
+ешқайсы
+емес
+бәрі
+барлық
+барша
+бар
+күллі
+бүкіл
+түгел
+өз
+өзім
+өзің
+өзінің
+өзіме
+өзіне
+өзімнің
+өзі
+өзге
+менде
+сенде
+онда
+менен
+сенен	онан
+одан
+ау
+па
+ей
+әй
+е
+уа
+уау
+уай
+я
+пай
+ә
+о
+оһо
+ой
+ие
+аһа
+ау
+беу
+мәссаған
+бәрекелді
+әттегенай
+жаракімалла
+масқарай
+астапыралла
+япырмай
+ойпырмай
+кәне
+кәнеки
+ал
+әйда
+кәні
+міне
+әні
+сорап
+қош-қош
+пфша
+пішә
+құрау-құрау
+шәйт
+шек
+моһ
+тәк
+құрау
+құр
+кә
+кәһ
+күшім
+күшім
+мышы
+пырс
+әукім
+алақай
+паһ-паһ
+бәрекелді
+ура
+әттең
+әттеген-ай
+қап
+түге
+пішту
+шіркін
+алатау
+пай-пай
+үшін
+сайын
+сияқты
+туралы
+арқылы
+бойы
+бойымен
+шамалы
+шақты
+қаралы
+ғұрлы
+ғұрлым
+шейін
+дейін
+қарай
+таман
+салым
+тарта
+жуық
+таяу
+гөрі
+бері
+кейін
+соң
+бұрын
+бетер
+қатар
+бірге
+қоса
+арс
+
+гүрс
+
+дүрс
+
+қорс
+
+тарс
+
+тырс
+
+ырс
+
+барқ
+
+борт
+
+күрт
+
+кірт
+
+морт
+
+сарт
+
+шырт
+
+дүңк
+
+күңк
+
+қыңқ
+
+мыңқ
+
+маңқ
+
+саңқ
+
+шаңқ
+
+шіңк
+
+сыңқ
+
+таңқ
+
+тыңқ
+
+ыңқ
+
+болп
+
+былп
+
+жалп
+
+желп
+
+қолп
+
+ірк
+
+ырқ
+
+сарт-сұрт
+
+тарс-тұрс
+
+арс-ұрс
+
+жалт-жалт
+
+жалт-жұлт
+
+қалт-қалт
+
+қалт-құлт
+
+қаңқ-қаңқ
+
+қаңқ-құңқ
+
+шаңқ-шаңқ
+
+шаңқ-шұңқ
+
+арбаң-арбаң
+
+бүгжең-бүгжең
+
+арсалаң-арсалаң
+
+ербелең-ербелең
+
+батыр-бұтыр
+
+далаң-далаң
+
+тарбаң-тарбаң
+
+қызараң-қызараң
+
+қаңғыр-күңгір
+
+қайқаң-құйқаң
+
+митың-митың
+
+салаң-сұлаң
+
+ыржың-тыржың
+бірақ
+алайда
+дегенмен
+әйтпесе
+әйткенмен
+себебі
+өйткені
+сондықтан
+үшін
+сайын
+сияқты
+туралы
+арқылы
+бойы
+бойымен
+шамалы
+шақты
+қаралы
+ғұрлы
+ғұрлым
+гөрі
+бері
+кейін
+соң
+бұрын
+бетер
+қатар
+бірге
+қоса
+шейін
+дейін
+қарай
+таман
+салым
+тарта
+жуық
+таяу
+арнайы
+осындай
+ғана
+қана
+тек
+әншейін

src/nltk_data/corpora/stopwords/nepali

@@ -0,0 +1,255 @@
+छ
+र
+पनि
+छन्
+लागि
+भएको
+गरेको
+भने
+गर्न
+गर्ने
+हो
+तथा
+यो
+रहेको
+उनले
+थियो
+हुने
+गरेका
+थिए
+गर्दै
+तर
+नै
+को
+मा
+हुन्
+भन्ने
+हुन
+गरी
+त
+हुन्छ
+अब
+के
+रहेका
+गरेर
+छैन
+दिए
+भए
+यस
+ले
+गर्नु
+औं
+सो
+त्यो
+कि
+जुन
+यी
+का
+गरि
+ती
+न
+छु
+छौं
+लाई
+नि
+उप
+अक्सर
+आदि
+कसरी
+क्रमशः
+चाले
+अगाडी
+अझै
+अनुसार
+अन्तर्गत
+अन्य
+अन्यत्र
+अन्यथा
+अरु
+अरुलाई
+अर्को
+अर्थात
+अर्थात्
+अलग
+आए
+आजको
+ओठ
+आत्म
+आफू
+आफूलाई
+आफ्नै
+आफ्नो
+आयो
+उदाहरण
+उनको
+उहालाई
+एउटै
+एक
+एकदम
+कतै
+कम से कम
+कसै
+कसैले
+कहाँबाट
+कहिलेकाहीं
+का
+किन
+किनभने
+कुनै
+कुरा
+कृपया
+केही
+कोही
+गए
+गरौं
+गर्छ
+गर्छु
+गर्नुपर्छ
+गयौ
+गैर
+चार
+चाहनुहुन्छ
+चाहन्छु
+चाहिए
+छू
+जताततै
+जब
+जबकि
+जसको
+जसबाट
+जसमा
+जसलाई
+जसले
+जस्तै
+जस्तो
+जस्तोसुकै
+जहाँ
+जान
+जाहिर
+जे
+जो
+ठीक
+तत्काल
+तदनुसार
+तपाईको
+तपाई
+पर्याप्त
+पहिले
+पहिलो
+पहिल्यै
+पाँच
+पाँचौं
+तल
+तापनी
+तिनी
+तिनीहरू
+तिनीहरुको
+तिनिहरुलाई
+तिमी
+तिर
+तीन
+तुरुन्तै
+तेस्रो
+तेस्कारण
+पूर्व
+प्रति
+प्रतेक
+प्लस
+फेरी
+बने
+त्सपछि
+त्सैले
+त्यहाँ
+थिएन
+दिनुभएको
+दिनुहुन्छ
+दुई
+देखि
+बरु
+बारे
+बाहिर
+देखिन्छ
+देखियो
+देखे
+देखेको
+देखेर
+दोस्रो
+धेरै
+नजिकै
+नत्र
+नयाँ
+निम्ति
+बाहेक
+बीच
+बीचमा
+भन
+निम्न
+निम्नानुसार
+निर्दिष्ट
+नौ
+पक्का
+पक्कै
+पछि
+पछिल्लो
+पटक
+पर्छ
+पर्थ्यो
+भन्छन्
+भन्
+भन्छु
+भन्दा
+भन्नुभयो
+भर
+भित्र
+भित्री
+म
+मलाई
+मात्र
+माथि
+मुख्य
+मेरो
+यति
+यथोचित
+यदि
+यद्यपि
+यसको
+यसपछि
+यसबाहेक
+यसरी
+यसो
+यस्तो
+यहाँ
+यहाँसम्म
+या
+रही
+राखे
+राख्छ
+राम्रो
+रूप
+लगभग
+वरीपरी
+वास्तवमा
+बिरुद्ध
+बिशेष
+सायद
+शायद
+संग
+संगै
+सक्छ
+सट्टा
+सधै
+सबै
+सबैलाई
+समय
+सम्भव
+सम्म
+सही
+साँच्चै
+सात
+साथ
+साथै
+सारा
+सोही
+स्पष्ट
+हरे
+हरेक

src/nltk_data/corpora/stopwords/norwegian

@@ -0,0 +1,176 @@
+og
+i
+jeg
+det
+at
+en
+et
+den
+til
+er
+som
+på
+de
+med
+han
+av
+ikke
+ikkje
+der
+så
+var
+meg
+seg
+men
+ett
+har
+om
+vi
+min
+mitt
+ha
+hadde
+hun
+nå
+over
+da
+ved
+fra
+du
+ut
+sin
+dem
+oss
+opp
+man
+kan
+hans
+hvor
+eller
+hva
+skal
+selv
+sjøl
+her
+alle
+vil
+bli
+ble
+blei
+blitt
+kunne
+inn
+når
+være
+kom
+noen
+noe
+ville
+dere
+som
+deres
+kun
+ja
+etter
+ned
+skulle
+denne
+for
+deg
+si
+sine
+sitt
+mot
+å
+meget
+hvorfor
+dette
+disse
+uten
+hvordan
+ingen
+din
+ditt
+blir
+samme
+hvilken
+hvilke
+sånn
+inni
+mellom
+vår
+hver
+hvem
+vors
+hvis
+både
+bare
+enn
+fordi
+før
+mange
+også
+slik
+vært
+være
+båe
+begge
+siden
+dykk
+dykkar
+dei
+deira
+deires
+deim
+di
+då
+eg
+ein
+eit
+eitt
+elles
+honom
+hjå
+ho
+hoe
+henne
+hennar
+hennes
+hoss
+hossen
+ikkje
+ingi
+inkje
+korleis
+korso
+kva
+kvar
+kvarhelst
+kven
+kvi
+kvifor
+me
+medan
+mi
+mine
+mykje
+no
+nokon
+noka
+nokor
+noko
+nokre
+si
+sia
+sidan
+so
+somt
+somme
+um
+upp
+vere
+vore
+verte
+vort
+varte
+vart

src/nltk_data/corpora/stopwords/portuguese

@@ -0,0 +1,207 @@
+a
+à
+ao
+aos
+aquela
+aquelas
+aquele
+aqueles
+aquilo
+as
+às
+até
+com
+como
+da
+das
+de
+dela
+delas
+dele
+deles
+depois
+do
+dos
+e
+é
+ela
+elas
+ele
+eles
+em
+entre
+era
+eram
+éramos
+essa
+essas
+esse
+esses
+esta
+está
+estamos
+estão
+estar
+estas
+estava
+estavam
+estávamos
+este
+esteja
+estejam
+estejamos
+estes
+esteve
+estive
+estivemos
+estiver
+estivera
+estiveram
+estivéramos
+estiverem
+estivermos
+estivesse
+estivessem
+estivéssemos
+estou
+eu
+foi
+fomos
+for
+fora
+foram
+fôramos
+forem
+formos
+fosse
+fossem
+fôssemos
+fui
+há
+haja
+hajam
+hajamos
+hão
+havemos
+haver
+hei
+houve
+houvemos
+houver
+houvera
+houverá
+houveram
+houvéramos
+houverão
+houverei
+houverem
+houveremos
+houveria
+houveriam
+houveríamos
+houvermos
+houvesse
+houvessem
+houvéssemos
+isso
+isto
+já
+lhe
+lhes
+mais
+mas
+me
+mesmo
+meu
+meus
+minha
+minhas
+muito
+na
+não
+nas
+nem
+no
+nos
+nós
+nossa
+nossas
+nosso
+nossos
+num
+numa
+o
+os
+ou
+para
+pela
+pelas
+pelo
+pelos
+por
+qual
+quando
+que
+quem
+são
+se
+seja
+sejam
+sejamos
+sem
+ser
+será
+serão
+serei
+seremos
+seria
+seriam
+seríamos
+seu
+seus
+só
+somos
+sou
+sua
+suas
+também
+te
+tem
+tém
+temos
+tenha
+tenham
+tenhamos
+tenho
+terá
+terão
+terei
+teremos
+teria
+teriam
+teríamos
+teu
+teus
+teve
+tinha
+tinham
+tínhamos
+tive
+tivemos
+tiver
+tivera
+tiveram
+tivéramos
+tiverem
+tivermos
+tivesse
+tivessem
+tivéssemos
+tu
+tua
+tuas
+um
+uma
+você
+vocês
+vos

src/nltk_data/corpora/stopwords/romanian

@@ -0,0 +1,356 @@
+a
+abia
+acea
+aceasta
+această
+aceea
+aceeasi
+acei
+aceia
+acel
+acela
+acelasi
+acele
+acelea
+acest
+acesta
+aceste
+acestea
+acestei
+acestia
+acestui
+aceşti
+aceştia
+adica
+ai
+aia
+aibă
+aici
+al
+ala
+ale
+alea
+alt
+alta
+altceva
+altcineva
+alte
+altfel
+alti
+altii
+altul
+am
+anume
+apoi
+ar
+are
+as
+asa
+asta
+astea
+astfel
+asupra
+atare
+atat
+atata
+atatea
+atatia
+ati
+atit
+atita
+atitea
+atitia
+atunci
+au
+avea
+avem
+aveţi
+avut
+aş
+aţi
+ba
+ca
+cam
+cand
+care
+careia
+carora
+caruia
+cat
+catre
+ce
+cea
+ceea
+cei
+ceilalti
+cel
+cele
+celor
+ceva
+chiar
+ci
+cind
+cine
+cineva
+cit
+cita
+cite
+citeva
+citi
+citiva
+cu
+cui
+cum
+cumva
+cât
+câte
+câtva
+câţi
+cînd
+cît
+cîte
+cîtva
+cîţi
+că
+căci
+cărei
+căror
+cărui
+către
+da
+daca
+dacă
+dar
+dat
+dată
+dau
+de
+deasupra
+deci
+decit
+deja
+desi
+despre
+deşi
+din
+dintr
+dintr-
+dintre
+doar
+doi
+doilea
+două
+drept
+dupa
+după
+dă
+e
+ea
+ei
+el
+ele
+era
+eram
+este
+eu
+eşti
+face
+fara
+fata
+fel
+fi
+fie
+fiecare
+fii
+fim
+fiu
+fiţi
+foarte
+fost
+fără
+i
+ia
+iar
+ii
+il
+imi
+in
+inainte
+inapoi
+inca
+incit
+insa
+intr
+intre
+isi
+iti
+la
+le
+li
+lor
+lui
+lângă
+lîngă
+m
+ma
+mai
+mea
+mei
+mele
+mereu
+meu
+mi
+mie
+mine
+mod
+mult
+multa
+multe
+multi
+multă
+mulţi
+mâine
+mîine
+mă
+ne
+ni
+nici
+nimeni
+nimic
+niste
+nişte
+noastre
+noastră
+noi
+nostri
+nostru
+nou
+noua
+nouă
+noştri
+nu
+numai
+o
+or
+ori
+oricare
+orice
+oricine
+oricum
+oricând
+oricât
+oricînd
+oricît
+oriunde
+pai
+parca
+patra
+patru
+pe
+pentru
+peste
+pic
+pina
+poate
+pot
+prea
+prima
+primul
+prin
+printr-
+putini
+puţin
+puţina
+puţină
+până
+pînă
+sa
+sa-mi
+sa-ti
+sai
+sale
+sau
+se
+si
+sint
+sintem
+spate
+spre
+sub
+sunt
+suntem
+sunteţi
+sus
+să
+săi
+său
+t
+ta
+tale
+te
+ti
+tine
+toata
+toate
+toată
+tocmai
+tot
+toti
+totul
+totusi
+totuşi
+toţi
+trei
+treia
+treilea
+tu
+tuturor
+tăi
+tău
+u
+ul
+ului
+un
+una
+unde
+undeva
+unei
+uneia
+unele
+uneori
+unii
+unor
+unora
+unu
+unui
+unuia
+unul
+v
+va
+vi
+voastre
+voastră
+voi
+vom
+vor
+vostru
+vouă
+voştri
+vreo
+vreun
+vă
+zi
+zice
+îi
+îl
+îmi
+în
+îţi
+ăla
+ălea
+ăsta
+ăstea
+ăştia
+şi
+ţi
+ţie

src/nltk_data/corpora/stopwords/russian

@@ -0,0 +1,151 @@
+и
+в
+во
+не
+что
+он
+на
+я
+с
+со
+как
+а
+то
+все
+она
+так
+его
+но
+да
+ты
+к
+у
+же
+вы
+за
+бы
+по
+только
+ее
+мне
+было
+вот
+от
+меня
+еще
+нет
+о
+из
+ему
+теперь
+когда
+даже
+ну
+вдруг
+ли
+если
+уже
+или
+ни
+быть
+был
+него
+до
+вас
+нибудь
+опять
+уж
+вам
+ведь
+там
+потом
+себя
+ничего
+ей
+может
+они
+тут
+где
+есть
+надо
+ней
+для
+мы
+тебя
+их
+чем
+была
+сам
+чтоб
+без
+будто
+чего
+раз
+тоже
+себе
+под
+будет
+ж
+тогда
+кто
+этот
+того
+потому
+этого
+какой
+совсем
+ним
+здесь
+этом
+один
+почти
+мой
+тем
+чтобы
+нее
+сейчас
+были
+куда
+зачем
+всех
+никогда
+можно
+при
+наконец
+два
+об
+другой
+хоть
+после
+над
+больше
+тот
+через
+эти
+нас
+про
+всего
+них
+какая
+много
+разве
+три
+эту
+моя
+впрочем
+хорошо
+свою
+этой
+перед
+иногда
+лучше
+чуть
+том
+нельзя
+такой
+им
+более
+всегда
+конечно
+всю
+между