model

app.py

@@ -0,0 +1,25 @@
+from flask import Flask, jsonify, request
+import pickle
+import pandas as pd
+
+from utils import predict
+
+
+app = Flask(__name__)
+
+
+@app.route('/')
+def index():
+    return "/model/text – predicts tag for string sample"
+
+
+@app.route('/model/text', methods=['POST'])
+def parse_string():
+    sample = request.get_data()
+    model = pickle.load(open('models/tags_model.pkl', 'rb'))
+    labelencoder = pickle.load(open('models/tags_encoder.pkl', 'rb'))
+    return predict(sample, model, labelencoder)
+
+
+if __name__ == '__main__':
+    app.run(debug=True)

models/tags_encoder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d16ff00b7f5cf88af997b48086fe4599ac4b22a841d2e468f4a95b6249cc65
+size 1925

models/tags_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3848747f6fc040cc4c3cad14cb0e9d73ea64805e86b9affd04949265819a75a2
+size 23629883

tags.py

@@ -0,0 +1,17 @@
+import pickle
+import numpy as np
+from utils import predict
+
+
+if __name__ == '__main__':
+    data = pickle.load(open('data/tags_data.pkl', 'rb'))
+    sample = data['orig_text']
+    model = pickle.load(open('models/tags_model.pkl', 'rb'))
+    labelencoder = pickle.load(open('models/tags_encoder.pkl', 'rb'))
+
+    data['prediction'] = predict(data['orig_text'],
+                                 model,
+                                 labelencoder,
+                                 preproc=False)
+
+    print(data.iloc[0]["orig_text"])

utils.py

@@ -0,0 +1,29 @@
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import RegexpTokenizer
+nltk.download('stopwords')
+from pymystem3 import Mystem
+import pandas as pd
+from pandarallel import pandarallel
+pandarallel.initialize(progress_bar=True)
+
+
+def preproc_text(x):
+    mystem = Mystem()
+    stop_words = stopwords.words('russian')
+    stop_words.extend([' ', ' \n', '  ', 'также', 'который', 'весь', 'заявлять', 'сообщать', 'риа'])
+    tokenizer = RegexpTokenizer(r'\w+')
+    data = ''.join(mystem.lemmatize(x))
+    data = tokenizer.tokenize(data)
+    return ' '.join([word for word in data if word not in stop_words])
+
+
+def predict(sample, model, label_encoder, preproc=True):
+    if preproc:
+        if type(sample) == pd.Series:
+            sample = sample.parallel_apply(preproc_text)
+            return label_encoder.inverse_transform(model.predict(sample))
+
+        elif type(sample) == bytes or type(sample) == str:
+            sample = [preproc_text(sample)]
+            return label_encoder.inverse_transform(model.predict(sample))[0]