add voting

Dockerfile

@@ -6,6 +6,8 @@ COPY ./requirements.txt /code/requirements.txt
 
 RUN pip install --upgrade -r /code/requirements.txt
 
+COPY ./embedder ./embedder
+COPY ./classifiers ./classifiers
 COPY ./model_finetuned_clear.pkl ./model_finetuned_clear.pkl
 COPY ./required_classes.py ./required_classes.py
 COPY ./app.py ./app.py

app.py

@@ -4,33 +4,86 @@ from flask import Flask, request
 import json
 import pickle
 import numpy as np
+import os
 
 from required_classes import BertEmbedder, PredictModel
 
 
-print('INFO: loading model')
+CLS_WEIGHTS = {'mlp': 0.3, 'svc': 0.4, 'xgboost': 0.3}
+
+print('INFO: loading models')
+try:
+    with open('embedder/embedder.pkl', 'rb') as f:
+        embedder = pickle.load(f)
+    print('INFO: embedder loaded')
+except Exception as e:
+    print(f"ERROR: loading embedder failed with: {str(e)}")
+
+
+classifiers_codes = {}
+try:
+    for clf_name in os.listdir('classifiers/codes'):
+        if '.' == clf_name[0]:
+            continue
+        with open('classifiers/codes/'+clf_name, 'rb') as f:
+            model = pickle.load(f)
+            classifiers_codes[clf_name.split('.')[0]] = model
+        print(f'INFO: classifier {clf_name} loaded')
+except Exception as e:
+    print(f"ERROR: loading classifiers failed with: {str(e)}")
+
+classifiers_groups = {}
 try:
-    with open('model_finetuned_clear.pkl', 'rb') as f:
-        model = pickle.load(f)
-    model.batch_size = 1
-    print('INFO: model loaded')
+    for clf_name in os.listdir('classifiers/groups'):
+        if '.' == clf_name[0]:
+            continue
+        with open('classifiers/groups/'+clf_name, 'rb') as f:
+            model = pickle.load(f)
+            classifiers_groups[clf_name.split('.')[0]] = model
+        print(f'INFO: classifier {clf_name} loaded')
 except Exception as e:
-    print(f"ERROR: loading models failed with: {str(e)}")
+    print(f"ERROR: loading classifiers failed with: {str(e)}")
+
 
 def classify_code(text, top_n):
-    embed = model._texts2vecs([text])
-    probs = model.classifier_code.predict_proba(embed)
-    best_n = np.flip(np.argsort(probs, axis=1,)[0,-top_n:])
-    preds = [{'code': model.classifier_code.classes_[i], 'proba': probs[0][i]} for i in best_n]
+    embed = [embedder(text)]
+    preds = {}
+    for clf_name in classifiers_codes.keys():
+        model = classifiers_codes[clf_name]
+        probs = model.predict_proba(embed)
+        best_n = np.flip(np.argsort(probs, axis=1,)[0,-top_n:])
+        clf_preds = {str(model.classes_[i]): float(probs[0][i]) for i in best_n}
+        preds[clf_name] = clf_preds
     return preds
 
+
 def classify_group(text, top_n):
-    embed = model._texts2vecs([text])
-    probs = model.classifier_group.predict_proba(embed)
-    best_n = np.flip(np.argsort(probs, axis=1,)[0,-top_n:])
-    preds = [{'group': model.classifier_group.classes_[i], 'proba': probs[0][i]} for i in best_n]
+    embed = [embedder(text)]
+    preds = {}
+    for clf_name in classifiers_groups.keys():
+        model = classifiers_groups[clf_name]
+        probs = model.predict_proba(embed)
+        best_n = np.flip(np.argsort(probs, axis=1,)[0,-top_n:])
+        clf_preds = {str(model.classes_[i]): float(probs[0][i]) for i in best_n}
+        preds[clf_name] = clf_preds
     return preds
 
+def get_top_result(preds):
+    total_scores = {}
+    for clf_name, scores in preds.items():
+        clf_name = clf_name.replace('_codes', '').replace('_groups', '')
+        for class_name, score in scores.items():
+            if class_name in total_scores:
+                total_scores[class_name] += CLS_WEIGHTS[clf_name] * score
+            else:
+                total_scores[class_name] = CLS_WEIGHTS[clf_name] * score
+
+    max_idx = np.array(total_scores.values()).argmax()
+    if list(total_scores.values())[max_idx] > 0.5:
+        return list(total_scores.keys())[max_idx]
+    else:
+        return None
+
 
 app = Flask(__name__)
 
@@ -49,8 +102,6 @@ def read_root():
     base64_bytes = str(data['textB64']).encode("ascii")
     sample_string_bytes = base64.b64decode(base64_bytes)
     text = sample_string_bytes.decode("ascii")
-    print(text)
-    # text = str(data['text'])
     top_n = int(data['top_n'])
 
     if top_n < 1:
@@ -60,11 +111,13 @@ def read_root():
     
     pred_codes = classify_code(text, top_n)
     pred_groups = classify_group(text, top_n)
+    pred_codes_top = get_top_result(pred_codes)
+    pred_groups_top = get_top_result(pred_groups)
     result = {
         "icd10": 
-            {'result': pred_codes[0]['code'], 'details': pred_codes},
+            {'result': pred_codes_top, 'details': pred_codes},
         "dx_group":
-            {'result': pred_groups[0]['group'], 'details': pred_groups}
+            {'result': pred_groups_top, 'details': pred_groups}
     }
     return result
 

model_finetuned_clear.pkl → classifiers/codes/mlp_codes.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c40076019c4b4767021bf208200a8104f0910669d0b56952e6b2eb62b1539d3
-size 434856921
+oid sha256:39eda1dc2f583525dd689e048b396f476835c42ed2a8fead4884b6c87bad639d
+size 5185392

classifiers/codes/svc_codes.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:669ddbb9dea179f38dcc5e94282074240eac2cb361e839416a4fc74ea718050d
+size 865684575

classifiers/codes/xgboost_codes.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a24f3740770aca1f546ccf44e2d2346ba18105920691255b587026cba962d7
+size 17972293

classifiers/groups/mlp_groups.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f494f298b214d3239be0a53ae986b6eadf6822ccb77fcfdf8149b1aad80c87f2
+size 4924655

classifiers/groups/svc_groups.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85d1f0874ebdc3986d412e6d8508d979a324c278d70b29dc4491477b209e69fa
+size 574657606

classifiers/groups/xgboost_groups.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f1d2fd1f46c18e3bf6b0b67ae8a6c6b52c8b4e06961eff4568cebee7f68989
+size 8361773

embedder/embedder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b27f378ed88ce2ae9e01e9fd4e5706fc2072957b665c4d12e02a84bbe4f4c4d8
+size 434228972

required_classes.py

@@ -25,7 +25,7 @@ class BertEmbedder:
                                        truncation=True).to(self.device)
         model_output = self.embedder(**encoded_input)
         text_embed = model_output.pooler_output[0].cpu()
-        return text_embed
+        return text_embed.tolist()
 
     def batch_predict(self, texts: List[str]):
         encoded_input = self.tokenizer(texts, 

requirements.txt

@@ -1,5 +1,6 @@
 numpy==1.22.4
 torch==2.0.1
-scikit-learn==1.2.2
+scikit-learn==1.3.0
 transformers==4.29.2
 flask==2.0.3
+xgboost==1.7.6