Spaces:
Sleeping
Sleeping
File size: 2,331 Bytes
5ed6105 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | from fastapi import FastAPI
import spacy
import json
from typing import List
from bs4 import BeautifulSoup
from markdown import markdown
TAG_PROBABILITY_THRESHOLD = 0.5
app = FastAPI()
nlp_text = spacy.load("./spacy_text_model")
nlp_code = spacy.load("./spacy_code_model")
with open('selected_tags.json', 'r') as openfile:
selected_tags = json.load(openfile)
def preprocess(texts):
tokens = []
removal = ['PUNCT', 'SPACE', 'NUM', 'SYM']
cleaned_texts = []
for summary in nlp_text.pipe(texts, disable=["transformer", "tagger", "parser", "attribute_ruler", "lemmatizer", "ner"]):
question_tokens = []
for token in summary:
if token.pos_ not in removal and token.is_alpha and len(question_tokens) < 512:
question_tokens.append(token.lower_)
cleaned_texts.append(" ".join(question_tokens))
return cleaned_texts
def get_text_and_code(body):
html = markdown(body)
bs = BeautifulSoup(html)
codes = bs.findAll('code')
code = '\n'.join([x.text for x in codes])
for x in codes:
x.decompose()
text = '\n'.join(bs.findAll(text=True))
return text, code
@app.post("/infer_tags")
async def infer_tags(questions: List[str]):
results = []
texts = []
codes = []
for question in questions:
text, code = get_text_and_code(question)
texts.append(text)
codes.append(code)
texts_preprocessed = preprocess(texts)
codes_preprocessed = preprocess(codes)
pred_text = []
pred_code = []
for summary in nlp_text.pipe(texts_preprocessed):
if summary.text != '':
pred_text.append(summary.cats)
else:
pred_text.append(dict.fromkeys(selected_tags, 0))
for summary in nlp_code.pipe(codes_preprocessed):
if summary.text != '':
pred_code.append(summary.cats)
else:
pred_code.append(dict.fromkeys(selected_tags, 0))
text_tags = [[x for x in selected_tags if y[x] > TAG_PROBABILITY_THRESHOLD]
for y in pred_text]
code_tags = [[x for x in selected_tags if y[x] > TAG_PROBABILITY_THRESHOLD]
for y in pred_code]
union_tags = []
for i in range(len(text_tags)):
union_tags.append(list(set(text_tags[i]) | set(code_tags[i])))
return union_tags
|