Add application file

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.venv
+.idea

Dockerfile

@@ -0,0 +1,14 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,16 @@
+from fastapi import FastAPI
+
+from summary import summarize
+
+app = FastAPI()
+
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}
+
+
+@app.post("/summary/")
+async def summary(text_request):
+    text = text_request.text
+    return summarize(text)

requirement.txt

@@ -0,0 +1,9 @@
+fastapi
+uvicorn
+langdetect
+protobuf
+sentencepiece
+transformers
+torch
+tensorflow
+tf-keras

summary.py

@@ -0,0 +1,85 @@
+import re
+
+from langdetect import detect
+from transformers import pipeline
+
+from utils.tag_utils import filter_tags
+
+AiSummaryVersion = 1
+summarization_pipeline = pipeline("summarization", model="csebuetnlp/mT5_multilingual_XLSum")
+en_translation_pipe = pipeline("translation", model="Helsinki-NLP/opus-mt-mul-en")
+classification_pipe = pipeline("text-classification", model="Yueh-Huan/news-category-classification-distilbert")
+tag_gen_pipe = pipeline("text2text-generation", model="fabiochiu/t5-base-tag-generation")
+
+
+def summarize(text: str):
+    if text is None or len(text) < 10:
+        return {
+            "ver": AiSummaryVersion
+        }
+    summary = get_summarization(text) if len(text) > 100 else text
+    translated = get_en_translation(summary)
+    tags1 = get_classification(translated)
+    tags2 = get_tags(translated)
+    tags = filter_tags(list(set(tags1 + tags2)))
+
+    return {
+        "ver": AiSummaryVersion,
+        "summary": summary,
+        "tags": tags,
+    }
+
+
+def get_summarization(text: str):
+    try:
+        result = summarization_pipeline(text)
+        return result[0]['summary_text'] if isinstance(result, list) else result['summary_text']
+    except:
+        return None
+
+
+def get_en_translation(text: str):
+    if text is None:
+        return None
+    try:
+        if is_english(text):
+            return text
+        result = en_translation_pipe(text)
+        return result[0]['translation_text'] if isinstance(result, list) else result['translation_text']
+    except:
+        return None
+
+
+def is_english(text):
+    try:
+        lang = detect(text)
+        return lang == 'en'
+    except:
+        return False
+
+
+def get_tags(text: str):
+    if text is None:
+        return []
+    try:
+        result = tag_gen_pipe(text)
+        tag_str = result[0]['generated_text'] if isinstance(result, list) else result['generated_text']
+        tags = re.split(r'[&,]', tag_str)
+        tags = [tag.strip() for tag in tags]
+        tags = [tag for tag in tags if len(tag) > 2 and len(tag.split(' ')) == 1]
+        return tags
+    except:
+        return []
+
+
+def get_classification(text: str):
+    if text is None:
+        return []
+    try:
+        result = classification_pipe(text)
+        if isinstance(result, list):
+            return [tag['label'].strip() for tag in result if tag['score'] > 0.75]
+        else:
+            return [result['label'].strip()] if result['score'] > 0.75 else []
+    except:
+        return []

utils/tag_utils.py

@@ -0,0 +1,14 @@
+from typing import List
+
+BlockTagNames = {'fifty'}
+BlockTagContents = ['voice']
+
+
+def filter_tags(tags: List[str]) -> List[str]:
+    tags = [tag.strip().lower() for tag in tags]
+    tags = [tag for tag in tags
+            if len(tag) > 2
+            and tag not in BlockTagNames
+            and not any(content in tag for content in BlockTagContents)]
+    return sorted(tags)
+