File size: 2,495 Bytes
56903cc f708fe0 beec65b 56903cc 5698cae 56903cc beec65b f708fe0 beec65b f708fe0 bfb7b0e 7b63e3d bfb7b0e 7b63e3d bfb7b0e 7b63e3d bfb7b0e 03a5442 5b9d21c 7b63e3d f708fe0 9136bea beec65b 03a5442 7b63e3d 9136bea beec65b 9136bea 03a5442 9136bea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
from fastapi import FastAPI
from huggingface_hub import snapshot_download
from huggingface_hub import hf_hub_download
import os
print("Version 1")
app = FastAPI()
def download_file_from_hf(repo_id, filename):
"""
Downloads a single file from a Hugging Face repo into ~/.sinatools
Args:
repo_id (str): Hugging Face repo id, e.g. "SinaLab/ArabGlossBERT"
filename (str): Path of the file inside the repo, e.g. "config.json"
Returns:
str: Absolute path to the downloaded file
"""
target_dir = os.path.expanduser("~/.sinatools")
os.makedirs(target_dir, exist_ok=True)
file_path = hf_hub_download(
repo_id=repo_id,
filename=filename,
local_dir=target_dir,
local_dir_use_symlinks=False
)
return file_path
def download_folder_from_hf(repo_id, folder_name):
"""
Downloads a folder from a Hugging Face model repo into ~/.sinatools
"""
target_dir = os.path.expanduser("~/.sinatools")
local_path = snapshot_download(
repo_id=repo_id,
allow_patterns=f"{folder_name}/**",
local_dir=target_dir,
local_dir_use_symlinks=False
)
return os.path.join(local_path, folder_name)
print("Start loading")
download_folder_from_hf("SinaLab/Wojood_model", "Wj27012000.tar")
download_folder_from_hf("SinaLab/ArabGlossBERT", "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01")
download_folder_from_hf("SinaLab/ArabGlossBERT", "bert-base-arabertv02")
download_file_from_hf("SinaLab/ArabGlossBERT","one_gram.pickle")
download_file_from_hf("SinaLab/ArabGlossBERT","two_grams.pickle")
download_file_from_hf("SinaLab/ArabGlossBERT","three_grams.pickle")
download_file_from_hf("SinaLab/ArabGlossBERT","four_grams.pickle")
download_file_from_hf("SinaLab/ArabGlossBERT","five_grams.pickle")
download_file_from_hf("SinaLab/ALMA","lemmas_dic.pickle")
print("Finish loading")
from sinatools.wsd.disambiguator import disambiguate
from pydantic import BaseModel
from fastapi.responses import JSONResponse
class SALMARequest(BaseModel):
text: str
@app.post("/predict")
def predict(request: SALMARequest):
# Load tagger
text = request.text
print("Start disambiguate")
salma_output = disambiguate(text)
content = {
"resp": salma_output,
"statusText": "OK",
"statusCode": 0,
}
return JSONResponse(
content=content,
media_type="application/json",
status_code=200,
)
|