Update README.md
Browse files
README.md
CHANGED
|
@@ -11,63 +11,13 @@ datasets:
|
|
| 11 |
|
| 12 |
It is a state-of-the-art language model for MPNet for Covid-19 dataset with focus on post-covid.
|
| 13 |
## How to use for Deepset Haystack
|
| 14 |
-
|
| 15 |
```
|
| 16 |
-
%cd /content/drive/MyDrive
|
| 17 |
!sudo apt-get install git-lfs
|
| 18 |
-
|
| 19 |
!git lfs install
|
| 20 |
-
|
| 21 |
!git clone https://huggingface.co/shaina/CoQUAD_MPNet
|
| 22 |
GIT_LFS_SKIP_SMUDGE=1
|
| 23 |
|
| 24 |
-
from haystack.utils import clean_wiki_text, convert_files_to_dicts, fetch_archive_from_http, print_answers
|
| 25 |
-
from haystack.nodes import FARMReader, TransformersReader
|
| 26 |
-
from haystack.utils import launch_es
|
| 27 |
-
|
| 28 |
-
launch_es()
|
| 29 |
-
! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
|
| 30 |
-
! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
|
| 31 |
-
! chown -R daemon:daemon elasticsearch-7.9.2
|
| 32 |
-
|
| 33 |
-
import os
|
| 34 |
-
from subprocess import Popen, PIPE, STDOUT
|
| 35 |
-
es_server = Popen(['elasticsearch-7.9.2/bin/elasticsearch'],
|
| 36 |
-
stdout=PIPE, stderr=STDOUT,
|
| 37 |
-
preexec_fn=lambda: os.setuid(1) # as daemon
|
| 38 |
-
)
|
| 39 |
-
! sleep 30
|
| 40 |
-
|
| 41 |
-
from haystack.document_stores import ElasticsearchDocumentStore
|
| 42 |
-
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document")
|
| 43 |
-
|
| 44 |
-
import pandas as pd
|
| 45 |
-
df=pd.read_excel('/content/covid.xlsx')
|
| 46 |
-
df.fillna(value="", inplace=True)
|
| 47 |
-
|
| 48 |
-
print(df.head())
|
| 49 |
-
|
| 50 |
-
from typing import List
|
| 51 |
-
import requests
|
| 52 |
-
import pandas as pd
|
| 53 |
-
from haystack import Document
|
| 54 |
-
from haystack.document_stores import FAISSDocumentStore
|
| 55 |
-
from haystack.nodes import RAGenerator, DensePassageRetriever
|
| 56 |
-
|
| 57 |
-
titles = list(df["document_identifier"].values)
|
| 58 |
-
texts = list(df["document_text"].values)
|
| 59 |
-
documents: List[Document] = []
|
| 60 |
-
for title, text in zip(titles, texts):
|
| 61 |
-
documents.append(
|
| 62 |
-
Document(
|
| 63 |
-
content=text,
|
| 64 |
-
meta={
|
| 65 |
-
"name": title or ""
|
| 66 |
-
}
|
| 67 |
-
)
|
| 68 |
-
)
|
| 69 |
-
document_store.write_documents(documents)
|
| 70 |
-
|
| 71 |
from haystack.nodes import ElasticsearchRetriever
|
| 72 |
retriever = ElasticsearchRetriever(document_store=document_store)
|
| 73 |
reader = FARMReader(model_name_or_path="/content/drive/MyDrive/CoQUAD_MPNet", use_gpu=True)
|
|
@@ -78,7 +28,6 @@ prediction = pipe.run(
|
|
| 78 |
query="What is post-COVID?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}}
|
| 79 |
)
|
| 80 |
from pprint import pprint
|
| 81 |
-
|
| 82 |
pprint(prediction)
|
| 83 |
```
|
| 84 |
|
|
|
|
| 11 |
|
| 12 |
It is a state-of-the-art language model for MPNet for Covid-19 dataset with focus on post-covid.
|
| 13 |
## How to use for Deepset Haystack
|
| 14 |
+
|
| 15 |
```
|
|
|
|
| 16 |
!sudo apt-get install git-lfs
|
|
|
|
| 17 |
!git lfs install
|
|
|
|
| 18 |
!git clone https://huggingface.co/shaina/CoQUAD_MPNet
|
| 19 |
GIT_LFS_SKIP_SMUDGE=1
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
from haystack.nodes import ElasticsearchRetriever
|
| 22 |
retriever = ElasticsearchRetriever(document_store=document_store)
|
| 23 |
reader = FARMReader(model_name_or_path="/content/drive/MyDrive/CoQUAD_MPNet", use_gpu=True)
|
|
|
|
| 28 |
query="What is post-COVID?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}}
|
| 29 |
)
|
| 30 |
from pprint import pprint
|
|
|
|
| 31 |
pprint(prediction)
|
| 32 |
```
|
| 33 |
|