Update README.md
Browse files
README.md
CHANGED
|
@@ -18,20 +18,13 @@ It is a state-of-the-art language model for MPNet for Covid-19 dataset with focu
|
|
| 18 |
!git lfs install
|
| 19 |
|
| 20 |
!git clone https://huggingface.co/shaina/CoQUAD_MPNet
|
| 21 |
-
# if you want to clone without large files – just their pointers
|
| 22 |
-
# prepend your git clone with the following env var:
|
| 23 |
GIT_LFS_SKIP_SMUDGE=1
|
| 24 |
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
from haystack.utils import clean_wiki_text, convert_files_to_dicts, fetch_archive_from_http, print_answers
|
| 29 |
from haystack.nodes import FARMReader, TransformersReader
|
| 30 |
-
# Recommended: Start Elasticsearch using Docker via the Haystack utility function
|
| 31 |
from haystack.utils import launch_es
|
| 32 |
|
| 33 |
launch_es()
|
| 34 |
-
# In Colab / No Docker environments: Start Elasticsearch from source
|
| 35 |
! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
|
| 36 |
! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
|
| 37 |
! chown -R daemon:daemon elasticsearch-7.9.2
|
|
@@ -42,9 +35,7 @@ es_server = Popen(['elasticsearch-7.9.2/bin/elasticsearch'],
|
|
| 42 |
stdout=PIPE, stderr=STDOUT,
|
| 43 |
preexec_fn=lambda: os.setuid(1) # as daemon
|
| 44 |
)
|
| 45 |
-
# wait until ES has started
|
| 46 |
! sleep 30
|
| 47 |
-
# Connect to Elasticsearch
|
| 48 |
|
| 49 |
from haystack.document_stores import ElasticsearchDocumentStore
|
| 50 |
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document")
|
|
@@ -62,7 +53,6 @@ from haystack import Document
|
|
| 62 |
from haystack.document_stores import FAISSDocumentStore
|
| 63 |
from haystack.nodes import RAGenerator, DensePassageRetriever
|
| 64 |
|
| 65 |
-
# Use data to initialize Document objects
|
| 66 |
titles = list(df["document_identifier"].values)
|
| 67 |
texts = list(df["document_text"].values)
|
| 68 |
documents: List[Document] = []
|
|
@@ -75,11 +65,6 @@ for title, text in zip(titles, texts):
|
|
| 75 |
}
|
| 76 |
)
|
| 77 |
)
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
# Now, let's write the dicts containing documents to our DB.
|
| 83 |
document_store.write_documents(documents)
|
| 84 |
|
| 85 |
from haystack.nodes import ElasticsearchRetriever
|
|
@@ -88,15 +73,9 @@ reader = FARMReader(model_name_or_path="/content/drive/MyDrive/CoQUAD_MPNet", us
|
|
| 88 |
|
| 89 |
from haystack.pipelines import ExtractiveQAPipeline
|
| 90 |
pipe = ExtractiveQAPipeline(reader, retriever)
|
| 91 |
-
# You can configure how many candidates the reader and retriever shall return
|
| 92 |
-
# The higher top_k_retriever, the better (but also the slower) your answers.
|
| 93 |
-
|
| 94 |
-
|
| 95 |
prediction = pipe.run(
|
| 96 |
query="What is post-COVID?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}}
|
| 97 |
)
|
| 98 |
-
|
| 99 |
-
# Now you can either print the object directly...
|
| 100 |
from pprint import pprint
|
| 101 |
|
| 102 |
pprint(prediction)
|
|
|
|
| 18 |
!git lfs install
|
| 19 |
|
| 20 |
!git clone https://huggingface.co/shaina/CoQUAD_MPNet
|
|
|
|
|
|
|
| 21 |
GIT_LFS_SKIP_SMUDGE=1
|
| 22 |
|
|
|
|
|
|
|
|
|
|
| 23 |
from haystack.utils import clean_wiki_text, convert_files_to_dicts, fetch_archive_from_http, print_answers
|
| 24 |
from haystack.nodes import FARMReader, TransformersReader
|
|
|
|
| 25 |
from haystack.utils import launch_es
|
| 26 |
|
| 27 |
launch_es()
|
|
|
|
| 28 |
! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
|
| 29 |
! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
|
| 30 |
! chown -R daemon:daemon elasticsearch-7.9.2
|
|
|
|
| 35 |
stdout=PIPE, stderr=STDOUT,
|
| 36 |
preexec_fn=lambda: os.setuid(1) # as daemon
|
| 37 |
)
|
|
|
|
| 38 |
! sleep 30
|
|
|
|
| 39 |
|
| 40 |
from haystack.document_stores import ElasticsearchDocumentStore
|
| 41 |
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document")
|
|
|
|
| 53 |
from haystack.document_stores import FAISSDocumentStore
|
| 54 |
from haystack.nodes import RAGenerator, DensePassageRetriever
|
| 55 |
|
|
|
|
| 56 |
titles = list(df["document_identifier"].values)
|
| 57 |
texts = list(df["document_text"].values)
|
| 58 |
documents: List[Document] = []
|
|
|
|
| 65 |
}
|
| 66 |
)
|
| 67 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
document_store.write_documents(documents)
|
| 69 |
|
| 70 |
from haystack.nodes import ElasticsearchRetriever
|
|
|
|
| 73 |
|
| 74 |
from haystack.pipelines import ExtractiveQAPipeline
|
| 75 |
pipe = ExtractiveQAPipeline(reader, retriever)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
prediction = pipe.run(
|
| 77 |
query="What is post-COVID?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}}
|
| 78 |
)
|
|
|
|
|
|
|
| 79 |
from pprint import pprint
|
| 80 |
|
| 81 |
pprint(prediction)
|