Update README.md
Browse files
README.md
CHANGED
|
@@ -25,7 +25,7 @@ The following snippet demonstrates a quick way that SpEL can be used to generate
|
|
| 25 |
```python
|
| 26 |
# download SpEL from https://github.com/shavarani/SpEL
|
| 27 |
from transformers import AutoTokenizer
|
| 28 |
-
from spel.model import SpELAnnotator
|
| 29 |
from spel.configuration import device
|
| 30 |
from spel.utils import get_subword_to_word_mapping
|
| 31 |
from spel.span_annotation import WordAnnotation, PhraseAnnotation
|
|
@@ -45,6 +45,8 @@ subword_annotations = spel.annotate_subword_ids(inputs.input_ids, k_for_top_k_to
|
|
| 45 |
# #################################### CREATE WORD-LEVEL ANNOTATIONS ##################################################
|
| 46 |
tokens_offsets = token_offsets[1:-1]
|
| 47 |
subword_annotations = subword_annotations[1:]
|
|
|
|
|
|
|
| 48 |
word_annotations = [WordAnnotation(subword_annotations[m[0]:m[1]], tokens_offsets[m[0]:m[1]])
|
| 49 |
for m in get_subword_to_word_mapping(inputs.tokens(), sentence)]
|
| 50 |
# ################################## CREATE PHRASE-LEVEL ANNOTATIONS ##################################################
|
|
@@ -56,6 +58,9 @@ for w in word_annotations:
|
|
| 56 |
phrase_annotations[-1].add(w)
|
| 57 |
else:
|
| 58 |
phrase_annotations.append(PhraseAnnotation(w))
|
|
|
|
|
|
|
|
|
|
| 59 |
```
|
| 60 |
|
| 61 |
|
|
|
|
| 25 |
```python
|
| 26 |
# download SpEL from https://github.com/shavarani/SpEL
|
| 27 |
from transformers import AutoTokenizer
|
| 28 |
+
from spel.model import SpELAnnotator, dl_sa
|
| 29 |
from spel.configuration import device
|
| 30 |
from spel.utils import get_subword_to_word_mapping
|
| 31 |
from spel.span_annotation import WordAnnotation, PhraseAnnotation
|
|
|
|
| 45 |
# #################################### CREATE WORD-LEVEL ANNOTATIONS ##################################################
|
| 46 |
tokens_offsets = token_offsets[1:-1]
|
| 47 |
subword_annotations = subword_annotations[1:]
|
| 48 |
+
for sa in subword_annotations:
|
| 49 |
+
sa.idx2tag = dl_sa.mentions_itos
|
| 50 |
word_annotations = [WordAnnotation(subword_annotations[m[0]:m[1]], tokens_offsets[m[0]:m[1]])
|
| 51 |
for m in get_subword_to_word_mapping(inputs.tokens(), sentence)]
|
| 52 |
# ################################## CREATE PHRASE-LEVEL ANNOTATIONS ##################################################
|
|
|
|
| 58 |
phrase_annotations[-1].add(w)
|
| 59 |
else:
|
| 60 |
phrase_annotations.append(PhraseAnnotation(w))
|
| 61 |
+
# ################################## PRINT OUT THE CREATED ANNOTATIONS ################################################
|
| 62 |
+
for phrase_annotation in phrase_annotations:
|
| 63 |
+
print(dl_sa.mentions_itos[phrase_annotation.resolved_annotation])
|
| 64 |
```
|
| 65 |
|
| 66 |
|