Update README.md
#1
by
MassMin
- opened
README.md
CHANGED
|
@@ -118,44 +118,44 @@ The model's performance is evaluated using the F1 score for NER. The predictions
|
|
| 118 |
[More Information Needed]
|
| 119 |
|
| 120 |
## Evaluation
|
| 121 |
-
'''python
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
|
| 160 |
|
| 161 |
<!-- This section describes the evaluation protocols and provides the results. -->
|
|
|
|
| 118 |
[More Information Needed]
|
| 119 |
|
| 120 |
## Evaluation
|
| 121 |
+
#'''python
|
| 122 |
+
import torch
|
| 123 |
+
|
| 124 |
+
from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
|
| 125 |
+
|
| 126 |
+
import pandas as pd
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
model_checkpoint = "MassMin/xlm-roberta-base-finetuned-panx-de" # Replace with your Hugging Face model name
|
| 130 |
+
|
| 131 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
|
| 135 |
+
|
| 136 |
+
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint).to(device)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
ner_pipeline = pipeline("ner", model=model, tokenizer=tokenizer, framework="pt", device=0 if torch.cuda.is_available() else -1)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def tag_text_with_pipeline(text, ner_pipeline):
|
| 143 |
+
|
| 144 |
+
# Use the NER pipeline to get predictions
|
| 145 |
+
results = ner_pipeline(text)
|
| 146 |
+
|
| 147 |
+
# Convert results to a DataFrame for easy viewing
|
| 148 |
+
df = pd.DataFrame(results)
|
| 149 |
+
df = df[['word', 'entity', 'score']]
|
| 150 |
+
df.columns = ['Tokens', 'Tags', 'Score'] # Rename columns for clarity
|
| 151 |
+
return df
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
text = "Jeff Dean works at Google in California."
|
| 155 |
+
|
| 156 |
+
result = tag_text_with_pipeline(text, ner_pipeline)
|
| 157 |
+
|
| 158 |
+
print(result)
|
| 159 |
|
| 160 |
|
| 161 |
<!-- This section describes the evaluation protocols and provides the results. -->
|