| | from transformers import AutoTokenizer, AutoModelForMaskedLM |
| | from transformers import RobertaTokenizer, RobertaTokenizerFast, RobertaForMaskedLM, pipeline |
| | import torch |
| |
|
| | def evaluate(framework): |
| | text = "På biblioteket kan du [MASK] en bok." |
| | if framework == "flax": |
| | model = AutoModelForMaskedLM.from_pretrained("./", from_flax=True) |
| | elif framework == "tensorflow": |
| | model = AutoModelForMaskedLM.from_pretrained("./", from_tf=True) |
| | else: |
| | model = AutoModelForMaskedLM.from_pretrained("./") |
| |
|
| | print("Testing with AutoTokenizer") |
| | tokenizer = AutoTokenizer.from_pretrained("./") |
| | my_unmasker_pipeline = pipeline('fill-mask', model=model, tokenizer=tokenizer) |
| | output = my_unmasker_pipeline(text) |
| | print(output) |
| |
|
| | |
| | |
| | |
| | |
| | |
| |
|
| | |
| | |
| | |
| | |
| | |
| |
|
| |
|
| | print("Evaluating PyTorch Model") |
| | evaluate("pytorch") |
| |
|
| |
|
| | |
| | |
| |
|
