from transformers import pipeline
folder_dir = "DarkBERT"
unmasker = pipeline('fill-mask', model=folder_dir)
unmasker("RagnarLocker, LockBit, and REvil are types of <mask>.")


from transformers import AutoModel, AutoTokenizer
model = AutoModel.from_pretrained(folder_dir)
tokenizer = AutoTokenizer.from_pretrained(folder_dir)
text = "Recent research has suggested that there are clear differences in the language used in the Dark Web compared to that of the Surface Web."
encoded = tokenizer(text, return_tensors="pt")
output = model(**encoded)
output[0].shape