| https://colab.research.google.com/drive/1Dlm8FA9JjjcqJIkfCagaIQWex8Ho5IKI#scrollTo=e8xIjRNsl3Bb | |
| ``` | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| tokenizer = AutoTokenizer.from_pretrained("GeneralRincewind/ShakespeareGPT") | |
| model = AutoModelForCausalLM.from_pretrained("GeneralRincewind/ShakespeareGPT") | |
| #### Generate text | |
| from transformers import TextStreamer | |
| tokenized_text = tokenizer("", return_tensors="pt", truncation=True) | |
| input_ids = tokenized_text.input_ids | |
| streamer = TextStreamer(tokenizer) | |
| model.eval() | |
| full_completion = model.generate(inputs=tokenized_text["input_ids"].to("cuda"), | |
| attention_mask=tokenized_text["attention_mask"].to("cuda"), | |
| temperature=0.9, | |
| top_k=80, | |
| top_p=0.65, | |
| do_sample=True, | |
| streamer=streamer, | |
| num_beams=1, | |
| max_new_tokens=500, | |
| eos_token_id=tokenizer.eos_token_id, | |
| pad_token_id=tokenizer.pad_token_id, | |
| repetition_penalty=1) | |
| decoded_text = tokenizer.decode(full_completion[0]) | |
| print(decoded_text) | |
| ``` |