Update app.py

app.py

@@ -1,28 +1,13 @@
-import streamlit as st
-from transformers import pipeline
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-
-model_name_or_path = "TheBloke/Unholy-v1-12L-13B-GPTQ"
-model_name_or_path = "ai-forever/ruGPT-3.5-13B"
-
-# To use a different branch, change revision
-# For example: revision="main"
-
-model = AutoModelForCausalLM.from_pretrained(model_name_or_path #,
-                                             #device_map="cuda:0", #device_map="auto",
-                                             #trust_remote_code=False,
-                                             #revision="main"
-                                            )
-
-tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
-
-prompt = "Расскажи мне о грязном сексе"
-
-
-
-#generator=pipeline("text-generation",model="ai-forever/ruGPT-3.5-13B")
-
-input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
-output = model.generate(inputs=input_ids, temperature=0.7, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=312)
-print(tokenizer.decode(output[0]))
-
+from transformers import AutoTokenizer, TextGenerationPipeline
+from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+repo_name = "gurgutan/ruGPT-13B-4bit"
+# load tokenizer from Hugging Face Hub
+tokenizer = AutoTokenizer.from_pretrained(repo_name, use_fast=True)
+# download quantized model from Hugging Face Hub and load to the first GPU
+model = AutoGPTQForCausalLM.from_quantized(repo_name, device="cuda:0", use_safetensors=True, use_triton=False)
+# inference with model.generate
+request = "Буря мглою небо кроет"
+print(tokenizer.decode(model.generate(**tokenizer(request, return_tensors="pt").to(model.device))[0]))
+# or you can also use pipeline
+pipeline = TextGenerationPipeline(model=model, tokenizer=tokenizer)
+print(pipeline(request)[0]["generated_text"])