tested env

app.py

@@ -5,7 +5,7 @@ from huggingface_hub import hf_hub_download
 import streamlit as st
 
 
-@st.cache
+@st.cache(allow_output_mutation=True)
 def load_model():
     fpath = hf_hub_download("OpenDungeon/gpt-j-8bit-ffbgem", "model.pt")
     qmodel = torch.load(fpath)
@@ -44,8 +44,8 @@ def PrintContinuation(prompt, local_model, single_hook=None, batch=1, limit_toke
 
 
 tokenizer, model = load_model()
-text = st.text_area("Prefix")
-batch = st.number_input("Variants", value=1)
+text = st.text_area("Prefix", value="DM: You enter the room.")
+batch = st.number_input("Variants", value=5)
 
 t = st.empty()
 firstline = ""
@@ -53,8 +53,13 @@ firstline = ""
 def PrintSome(text):
     global t, firstline
     firstline += text
-    t.markdown(f"## {firstline}...")
+    t.markdown(f"{firstline}...")
 
 choices, batch_time = PrintContinuation(text, model, PrintSome, batch, 50)
 
-t.markdown("  \n\n".join(choices) + f"  \n\nBatch:Seconds per batch: {batch_time}, Batch: {batch}")
+final_page = ""
+for i in range(batch):
+    final_page += f"## choice №{i + 1}  \n{choices[i]}  \n______  \n"
+final_page = f"Seconds per batch: {batch_time}, Batch: {batch}"
+
+t.markdown(final_page)

requirements.txt

@@ -1,3 +1,3 @@
 transformers
 torch
-gradio
+streamlit