first commit

app.py

@@ -49,5 +49,5 @@ with gr.Blocks() as block:
         outputs=outputs,
         queue=True
     )
-block.queue(concurrency_count=5).launch(server_name="localhost", share=True)
-# block.queue().launch()
+# block.queue(concurrency_count=5).launch(server_name="localhost", share=True)
+block.queue().launch()

sample.py

@@ -17,9 +17,9 @@ max_new_tokens = 500 # number of tokens generated in each sample
 temperature = 0.8 # 1.0 = no change, < 1.0 = less random, > 1.0 = more random, in predictions
 top_k = 200 # retain only the top_k most likely tokens, clamp others to have 0 probability
 seed = 1337
-device = 'cuda' # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1', etc.
+device = 'cpu' # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1', etc.
 dtype = 'bfloat16' # 'float32' or 'bfloat16' or 'float16'
-compile = False # use PyTorch 2.0 to compile the model to be faster
+compile = True # use PyTorch 2.0 to compile the model to be faster
 exec(open('configurator.py').read()) # overrides from command line or config file
 # -----------------------------------------------------------------------------