new

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 import requests
 import json
 import os
+from transformers import 
 from screenshot import (
     before_prompt,
     prompt_to_generation,
@@ -10,11 +11,13 @@ from screenshot import (
     js_load_script,
 )
 from spaces_info import description, examples, initial_prompt_value
+from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, set_seed
+
 
 #API_URL = os.getenv("API_URL")
 #HF_API_TOKEN = os.getenv("HF_API_TOKEN")
 
-API_URL = "https://api-inference.huggingface.co/models/bigscience/bloom-560m"
+API_URL = "https://api-inference.huggingface.co/models/hf-internal-testing/tiny-random-bloom"
 HF_API_TOKEN = "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"
 
 def query(payload):
@@ -30,7 +33,7 @@ def inference(input_sentence, max_length, sample_or_greedy, seed=42):
             "max_new_tokens": max_length,
             "top_p": 0.9,
             "do_sample": True,
-            "seed": seed,
+            #"seed": seed,
             "early_stopping": False,
             "length_penalty": 0.0,
             "eos_token_id": None,
@@ -39,29 +42,43 @@ def inference(input_sentence, max_length, sample_or_greedy, seed=42):
         parameters = {
             "max_new_tokens": max_length,
             "do_sample": False,
-            "seed": seed,
+            #"seed": seed,
             "early_stopping": False,
             "length_penalty": 0.0,
             "eos_token_id": None,
         }
 
     payload = {"inputs": input_sentence, "parameters": parameters,"options" : {"use_cache": False} }
+    model_name = 'bigscience/bloomz-560m'
+    pipe = pipeline("text-generation", 
+        model = model_name, 
+        tokenizer = model_name, 
+        max_new_tokens = max_length, 
+        do_sample = False, 
+        length_penalty = 0.0, 
+        early_stopping = False, 
+        eos_token_id = None
+    )
+    res = pipe(input_sentence)
+    generation = res["generated_text"]
 
-    data = query(payload)
+    #data = query(payload)
 
-    if "error" in data:
-        return (None, None, f"<span style='color:red'>ERROR: {data['error']} </span>")
+    #if "error" in data:
+    #    return (None, None, f"<span style='color:red'>ERROR: {data['error']} </span>")
 
-    generation = data[0]["generated_text"].split(input_sentence, 1)[1]
-    return (
-        before_prompt
-        + input_sentence
-        + prompt_to_generation
-        + generation
-        + after_generation,
-        data[0]["generated_text"],
-        "",
-    )
+    #generation = data[0]["generated_text"].split(input_sentence, 1)[1]
+
+    # return (
+    #     before_prompt
+    #     + input_sentence
+    #     + prompt_to_generation
+    #     + generation
+    #     + after_generation,
+    #     data[0]["generated_text"],
+    #     "",
+    # )
+    return generation
 
 
 if __name__ == "__main__":

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+accelerate