Create app.py

app.py

@@ -0,0 +1,81 @@
+import os
+import json
+import shutil
+import gradio as gr
+from datasets import load_dataset
+from huggingface_hub import Repository
+
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+
+def load_data():
+    dataset = load_dataset("ArmelR/oasst1_guanaco_english", use_auth_token=HF_TOKEN)
+    return dataset
+
+samples = load_data()
+splits = list(samples.keys())
+languages = ["Wolof"]
+
+custom_css = """
+#banner-image {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+}
+#chat-message {
+    font-size: 14px;
+    min-height: 300px;
+}
+"""
+def caller_split(s):
+    return 0, samples[s][0]["prompt"], samples[s][0]["completion"]
+
+def identity(index, split):
+    ds = samples[split][index]
+    return ds["prompt"], ds["completion"]
+    
+def save(index, language, split, prompt, completion):
+    with open("./output.jsonl", "a") as fout :
+        fout.write(
+            json.dumps(
+                {
+                    "prompt" : prompt,
+                    "completion" : completion,
+                    "language" : language,
+                    "index" : index
+                }
+            )+"\n"
+        )
+    next_index = min(1+index, len(samples[split]-1))
+    return next_index, ds[split][next_index]["prompt"], ds[split][next_index]["completion"]
+
+with gr.Blocks(analytics_enabled=False, css=custom_css) as demo:
+    gr.HTML("""<h1 align="center">Self-instruct StarCoder 💫</h1>""")
+    gr.Markdown(
+        """
+        This space provides a visualization tool for the samples from this [dataset](https://huggingface.co/datasets/codeparrot/self-instruct-starcoder) generated by applying
+        the self-instruct procedure to Starcoder💫. For each instruction, we have its ouput, as well as some instructions generated along the way that are similar to it with the corresponding
+        score.
+        """
+        )
+    with gr.Row() : 
+        split = gr.Dropdown(choices=splits, label="Dataset split", value=splits[0])
+    with gr.Row() :
+        index_example = gr.Slider(minimum=0, maximum=10000, step=1, value=0, interactive=True, info=f"Index of the chosen instruction-output pair.")
+        with gr.Column():
+            prompt = gr.Textbox(label="prompt")
+        with gr.Column()
+            completion = gr.Textbox(label="Completion")
+    with gr.Row() :
+        language = gr.Dropdown(choices=splits, label="Translation language", value=languages[0])
+        with gr.Column() :
+            translated_prompt = gr.Textbox(label="Translated prompt")
+        with gr.Column() :
+            translated_completion = gr.Textbox(label="Translated completion")
+    with gr.Row() :
+        button = gr.Button(value="Submit")
+    
+    split.change(caller_split, inputs=[split], outputs=[index_example, prompt, completion])
+    index_example.release(identity, inputs=[index_example, split], outputs=[prompt, completion])
+    btn.click(save, inputs=[index_example, language, split, translated_prompt, translated_completion], outputs=[index_example, prompt, completion])
+    
+demo.queue(concurrency_count=16).launch(debug=True)