Upload folder using huggingface_hub

.env

@@ -0,0 +1 @@
+OPENAI_API_KEY="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"

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

README.md

@@ -1,12 +1,6 @@
 ---
-title: Translation Game
-emoji: 🦀
-colorFrom: purple
-colorTo: purple
+title: translation_game
+app_file: app/app.py
 sdk: gradio
-sdk_version: 4.0.0
-app_file: app.py
-pinned: false
+sdk_version: 3.37.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app/app.py

@@ -0,0 +1,234 @@
+import gradio as gr
+import emoji
+from random import shuffle
+from loguru import logger
+from dotenv import load_dotenv
+load_dotenv()
+from retry import retry
+from typing import List, Dict
+
+# 
+#  -- llm.py --
+#  
+
+load_dotenv()
+from retry import retry
+from typing import List, Dict
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+country = {"nl": {"language":"Dutch", "country":"Netherlands"},
+            "de": {"language":"German", "country":"Germany"},
+            "se": {"language":"Swedish", "country":"Sweden"},
+            "en": {"language":"English", "country":"England"}
+            }
+            
+levels = {"easy": [3, 6], 
+          "medium": [5, 12],
+          "hard": [12, 20]}
+
+
+def get_languages():
+    return list(country.keys())
+
+def get_level():
+    return list(levels.keys())
+
+def parse_response(response:dict, min_length:int=10) -> List[Dict[str, str]]:
+
+    logger.debug(f"{response = }")
+
+    logger.info(f"Parsing response from OpenAI API.")
+
+    text = response.choices[0].message.content
+    tokens = text.split("Sentence ")
+
+    logger.debug(f"Number of tokens: {len(tokens)}")
+    logger.debug(f"{tokens = }")
+
+
+    qas = []
+    for token in tokens:
+        if len(token) == 0:
+            continue
+        lines = token.split("\n")
+        #print(f"{lines = }")
+        original = lines[0][3:]
+
+        true_trans = ""
+        for line in lines:
+            if line.startswith("True: "):
+                true_trans = line[6:]
+
+        # get line starting with "False: "
+        false_trans = ""
+        for line in lines:
+            if line.startswith("False: "):
+                false_trans = line[7:]
+
+        # get line starting with "Funny: "
+        funny_trans = ""
+        for line in lines:
+            if line.startswith("Funny: "):
+                funny_trans = line[7:]
+
+        qa = {"original": original, "true": true_trans, "false": false_trans, "funny": funny_trans}
+
+        logger.debug(f"------------------------")
+        logger.debug(f"{qa = }")
+        logger.debug(f"------------------------")
+
+        if len(qa["original"]) > min_length and len(qa["true"]) > min_length and len(qa["false"]) > min_length and len(qa["funny"]) > min_length:
+            qas.append(qa)
+
+    logger.info(f"Returning {len(qas)} valid QA pairs.")
+
+    return qas
+
+@retry(delay=0, backoff=2, max_delay=20, tries=10, logger=logger)
+def generate(n:int, input_country:str, target_country:str, level:str, temperature:float=0.80) -> Dict[str, str]:
+    input_length = levels[level]
+
+    logger.info(f"Generating {str(n)} QA pairs for {country[input_country]['language']} to {country[target_country]['language']} with level: {level}.")
+
+    assert temperature >= 0.0 and temperature <= 1.0, "temperature must be between 0 and 1"
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": 
+            f"You are a helpful assistant at creating a translation game.\
+                You create sentences in {str(country[input_country]['language'])} with a length that is exactly between {str(input_length[0])} to {str(input_length[1])} words.\
+                You create translations in  {str(country[target_country]['language'])}.\
+            Formulate your answer in exactly this format: Sentence N: [X],\True: [A],\nFalse: [B],\nFunny: [C]."},
+            {"role": "assistant", "content": 
+            f"Create original sentences with words, locations, concepts and phrases are typical for {str(country[input_country]['country'])} ."},
+            {"role": "user", "content": 
+            f"Generate {str(n)} funny sentences. You create 1 correct translation, 1 incorrect translations, and 1 which very wrong and funny."},
+        ],
+        temperature=temperature,
+        
+    )
+    return response
+
+
+
+# n = 10
+# input_country = "nl"
+# target_country = "de"
+# input_length = "easy"
+
+def get_QAs(n:int, input_country:str, target_country:str, level:str, debug:bool):
+  
+    if debug:
+        return [ {"original": "The Netherlands is a country in Europe.",
+                    "true": "Nederland is een land in Europa.",
+                    "false": "Nederland is een land in Azië.",
+                    "funny": "Nederland is een aap in Europa."},
+                {"original": "Aap, noot, mies.",
+                    "true": "Aap.",
+                    "false": "Noot.",
+                    "funny": "Mies."} ]
+    else:
+        return parse_response(generate(n=n, input_country=input_country, target_country=target_country, level=level))
+    
+# 
+#  -- gui.py --
+#  
+languages = get_languages()
+levels = get_level()
+
+global input_language
+input_language = "en"
+global target_language
+target_language = "nl"
+global level
+level = "medium"
+global debug
+debug = False
+global share
+share = True
+
+qas = llm.get_QAs(n=10, input_country=input_language, target_country=target_language, level=level, debug=debug)
+
+def get_qa(n:int=1, input_country:str=input_language, target_country:str=target_language, level:str=level, debug:bool=debug):
+    logger.info(f"QAs left: {len(qas)}")
+    qa = qas.pop(0)
+
+    if len(qas) < 2:
+        logger.info(f"Generating new QAs...")
+        logger.debug(f"{debug = }")
+        qas.extend(llm.get_QAs(n=n,
+                            input_country=input_country,
+                            target_country=target_country,
+                            level=level,
+                            debug=debug))
+
+    question_md = f"## {qa['original']}"
+    options = [qa["true"], qa["false"], qa["funny"]]
+    true = qa["true"]
+    shuffle(options)
+    options_md = f"""### A: {options[0]}\n\n### B: {options[1]}\n\n### C: {options[2]}"""
+
+    global correct_answer
+    correct_answer = [letter for letter, option in zip(["A", "B", "C"], options) if option == true][0] # set global
+
+    logger.info(f"get_qa()")
+    logger.info(f"Question: {question_md}")
+    logger.info(f"Options: {options}")
+    logger.info(f"Correct answer: {correct_answer}")
+    return question_md, options_md
+    
+def update(answer):
+
+    # check answer
+    logger.info(f"Checking answer \"{answer}\" against \"{correct_answer}\"")
+    result = answer == correct_answer
+    # display output
+    gr.Info(emoji.emojize(":thumbs_up:"), ) if result else gr.Info(emoji.emojize(":thumbs_down:"))
+    #output = emoji.emojize("# :thumbs_up:") if result else emoji.emojize("# :thumbs_down:")
+
+    # update question
+    question, options = get_qa()
+
+    logger.info(f"update()")
+    logger.info(f"Question: {question}")
+    logger.info(f"Options: {options}")
+
+    return question, options
+    
+
+def get_interface(question, options):
+
+    with gr.Blocks() as blocks:
+        with gr.Column():
+            # show question
+            question_md = gr.Markdown(question)
+
+            # show answers
+            answers_md = gr.Markdown(options)
+
+            # add radio buttons
+            radio = gr.Radio(choices=["A", "B", "C"], label=None, show_label=False, info=None)
+
+            # add submit button
+            button = gr.Button("Submit")
+
+            # # show output box
+            # output = gr.Markdown()
+
+            # event listener
+            button.click(fn=update, inputs=radio, outputs=[question_md, answers_md], scroll_to_output=True)
+
+
+        with gr.Accordion("See Details", open=False):
+            # Advanced options
+            input_language = gr.Radio(languages, label="Input language", value="se", interactive=True)
+            target_language = gr.Radio(languages, label="Output language", value="nl", interactive=True)
+            level = gr.Radio(levels, label="Level", value="medium", interactive=True)
+            
+    return blocks.queue().launch(share=share, debug=debug)
+
+
+question, options = get_qa(n=1, input_country=input_language, target_country=target_language, level=level, debug=debug)
+interface = get_interface(question, options)

requirements.txt

@@ -0,0 +1,4 @@
+emoji
+gradio
+dotenv
+openai

src/game/gui.py

@@ -0,0 +1,103 @@
+import gradio as gr
+import emoji
+from random import shuffle
+from . import llm
+from loguru import logger
+
+languages = llm.get_languages()
+levels = llm.get_level()
+
+global input_language
+input_language = "en"
+global target_language
+target_language = "nl"
+global level
+level = "medium"
+global debug
+debug = False
+global share
+share = True
+
+qas = llm.get_QAs(n=10, input_country=input_language, target_country=target_language, level=level, debug=debug)
+
+def get_qa(n:int=1, input_country:str=input_language, target_country:str=target_language, level:str=level, debug:bool=debug):
+    logger.info(f"QAs left: {len(qas)}")
+    qa = qas.pop(0)
+
+    if len(qas) < 2:
+        logger.info(f"Generating new QAs...")
+        logger.debug(f"{debug = }")
+        qas.extend(llm.get_QAs(n=n,
+                            input_country=input_country,
+                            target_country=target_country,
+                            level=level,
+                            debug=debug))
+
+    question_md = f"## {qa['original']}"
+    options = [qa["true"], qa["false"], qa["funny"]]
+    true = qa["true"]
+    shuffle(options)
+    options_md = f"""### A: {options[0]}\n\n### B: {options[1]}\n\n### C: {options[2]}"""
+
+    global correct_answer
+    correct_answer = [letter for letter, option in zip(["A", "B", "C"], options) if option == true][0] # set global
+
+    logger.info(f"get_qa()")
+    logger.info(f"Question: {question_md}")
+    logger.info(f"Options: {options}")
+    logger.info(f"Correct answer: {correct_answer}")
+    return question_md, options_md
+    
+def update(answer):
+
+    # check answer
+    logger.info(f"Checking answer \"{answer}\" against \"{correct_answer}\"")
+    result = answer == correct_answer
+    # display output
+    gr.Info(emoji.emojize(":thumbs_up:"), ) if result else gr.Info(emoji.emojize(":thumbs_down:"))
+    #output = emoji.emojize("# :thumbs_up:") if result else emoji.emojize("# :thumbs_down:")
+
+    # update question
+    question, options = get_qa()
+
+    logger.info(f"update()")
+    logger.info(f"Question: {question}")
+    logger.info(f"Options: {options}")
+
+    return question, options
+    
+
+def get_interface(question, options):
+
+    with gr.Blocks() as blocks:
+        with gr.Column():
+            # show question
+            question_md = gr.Markdown(question)
+
+            # show answers
+            answers_md = gr.Markdown(options)
+
+            # add radio buttons
+            radio = gr.Radio(choices=["A", "B", "C"], label=None, show_label=False, info=None)
+
+            # add submit button
+            button = gr.Button("Submit")
+
+            # # show output box
+            # output = gr.Markdown()
+
+            # event listener
+            button.click(fn=update, inputs=radio, outputs=[question_md, answers_md], scroll_to_output=True)
+
+
+        with gr.Accordion("See Details", open=False):
+            # Advanced options
+            input_language = gr.Radio(languages, label="Input language", value="se", interactive=True)
+            target_language = gr.Radio(languages, label="Output language", value="nl", interactive=True)
+            level = gr.Radio(levels, label="Level", value="medium", interactive=True)
+            
+    return blocks.queue().launch(share=share, debug=debug)
+
+
+question, options = get_qa(n=1, input_country=input_language, target_country=target_language, level=level, debug=debug)
+interface = get_interface(question, options)

src/game/llm.py

@@ -0,0 +1,125 @@
+import openai
+import os
+from loguru import logger
+from dotenv import load_dotenv
+load_dotenv()
+import gradio as gr
+from retry import retry
+from typing import List, Dict
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+country = {"nl": {"language":"Dutch", "country":"Netherlands"},
+            "de": {"language":"German", "country":"Germany"},
+            "se": {"language":"Swedish", "country":"Sweden"},
+            "en": {"language":"English", "country":"England"}
+            }
+            
+levels = {"easy": [3, 6], 
+          "medium": [5, 12],
+          "hard": [12, 20]}
+
+
+def get_languages():
+    return list(country.keys())
+
+def get_level():
+    return list(levels.keys())
+
+def parse_response(response:dict, min_length:int=10) -> List[Dict[str, str]]:
+
+    logger.debug(f"{response = }")
+
+    logger.info(f"Parsing response from OpenAI API.")
+
+    text = response.choices[0].message.content
+    tokens = text.split("Sentence ")
+
+    logger.debug(f"Number of tokens: {len(tokens)}")
+    logger.debug(f"{tokens = }")
+
+
+    qas = []
+    for token in tokens:
+        if len(token) == 0:
+            continue
+        lines = token.split("\n")
+        #print(f"{lines = }")
+        original = lines[0][3:]
+
+        true_trans = ""
+        for line in lines:
+            if line.startswith("True: "):
+                true_trans = line[6:]
+
+        # get line starting with "False: "
+        false_trans = ""
+        for line in lines:
+            if line.startswith("False: "):
+                false_trans = line[7:]
+
+        # get line starting with "Funny: "
+        funny_trans = ""
+        for line in lines:
+            if line.startswith("Funny: "):
+                funny_trans = line[7:]
+
+        qa = {"original": original, "true": true_trans, "false": false_trans, "funny": funny_trans}
+
+        logger.debug(f"------------------------")
+        logger.debug(f"{qa = }")
+        logger.debug(f"------------------------")
+
+        if len(qa["original"]) > min_length and len(qa["true"]) > min_length and len(qa["false"]) > min_length and len(qa["funny"]) > min_length:
+            qas.append(qa)
+
+    logger.info(f"Returning {len(qas)} valid QA pairs.")
+
+    return qas
+
+@retry(delay=0, backoff=2, max_delay=20, tries=10, logger=logger)
+def generate(n:int, input_country:str, target_country:str, level:str, temperature:float=0.80) -> Dict[str, str]:
+    input_length = levels[level]
+
+    logger.info(f"Generating {str(n)} QA pairs for {country[input_country]['language']} to {country[target_country]['language']} with level: {level}.")
+
+    assert temperature >= 0.0 and temperature <= 1.0, "temperature must be between 0 and 1"
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": 
+            f"You are a helpful assistant at creating a translation game.\
+                You create sentences in {str(country[input_country]['language'])} with a length that is exactly between {str(input_length[0])} to {str(input_length[1])} words.\
+                You create translations in  {str(country[target_country]['language'])}.\
+            Formulate your answer in exactly this format: Sentence N: [X],\True: [A],\nFalse: [B],\nFunny: [C]."},
+            {"role": "assistant", "content": 
+            f"Create original sentences with words, locations, concepts and phrases are typical for {str(country[input_country]['country'])} ."},
+            {"role": "user", "content": 
+            f"Generate {str(n)} funny sentences. You create 1 correct translation, 1 incorrect translations, and 1 which very wrong and funny."},
+        ],
+        temperature=temperature,
+        
+    )
+    return response
+
+
+
+# n = 10
+# input_country = "nl"
+# target_country = "de"
+# input_length = "easy"
+
+def get_QAs(n:int, input_country:str, target_country:str, level:str, debug:bool):
+  
+    if debug:
+        return [ {"original": "The Netherlands is a country in Europe.",
+                    "true": "Nederland is een land in Europa.",
+                    "false": "Nederland is een land in Azië.",
+                    "funny": "Nederland is een aap in Europa."},
+                {"original": "Aap, noot, mies.",
+                    "true": "Aap.",
+                    "false": "Noot.",
+                    "funny": "Mies."} ]
+    else:
+        return parse_response(generate(n=n, input_country=input_country, target_country=target_country, level=level))

transfer.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Source and destination paths
+source_file="models/llama-2-7b-chat.ggmlv3.q4_0.bin"
+destination_host="walter@katana2"
+destination_path="/home/walter/repos/translation_game/models"
+
+# rsync command with options
+rsync -avz --partial --progress --append-verify -L -e "ssh" "$source_file" "$destination_host:$destination_path"