GUI for evaluation feedback

.gitignore

@@ -1,2 +1,3 @@
 .sesskey
-.idea
+.idea
+data/*

eval_code.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import time
 
@@ -7,6 +8,20 @@ We will handle here the code evaluation phase.
 """
 import json
 
+
+EVAL_ANSWER_NOEVAL = 0
+EVAL_ANSWER_POSITIVE = 1
+EVAL_ANSWER_NEGATIVE = -1
+
+CODE_AUGMENTATIONS=[
+    ("NO_COMPILE", "The code provided is not a valid C code"),
+    ("DRY", "Don't repeat yourself."),
+    ("SRP", "Single object[function] responsability"),
+    ("MC", "Magic constants."),
+    ("NAME", "Meaningful names in the code."),
+]
+
+
 def clean_prompt_answer(answer):
     """
     Chatgpt4 is ok, does not pollute the code but 3.5 encloses it in ```
@@ -48,6 +63,11 @@ def eval_code_by_chatgpt(openai_client, ccode):
     "criteria": "DRY",
     "explanation": "The memory allocation and initialization for ``p1``, ``p2``, and ``p3`` are repetitive. Consider creating a function like ``allocateAndInitializeMemory``."
   },
+  {
+    "criteria": "DRY",
+    "explanation": "Tne second DRY failure, because this is the observed ChatGPT behaviour."
+  },
+
   {
     "criteria": "SRP",
     "explanation": "The ``main`` function handles memory allocation, initialization, and printing. You should separate these responsibilities into different functions like ``allocateMemory``, ``initializeData``, and ``printData``."
@@ -63,15 +83,51 @@ def eval_code_by_chatgpt(openai_client, ccode):
         traceback.print_exc()
         return {"error":"There was an error while parsing the answer. Maybe ChatGPT is overloaded?"}
 
+
+
+def add_evaluation_fields_on_js_answer(json_answer, all_criterias = None):
+    """
+    Adds some JSON fields to store the human feedback.
+
+    The textual human feedback will always be in the 0 position.
+
+    :param json_answer:
+    :return:
+    """
+    if all_criterias is None:
+        all_criterias = CODE_AUGMENTATIONS
+    enhanced_answer = []
+    overall_feedback = {
+        "criteria":"HUMAN_FEEDBACK",
+        "explanation":"",
+        "EVAL": EVAL_ANSWER_NOEVAL
+    }
+
+    if all_criterias is not None:
+        existing = {c["criteria"] for c in json_answer}
+        for criteria in all_criterias:
+            if criteria[0] not in existing:
+                json_answer.append({"criteria":criteria[0], "explanation":"Not infringed"})
+
+    enhanced_answer.append(overall_feedback)
+    for ans in json_answer:
+        ans = copy.deepcopy(ans)
+        ans["EVAL"] = EVAL_ANSWER_NOEVAL
+        enhanced_answer.append(ans)
+
+    return enhanced_answer
+
 def eval_the_piece_of_c_code(openai_client, ccode):
     """
-    Main entrypoint to this module. Will be called from backend
+    Main entrypoint to this module. Will be called from backend. Will block so multithreading pls.
+
+    Will return a proprer json and will have EVAL fields, too.
 
     :param ccode:
     :return:
     """
     chatgpt_ans = eval_code_by_chatgpt(openai_client, ccode)
     chatgpt_js = parse_chatgpt_answer(chatgpt_ans)
-    return chatgpt_js
-
+    enhanced_answer = add_evaluation_fields_on_js_answer(chatgpt_js)
+    return enhanced_answer
 

main.py

@@ -1,12 +1,16 @@
 import json
 import uuid
-
-from fasthtml.common import *
 import os
-import eval_code
+import pathlib
+
 from datetime import datetime
+from fasthtml.common import *
 from sqlite_minutils.db import Database
 
+import eval_code
+
+
+
 # the secrets. Will be loaded from HF, or for docker --env-file or from IDE
 
 OAUTH_CLIENT_ID = os.environ.get('OAUTH_CLIENT_ID')
@@ -16,17 +20,34 @@ OPENID_PROVIDER_URL = os.environ.get('OPENID_PROVIDER_URL')
 SPACE_HOST = os.environ.get('SPACE_HOST')
 
 # The database. In memory bc it is efemerial on HF anyway.
+DATABASE_NAME = "data/sessions_meta.db"
+pathlib.Path(DATABASE_NAME).unlink(missing_ok=True)
+
 
-global_database = Database(memory=True, memory_name="mem_db_name")
+global_database = Database(DATABASE_NAME)
 global_database_tables = global_database.t
+
+# We store session specific feedback from registered users.
+# Will be later used to submit/storage the answer.
+question_evaluation_table = global_database_tables.question_answer
+if question_evaluation_table not in global_database_tables:
+    question_evaluation_table.create(id=int, code_text=str,
+                                     answer_eval_text=str, submitted=int, pk='id')
+Question_Evaluation_cls = question_evaluation_table.dataclass()
+
+# We store real-time session IDs and the state of the questions. Will link to question_answer
 session_state_table = global_database_tables.session_state
 if session_state_table not in global_database_tables:
     # session_id stored in cookies
     # see EVAL_STATUS_x below for state
     session_state_table.create(id=int, session_id=str, state=int, submitted=datetime,
-                               completed=datetime, answer=str, code=str, pk='id')
+                               completed=datetime, current_qeval=int, pk='id',)
+                               # Can't really nail the fk specs
+                               # foreign_keys=[("current_qeval", question_evaluation_table, "id")])
 Session_State_cls = session_state_table.dataclass()
 
+
+
 EVAL_STATE_NEW=0
 EVAL_STATE_QUERY=1
 EVAL_STATE_TIMEDOUT=2
@@ -49,28 +70,109 @@ else:
     app, rt = fast_app(debug=False, live=False, hdrs=hdrs)
     REFRESH_TIME = 1
 
-CODE_AUGMENTATIONS=[
-    ("DRY", "Don't repeat yourself."),
-    ("SRP", "Single object[function] responsability"),
-    ("MC", "Magic constants."),
-    ("NAME", "Meaningful names in the code."),
-]
+def html_create_feedback_updown_button(qe_id, ans_id, selected=0):
+    html_target_id = f"buttons_{ans_id}"
+    colors = ["grey", "blue"]
+    up_col = colors[0]
+    down_col = colors[0]
+    if selected == 1: up_col = colors[1]
+    if selected == -1: down_col = colors[1]
+    toggle_url = f"toggle_up_down/{qe_id}/{ans_id}/"
+    up = Button("+", hx_put=f"{toggle_url}?which=1", hx_swap="outerHTML",
+                hx_target="#" + html_target_id, style=f"background-color:{up_col}")
+    down = Button("-", hx_put=f"{toggle_url}?which=-1", hx_swap="outerHTML",
+                  hx_target="#" + html_target_id, style=f"background-color:{down_col}")
+    button_row = Div(up, down, _id=html_target_id)
+    return button_row
+
+
+def html_augment_evaluation_text_with_feedback(eval_html, qe_id, ans_id, selected=0):
+    """
+    Will plot the + / - buttons for feedback.
+
+    :param eval_html:
+    :param qe_id:
+    :param ans_id:
+    :return:
+    """
+
+    buttons = html_create_feedback_updown_button(qe_id, ans_id, selected)
+    final_div = Div(eval_html, buttons, style=" background-color: #f0f0f0;")
+    return final_div
+
+
+@rt("/toggle_up_down/{qe_id}/{ans_id}")
+def put(session, qe_id:int, ans_id:int, which:int):
+    """
+    Answer to the +/- button presses
+
+    :param session:
+    :param qe_id:
+    :param ans_id:
+    :param which:
+    :return:
+    """
+    print(qe_id, ans_id, which)
+    if which not in {-1, 1}:
+        return None
+    if 'session_id' not in session: return None
+    session_id = session["session_id"]
+    state_rows = session_state_table(limit=1, where=f"session_id == '{session_id}'", order_by="id DESC")
+    if len(state_rows) <= 0:
+        return None
+    answer_id = state_rows[0].current_qeval
+    qa_obj_row = question_evaluation_table(limit=1, where=f"id == {answer_id}")
+    if len(qa_obj_row) <= 0:
+        return None
+    qa_obj = qa_obj_row[0]
+    if qe_id != qa_obj.id:
+        print(f"QE {qe_id} does not belong to {qa_obj.id}")
+        return None
+    answer_eval_js = json.loads(qa_obj.answer_eval_text)
+    crt_selection = answer_eval_js[ans_id]["EVAL"]
+    input_button = which
+    out_selection = (input_button if crt_selection == 0 else (0 if crt_selection == input_button else input_button))
+    print(f"out selection: {out_selection}")
+    # store it back in DB
+    answer_eval_js[ans_id]["EVAL"] = out_selection
+    qa_obj.answer_eval_text = answer_eval_js
+    qa_obj.submitted = False # mark object as dirty
+    question_evaluation_table.upsert(qa_obj)
+
+    buttons= html_create_feedback_updown_button(qe_id, ans_id, selected=out_selection)
+    return buttons
+
+
+def html_get_textual_feedback_form(qe_obj):
+    form = Form(Input(), Button("Submit"))
+    div = Div(P("Give us a general feedback for the evaluation (optional)"), form)
+    return div
+
 
-def format_code_review(feedback_js, c_code, html_id=""):
+def html_format_code_review_form(qe_obj, html_id=""):
+    """
+    Formats the code review, adding fields for feedback if it is required.
+
+
+    :param feedback_js:
+    :param c_code:
+    :param html_id:
+    :return:
+    """
+    c_code = qe_obj.code_text
+    enhanced_answer = json.loads(qe_obj.answer_eval_text)
     list_of_citerias = []
-    for caug_code, caug_txt in CODE_AUGMENTATIONS:
+    for caug_code, caug_txt in eval_code.CODE_AUGMENTATIONS:
         crit_tag = [H3(caug_code), P(caug_txt)]
         list_of_citerias.extend(crit_tag)
         # yeah, I know . . .
-        criteria_found = False
-        for eval_line in feedback_js:
+        for k, eval_line in enumerate(enhanced_answer):
             if caug_code == eval_line["criteria"]:
                 eval_txt = P(eval_line["explanation"])
-                list_of_citerias.append(eval_txt)
-                criteria_found = True
-        if criteria_found is False:
-            list_of_citerias.append(P("Not infringed"))
-    return Div(html_render_code_output(c_code), *list_of_citerias, _id=html_id)
+                eval_txt_fb = html_augment_evaluation_text_with_feedback(eval_txt, qe_obj.id, k)
+                list_of_citerias.append(eval_txt_fb)
+    textual_feedback = html_get_textual_feedback_form(qe_obj)
+    return Div(html_render_code_output(c_code), *list_of_citerias, textual_feedback, _id=html_id)
 
 def html_default_results(html_id):
     return Div(P("This is where criterias will show up once the code is evaluated"), _id=html_id)
@@ -103,8 +205,12 @@ def html_render_answer_from_db(session_id, html_id):
     if eval_request_status == EVAL_STATE_NEW:
         return html_default_results(html_id), #, html_render_inputbox(HTML_RESULTS_AREA, HTML_SUBMIT_CODE_AREA)
     if eval_request_status == EVAL_STATE_ANSWER:
-        gpt_js_eval = json.loads(state_obj.answer)
-        return (format_code_review(gpt_js_eval, state_obj.code, html_id),
+        qe_obj_lst = question_evaluation_table(limit=1, where=f"id == {state_obj.current_qeval}")
+        if len(qe_obj_lst) < 1:
+            print(f"Object id {state_obj.current_qeval} can't be found in question_evaluation_table")
+            return (None,)
+        qe_obj = qe_obj_lst[0]
+        return (html_format_code_review_form(qe_obj, html_id),
                 html_render_inputbox(target_html_id=HTML_RESULTS_AREA, region_html_id=HTML_SUBMIT_CODE_AREA)) #, html_render_code_output(HTML_SUBMIT_CODE_AREA, state_obj.code)
     if eval_request_status == EVAL_STATE_QUERY:
         return html_waiting_for_results(html_id),
@@ -113,34 +219,52 @@ def html_render_answer_from_db(session_id, html_id):
 
 # How can I timeout? Well ... TBD.
 @threaded
-def call_gpt_and_store_result(pkid, code_to_check):
+def call_gpt_and_store_result(session_obj_id, code_to_check):
+    """
+    Threaded function that will submit code to LLM and wait for the answer.
+
+    Communication with "main" thread is through db.
+
+    All parameters must be pickable.
+
+    :param session_obj_id:
+    :param code_to_check:
+    :return:
+    """
+    # TODO refactor considering new join!
     # print("evaluatign code")
     try:
         # Pesky way to get a new cursor, in a thread safe way, into the db. This code runs in another thread.
         # Can we do better?
-        local_database = Database(memory=True, memory_name="mem_db_name")
-        local_cursor = local_database.t.session_state
-        Session_State_cls = local_cursor.dataclass()
-        local_sess_obj_lst = local_cursor(limit=1, where=f"id == {pkid}")
+        local_database = Database(DATABASE_NAME)
+        local_sess_state = local_database.t.session_state
+        local_sess_state_cls = local_sess_state.dataclass()
+        local_sess_obj_lst = local_sess_state(limit=1, where=f"id == {session_obj_id}")
         local_sess_obj = local_sess_obj_lst[0]
-        gpt_js_eval = eval_code.eval_the_piece_of_c_code(openai_client=None, ccode=code_to_check)
-        # print("done eval with ", gpt_js_eval)
-        # new_sess_obj = Session_State_cls(session_obj)
-        if "error" in gpt_js_eval:
+
+        # Trigger the lenghtly operation
+        enhanced_answer = eval_code.eval_the_piece_of_c_code(openai_client=None, ccode=code_to_check)
+
+        # we create a new QA entry.
+        qa_obj = Question_Evaluation_cls(code_text=code_to_check, answer_eval_text=enhanced_answer, submitted=0)
+        qa_obj = question_evaluation_table.insert(qa_obj)
+        local_sess_obj.current_qeval = qa_obj.id
+
+        # TODO save the outcome in a table, where it will be backed-up later. SqlLite is volatile.
+
+        if "error" in enhanced_answer:
             local_sess_obj.state = EVAL_STATE_ERROR
-            local_sess_obj.answer = gpt_js_eval["error"]
+            local_sess_obj.answer = enhanced_answer["error"]
             local_sess_obj.completed = datetime.utcnow()
         else:
             local_sess_obj.state = EVAL_STATE_ANSWER
-            local_sess_obj.answer = json.dumps(gpt_js_eval)
             local_sess_obj.completed = datetime.utcnow()
-        local_cursor.update(local_sess_obj)
+        local_sess_state.update(local_sess_obj)
     except:
         import traceback
         traceback.print_exc()
 
 
-
 def html_render_inputbox(target_html_id, region_html_id):
     txtarea = Textarea(id="ccodetoeval", name="ccodetoeval", placeholder="Enter a piece of C code", rows=3)
     form = Form(Group(txtarea, Button("Evaluate")),
@@ -190,6 +314,36 @@ def get(session):
                 P("!! The data will be saved and maybe made public !!"),
                 ]
 
+#     ############
+#     # !!! FOR DEBUGGING PURPOSES !!!
+#
+#     # insert an answer in db.
+#     code = """
+# #include <stdio.h>
+# int main() {
+#    // printf() displays the string inside quotation
+#    printf("Hello, World!");
+#    return 0;
+# }
+# """
+#     answer = eval_code.eval_the_piece_of_c_code(None, None)
+#     enhanced_answer = eval_code.add_evaluation_fields_on_js_answer(answer, CODE_AUGMENTATIONS)
+#     session_obj = Session_State_cls(
+#         session_id=session_id,
+#         state=EVAL_STATE_ANSWER,
+#         submitted=datetime.utcnow(),
+#         completed=datetime.utcnow(),
+#     )
+#
+#     # we create a new QA entry.
+#     qa_obj = Question_Evaluation_cls(code_text=code,
+#                                      answer_eval_text=enhanced_answer, submitted=0)
+#     qa_obj = question_evaluation_table.insert(qa_obj)
+#     session_obj.current_qeval = qa_obj.id
+#     session_obj = session_state_table.insert(session_obj)
+#
+#     ############
+
     input_area = html_render_inputbox(target_html_id=HTML_RESULTS_AREA, region_html_id=HTML_SUBMIT_CODE_AREA)
     results_area = html_render_answer_from_db(session_id, HTML_RESULTS_AREA)
     clear_area = render_clear_area(session_id, HTML_CLEAR_FORM)
@@ -211,9 +365,10 @@ def post(ccodetoeval:str, session):
         session_id=session_id,
         state=EVAL_STATE_QUERY,
         submitted=datetime.utcnow(),
-        code=ccodetoeval,
-        answer=""
     )
+
+    # assert False, "Need to refactor to insert new answer request and update the current answerID"
+
     # we insert and we get the new primary key
     session_obj = session_state_table.insert(session_obj)
     # will be executed in another thread with magic @threaded
@@ -232,8 +387,6 @@ def get(session):
         session_id=session_id,
         state=EVAL_STATE_NEW,
         submitted=datetime.utcnow(),
-        code="",
-        answer=""
     )
     session_state_table.insert(session_obj)
     # re-issue forms

test/test_dbops.py

@@ -1,5 +1,9 @@
 from fasthtml.common import *
 from sqlite_minutils.db import Database
+import json
+
+import eval_code
+
 def test_some_sqldb_inmem():
 
     db = Database(memory=True)
@@ -25,3 +29,21 @@ def test_some_sqldb_inmem():
     assert len(newg) == 1
     assert newg[0].prompt == "New prompt"
 
+
+def test_sqlite_jsonops():
+    db = Database(memory=True)
+    ans_js = eval_code.eval_the_piece_of_c_code(None, "")
+    enh_ans_js = eval_code.add_evaluation_fields_on_js_answer(ans_js)
+    qa_table = db.t.question_answers.create(id=int, session_id=str, question=str,
+                                 answerjs=str, pk="id")
+    qa_cls = qa_table.dataclass()
+    qa1 = qa_cls(question="Question", answerjs=enh_ans_js)
+    assert qa1.id is None
+    qa1 = qa_table.insert(qa1)
+    assert qa1.id >= 0
+    newans = json.loads(qa1.answerjs)
+    newans[2]["EVAL"] = eval_code.EVAL_ANSWER_POSITIVE
+    qa1.answerjs = newans
+    qa_table.update(qa1)
+    got_qa = qa_table(where=f"id == {qa1.id}")
+    assert "EVAL\": 1" in got_qa[0].answerjs