add oai and modules

README.md

@@ -8,9 +8,6 @@ sdk_version: 3.40.1
 app_file: app.py
 pinned: false
 license: apache-2.0
-
-hf_oauth: true
-hf_oauth_redirect_path: /custom_callback_route # optional, see "Redirect URLs" below
 ---
 
 

app.py

@@ -5,6 +5,7 @@ from typing import Optional
 import json
 
 from load_db import load_vectorestore_from_pdf
+from oai import chat_prompt, get_comparison
 
 
 TEMP_PDF_PATH = "temp.pdf"
@@ -12,6 +13,18 @@ retriever = None
 db = None
 documents = None
 
+with open("all_unique_modules.json","r",encoding="UTF-8") as f:
+    courses = json.load(f)
+
+def get_course(code:str)->str|None:
+    #print(len(courses))
+    for c in courses:
+        if not "Kurscode" in c:
+            continue
+        if c["Kurscode"].strip()==code.strip():
+            return f"Kursbeschreibung:\n{c['Description']}\nKursziele:\n{c['Kursziele']}\nKursinhalte:\n{c['Kursinhalt']}"
+    return None
+
 def pdf_to_text(file_path:str, page_num:Optional[int]=None):
     reader = PdfReader(file_path)
     if page_num:
@@ -37,6 +50,102 @@ def load_pdf(inp):
     #print(text)
     return text
 
+def compare_with_gpt(iu_code:str,text:str, model_name:Optional[str]=None)->str|None:
+    iu_course = get_course(iu_code)
+    if iu_course == None:
+        print("IU Course not found")
+        return None 
+    if model_name != None:
+        return get_comparison(iu_course,text,model_name=model_name)
+    return get_comparison(iu_course,text,model_name="gpt-4")
+
+def get_relevant_docs(search:str, iu_course:str):
+    global db, retriever, documents
+    documents = []
+    final_res = ""
+    documents = retriever.get_relevant_documents(f"Wo ist Modul/Kurs: {search.strip()}")
+    for document_ in documents:
+        print(f'\n>{document_.metadata["source"]} (Page {document_.metadata["page"]})')
+        #print(">",document.page_content,"\n\n")
+        final_res = f'{final_res}\n>{document_.metadata["source"]} (Page {document_.metadata["page"]}) :'
+    get_relevant_docs_promt = f"""
+
+    Du wirst 4 verschiedene Objecte in JSON erhalten und musst herausfinden welches ich suche.
+    Es handelt sich dabei um Ausschnitte eines Modulhandbuches einer Hochschule.
+    Ich suche die Modulbeschreibung von '{search}'. Suche danach in dem property 'page_content'
+    Nenne die Page in den Metadaten des richtigen Objektes. Denk dir keinen Quatsch aus, wenn du die Modulbeschreibung nicht findest, sag es.
+    Antworte in JSON format und fülle die werte page_found (bool) und page (int|None) aus.
+    """.strip()
+    messages = [
+        {
+            "role": "system", "content": f"{get_relevant_docs_promt}"
+        },
+        {
+            "role": "system", "content": f"Das sind die 4 Dokumente:\n{documents}"
+        }
+    ]
+    res = chat_prompt(messages=messages,model_name="gpt-4") #gpt-3.5-turbo-16k doesnt understand the task
+
+    final_res = f'{final_res}\n\n{res}\n'
+    res = json.loads(res)
+    if res["page_found"] ==False:
+        return final_res
+    page_num = int(res["page"])
+    
+    final_res = f'{final_res}\n\nDocument auf Seite >{page_num} identifiziert.\nPrüfe auf Vollständigkeit...\n'
+
+    for doc in documents:
+        if page_num == doc.metadata["page"]:
+            print(page_num,"==",doc.metadata["page"])
+            document = doc
+            break 
+    next_page = documents = db.get(where={"page":page_num+1}) #pdf_to_text("temp.pdf",page_num=page_num+1)
+    if len(next_page["documents"]) > 0:
+        next_page = next_page["documents"][0]
+        words = next_page.split()
+        first_40_words = words[:40]
+        first_40_words_str = ' '.join(first_40_words)
+
+        # get the 50 last words of the description
+        words = document.page_content.split()
+        last_50_words = words[-50:]
+        # Join the words back into a string
+        last_50_words_str = ' '.join(last_50_words)
+
+        check_doc_complete_prompt= f"""
+            Du wirst einen Auszug einer Modulbeschreibung eines Modules von einer Hochschule erhalten und sollst herausfinden, ob diese vollständig ist oder ob eventuell die nächste
+            Seite auch noch Inhalte zum Modul hat. Solltest du eine Auflistung der Kompetenzen und Inhalte finden, ist die Modulbeschreibung vollständig. Wenn auf der Folgeseite nur noch Literaturangaben sind, ist die Modulbeschreibung bereits vollständig.
+            Wenn der Auszug mit den Literaturangaben endet, ist die Modulbeschreibung vollständig.
+            Antworte nur mit 'vollständig' und 'unvollständig'.
+            Das Modul heißt [{search}]. Ist die Beschreibung vollständig oder fehlt etwas? Hier ist der Text: \n[{last_50_words_str}].
+            Das ist hier ist der Text der nächsten Seite:\n[{first_40_words_str}]
+        """.strip()
+
+        messages = [
+            {
+                "role": "system", "content": f"{check_doc_complete_prompt}"
+            }
+        ]
+
+        res = chat_prompt(messages=messages,model_name="gpt-3.5-turbo-16k") #gpt-3.5-turbo-16k
+        #print("\nDie Beschreibung ist: ",res,"\n")
+        final_res = f'{final_res}\nDie Beschreibung ist: {res}\n'
+
+        description = f"{document.page_content}"
+
+        if res == "unvollständig":
+            description = f"{description}\n{next_page}"
+    else:
+        
+        description = f"{document.page_content}"
+        final_res = f'{final_res}\nDie Beschreibung ist: {description}\n'
+    
+    final_res = f'{final_res}\nSuche nach: {search}\nBeschreibung: {description}\n\n'
+    res = compare_with_gpt(iu_course,description,model_name="gpt-3.5-turbo-16k")
+
+    final_res = f'{final_res}\nIU-Kurs:{iu_course}\nErgebnis: {res}\n'
+
+    return final_res
 
 with gr.Blocks() as app:
     file = gr.File(type="binary") 
@@ -47,6 +156,15 @@ with gr.Blocks() as app:
     load_file_button.click(load_pdf,inputs=file,outputs=handbook)
 
 
+    with gr.Accordion("Kurse finden",open=False):
+        with gr.Row():
+            course_query = gr.Textbox(label="Kursnamen der zu prüfenden Kurse")
+            iu_ccode = gr.Textbox(label="IU Kurscode")
+        course_description = gr.TextArea(label="Ergebnis der Prüfung")
+        
+
+        course_query.submit(get_relevant_docs,inputs=[course_query,iu_ccode],outputs=course_description)
+
 
 
 if __name__ == "__main__":

oai.py

@@ -0,0 +1,54 @@
+import os
+import openai
+from dotenv import load_dotenv
+
+load_dotenv()
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+def get_comparison(iu_course,foreign_course,model_name:str="gpt-3.5-turbo")->str|None:
+    try:
+        completion = openai.ChatCompletion.create(
+        model=model_name,
+        messages=[
+            {"role": "system", "content": f"""
+            You are comparing one educational module between IU - International University of Applied Sciences and one module of another university for equivalency. The module of the other university may include several submodules, all submodules count as one.
+
+            User will provide a description of the module of another university and your job is to figure out, if this module is equivalent to the IU module.
+
+            List which key competencies in the IU module are not present in the other module.
+
+            List which content in the IU module is not present in the other module.
+
+            Find the most important  key competencies in the IU Module and list them, judge if they are present in the other module (if present and if the other module contains submodules, name the submodule).
+
+            Find the most important  key competencies in the other module and list them, judge if they are present in the IU module  (if present and if the other module contains submodules, name the submodule).
+
+            Finally, estimate how much of the modules are equivalent in percent considering the key competencies and content that is present in IU Module but not in the other (the more present the more equivalent). Make a judge if the compared modules are equivalent, keep in mind that same topics might be called different.
+
+            Answer only in german language.
+
+            This is the module from IU - International University of Applied Sciences: \n{iu_course}
+            """},
+            {"role": "user", "content": f"{foreign_course}"}
+        ],
+        temperature=0,
+        top_p=0.9
+        )
+        return completion.choices[0].message["content"]
+    except Exception as e:
+        print(e)
+        return None
+    
+def chat_prompt(messages,temperature:float=0.0,top_p:float=0.9,model_name:str="gpt-3.5-turbo")->str|None:
+    try:
+        completion = openai.ChatCompletion.create(
+        model=model_name,
+        messages=messages,
+        temperature=temperature,
+        top_p=top_p
+        )
+        return completion.choices[0].message["content"]
+    except Exception as e:
+        print(e)
+        return None