add skills query

app/app.py

@@ -20,6 +20,7 @@ def retrieve_documents(occupation,skills):
     output.append(f"<div style=\"text-align: center; font-size: 24px;\">Empfehlungsergebnisse:</div>")
     oc_uri = occupations.get(occupation, "")
     skill_query = ''
+    candidate_doc = []
     if isinstance(oc_uri, int):
         df = pd.read_csv("/app/data/berufe_info.csv")
         target_occupation = df[df['id'] == oc_uri]
@@ -32,13 +33,26 @@ def retrieve_documents(occupation,skills):
         target_occupation_name, target_occupation_dsp, target_occupation_query = build_occupation_query(target_occupation)
     for german_label in skills:
         skill_query += german_label + ' '
+        skills_doc = retriever.get_relevant_documents(german_label)
+        candidate_doc.extend(skills_doc[:2])
     query = 'target occupation: ' + target_occupation_query + ' Skills gap:' + skill_query
     llama_query = 'info:' + target_occupation_name + ' ' + 'Skills gap:' + skill_query
     print(query)
     docs = retriever.get_relevant_documents(query)
+    candidate_doc.extend(docs[:5])
+
+    #remove duplicates
+    seen_course_ids = set()
+    candidate_doc_unique = []
+
+    for doc in candidate_doc:
+        course_id = doc['metadata'].get('course_id')
+        if course_id not in seen_course_ids:
+            candidate_doc_unique.append(doc)
+            seen_course_ids.add(course_id)
 
     partial_compare_docs = functools.partial(compare_docs_with_context, target_occupation_name=target_occupation_name, target_occupation_dsp=target_occupation_dsp,skill_gap = skill_query)
-    sorted_docs = sorted(docs, key=functools.cmp_to_key(partial_compare_docs), reverse=True)
+    sorted_docs = sorted(candidate_doc, key=functools.cmp_to_key(partial_compare_docs), reverse=True)
 
     
     batch_prompts = []