first commit

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.sqlite3 filter=lfs diff=lfs merge=lfs -text
+src/chroma_db/*.sqlite3 filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,21 +1,36 @@
 FROM python:3.9-slim
 
-WORKDIR /app
-
+# --- System setup ---
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
-    software-properties-common \
     git \
     && rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt ./
-COPY src/ ./src/
+# Create non-root user with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Set ENV vars for common cache locations to avoid permission issues
+ENV HOME=/home/user \
+    XDG_CACHE_HOME=/home/user/.cache \
+    HF_HOME=/home/user/.cache \
+    TRANSFORMERS_CACHE=/home/user/.cache \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set working directory
+WORKDIR $HOME/app
+
+# Copy files with correct ownership
+COPY --chown=user:user requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY --chown=user:user ./src ./src
 
-RUN pip3 install -r requirements.txt
+# Use non-root user from now on
+USER user
 
 EXPOSE 8501
 
 HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
 
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["streamlit", "run", "src/app.py", "--server.port=8501", "--server.address=0.0.0.0"]

README.md

@@ -1,19 +0,0 @@
----
-title: Chatbot
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
-pinned: false
-short_description: Streamlit template space
----
-
-# Welcome to Streamlit!
-
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
-
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).

src/Reranker.py

@@ -0,0 +1,54 @@
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import BaseModel, Field
+from typing import List
+from langchain_groq import ChatGroq
+
+llm = ChatGroq(
+    model="meta-llama/llama-4-scout-17b-16e-instruct",  
+    temperature=0,
+    max_tokens=None,
+    timeout=None,
+    max_retries=2,
+
+)
+
+class RerankResult(BaseModel):
+     top_indexes: List[int] = Field(..., description="A releváns dokumentumok indexei")
+     message: str = Field(..., description="A felhasználónak megjelenítendő üzenet")
+
+class Reranker:
+    def __init__(self, model):
+        self.llm = model
+        self.parser = self.llm.with_structured_output(RerankResult)
+
+    def rerank(self, query: str, docs: List) -> RerankResult:
+        context = "\n\n".join(
+            [f"[{i}] {doc[0]} (score={doc[1]:.3f})" for i, doc in enumerate(docs)]
+        )
+
+        prompt = ChatPromptTemplate.from_template("""
+        Egy intelligens asszisztens vagy, akinek feladata a dokumentumok szűrése azok relevanciája alapján egy kérdés megválaszolásához.
+        Az alábbi a felhasználó kérdése: "{query}"
+        És itt vannak a lekért dokumentumok:
+        {context}
+        Feladatod:
+        1. Csak azokat a dokumentumokat válaszd ki, amelyek valóban relevánsak (a többit hagyd figyelmen kívül).
+        2. Csak a releváns dokumentumok indexeit add vissza (ugyanabban a sorrendben, ahogy a listában szerepelnek).
+        3. Ha egyik dokumentum sem releváns, akkor a "top_indexes" értéke legyen egy üres lista, és az üzenet: "Kérjük, pontosítsa kérdését."
+        ⚠️ Nagyon fontos: A válaszod **szigorúan érvényes JSON formátumú legyen** (se előtte, se utána ne legyen szöveg), az alábbi szabályokkal:
+        - A kulcsok legyenek dupla idézőjelek között (`"`)
+        - A logikai értékek kisbetűsek legyenek (`true`/`false`)
+        - Ne használj sortörést a JSON-on kívül
+        Íme a várt kimeneti formátum egy példája:
+        {{
+        "top_indexes": [0, 2],
+        "message": ""
+        }}
+        """)
+
+
+        formatted_prompt = prompt.format_prompt(query=query, context=context)
+        result = self.parser.invoke(formatted_prompt.to_messages())
+
+        return result  

src/app.py

@@ -0,0 +1,303 @@
+import streamlit as st
+import os
+from chromadb import Client as ChromaClient
+from chroma_storage import ChromaStorage
+from retrieval import Retriever
+
+from generation import RAGGenerator
+from langchain_community.vectorstores import Chroma
+from langchain_huggingface import HuggingFaceEmbeddings
+from constant import categorie
+from dotenv import load_dotenv
+
+import chromadb
+from Reranker import Reranker 
+
+chromadb.api.client.SharedSystemClient.clear_system_cache()
+load_dotenv()
+import os
+
+
+
+
+WANDB_PROJECT   = "rag"
+CHROMA_DIR      = "src/chroma_db"
+COLLECTION_NAME = "my_collection"
+
+st.markdown("""
+<style>
+body {
+    font-family: 'Segoe UI', Roboto, sans-serif;
+}
+.header {
+    background-color: #4A90E2;
+    color: white;
+    padding: 10px;
+    display: flex;
+    align-items: center;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.2);
+}
+.header img {
+    height: 40px;
+    margin-right: 10px;
+}
+.header h1 {
+    margin: 0;
+    font-size: 24px;
+}
+.chat-container {
+    padding: 10px;
+}
+.bot-message, .user-message {
+    display: flex;
+    align-items: flex-start;
+    margin: 8px 0;
+}
+.bot-message .bubble {
+    margin-left: 10px;
+    background-color: #f0f0f0;
+    color: #000;
+}
+.user-message {
+    justify-content: flex-end;
+}
+.user-message .bubble {
+    margin-right: 10px;
+    background-color: #4A90E2;
+    color: #fff;
+}
+.bubble {
+    border-radius: 8px;
+    padding: 10px;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    max-width: 70%;
+}
+.avatar {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+}
+@media (max-width: 600px) {
+    .header h1 {
+        font-size: 18px;
+    }
+    .bubble {
+        max-width: 90%;
+    }
+}
+</style>
+""", unsafe_allow_html=True)
+
+# --- En-tête bleu avec logo et titre ---
+st.markdown(f'''
+<div class="header">
+  <img src="https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fwww.aplicasoft.com%2Fimages%2Flogos%2F4d_logo.gif&f=1" alt="Logo 4D">
+  <h1>4D Assistant - En ligne</h1>
+</div>
+''', unsafe_allow_html=True)
+
+
+if "categories" not in st.session_state:
+    st.session_state.categories = categorie
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "step" not in st.session_state:
+    st.session_state.step = 0
+if "selected_category" not in st.session_state:
+    st.session_state.selected_category = None
+
+
+
+# --- Affichage de la sélection de catégorie centrée ---
+if st.session_state.selected_category is None:
+    st.markdown(
+        """
+        <div style="display: flex; justify-content: center; align-items: center; height: 60vh;">
+            <div>
+                <h2 style="text-align:center;">Choisissez une catégorie</h2>
+            </div>
+        </div>
+        """,
+        unsafe_allow_html=True,
+    )
+    selected = st.selectbox(
+        "", st.session_state.categories, key="category_select", index=0
+    )
+    if st.button("Valider la catégorie"):
+        st.session_state.selected_category = selected
+        st.rerun()
+    st.stop()  
+
+# --- Affichage du chat uniquement après sélection ---
+sel = st.session_state.selected_category
+
+
+
+# Ajouter le message initial du bot à l'étape 1
+if st.session_state.step == 0:
+    st.session_state.messages.append({
+        "role": "assistant", 
+        "content": "je suis votre assistant 4D"
+    })
+  #  save_conversation_to_mongo(st.session_state.messages)
+    st.session_state.step = 1
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 2) INITIALISATION UNIQUE (cache via session_state)
+# ─────────────────────────────────────────────────────────────────────────────
+if "initialized" not in st.session_state:
+    st.session_state.initialized = True
+    retriever = Retriever(
+        chroma_dir=CHROMA_DIR,
+        collection_name=COLLECTION_NAME
+       
+    )
+   
+   
+    
+    generator = RAGGenerator()
+    reranker = Reranker()
+    st.session_state.retriever   = retriever
+    st.session_state.generator   = generator
+    st.session_state.reranker    = reranker
+   # st.session_state.categories  = categorie
+   
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 3) UI & État de chat
+# ─────────────────────────────────────────────────────────────────────────────
+query = st.chat_input("chat_query")
+#if "last_rerank" in st.session_state:
+   # st.code(f"Rerank result: {st.session_state.last_rerank}")
+
+# Si l'utilisateur a saisi une requête
+if query:
+    # Si on est en phase de clarification
+    if "qa_pairs" in st.session_state and "clarif_idx" in st.session_state:
+        idx = st.session_state.clarif_idx
+        st.session_state.messages.append({"role": "user", "content": query})
+      #  save_conversation_to_mongo(st.session_state.messages)
+        st.session_state.qa_pairs[idx]["response"] = query
+        st.session_state.clarif_idx += 1
+    else:
+        # Requête initiale normale
+        st.session_state.messages.append({"role": "user", "content": query})
+      #  save_conversation_to_mongo(st.session_state.messages)
+        try:
+            # 1. Récupération des documents initiaux
+           # with st.spinner():
+            docs = st.session_state.retriever.retrieve(query)
+                
+
+            passages = [(doc_tuple[0].page_content, doc_tuple[1]) for doc_tuple in docs]
+
+
+              #  st.code(passages)
+            
+            # 2. Reranking des documents pour améliorer la pertinence
+            if docs:
+               # with st.spinner():
+                    rerank_result = st.session_state.reranker.rerank(query, passages)
+                    st.session_state.last_rerank = rerank_result.model_dump()
+
+                    #st.code(rerank_result)
+
+                   
+                    
+                    # Vérification si un feedback est requis
+                    if rerank_result.feedback_required:
+                        st.session_state.messages.append({
+                            "role": "assistant",
+                            "content": rerank_result.message
+                        })
+                        #save_conversation_to_mongo(st.session_state.messages)
+                        st.rerun() 
+                         # Utiliser st.rerun() au lieu de continue
+                    
+                    # Filtrage et réorganisation des documents selon le reranking
+                    if rerank_result.top_indexes:
+                        reranked_docs = [docs[i] for i in rerank_result.top_indexes if i < len(docs)]
+                        
+                        docs = reranked_docs
+                    
+            # 3. Génération de la réponse avec les documents reranked
+            if docs:  # Vérifier que nous avons des documents avant de continuer
+                #with st.spinner():
+                    qa_pairs,solution, lang = st.session_state.generator.retrieve_qa(
+                        query, category=sel, docs_scores=docs
+                    )
+                    
+                    st.session_state.qa_pairs = qa_pairs
+                    st.session_state.solution = solution
+                    st.session_state.lang = lang
+            else:
+                # Aucun document disponible
+                st.session_state.messages.append({
+                    "role": "assistant",
+                    "content": "Aucun document pertinent trouvé pour votre requête. Pouvez-vous la reformuler ?"
+                })
+                #save_conversation_to_mongo(st.session_state.messages)
+                
+        except ValueError as e:
+            st.session_state.messages.append({
+                "role": "assistant",
+                "content": str(e)
+            })
+         #   save_conversation_to_mongo(st.session_state.messages)
+        except Exception as e:
+            st.error(f"❌ Erreur : {e}")
+
+# ───────────────────────────────────────────────
+# Gestion des clarifications
+# ───────────────────────────────────────────────
+if "qa_pairs" in st.session_state:
+    if "clarif_shown" not in st.session_state:
+        questions_text = "\n".join([
+            f"{i+1}. {pair['question']}" for i, pair in enumerate(st.session_state.qa_pairs)
+        ])
+        st.session_state.messages.append({
+            "role": "assistant",
+            "content": questions_text
+        })
+      #  save_conversation_to_mongo(st.session_state.messages)
+        st.session_state.clarif_shown = True
+
+    if "clarif_idx" not in st.session_state:
+        st.session_state.clarif_idx = 0
+
+    # Lorsque toutes les clarifications sont remplies, générer la réponse finale
+    if st.session_state.clarif_idx >= len(st.session_state.qa_pairs):
+        final = st.session_state.generator.generate_answer(
+            st.session_state.messages,
+            st.session_state.solution,
+            st.session_state.lang
+        )
+        st.session_state.messages.append({"role": "assistant", "content": final})
+       # save_conversation_to_mongo(st.session_state.messages)
+        for key in ["qa_pairs", "solution", "lang", "clarif_idx", "clarif_shown"]:
+            st.session_state.pop(key, None)
+
+# --- Affichage de l'historique complet des messages ---
+st.markdown('<div class="chat-container">', unsafe_allow_html=True)
+for msg in st.session_state.messages:
+    if not msg.get("content"):
+        continue
+    if msg["role"] == "assistant":
+        st.markdown(f'''
+        <div class="bot-message">
+          <img src="https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fwww.aplicasoft.com%2Fimages%2Flogos%2F4d_logo.gif&f=1" class="avatar">
+          <div class="bubble">{msg["content"]}</div>
+        </div>
+        ''', unsafe_allow_html=True)
+    else:
+        st.markdown(f'''
+        <div class="user-message">
+          <div class="bubble">{msg["content"]}</div>
+          <img src="https://img.icons8.com/ios-filled/50/4A90E2/user-male-circle.png" class="avatar">
+        </div>
+        ''', unsafe_allow_html=True)
+st.markdown('</div>', unsafe_allow_html=True)
+
+

src/chroma_db/7fb02c85-346a-45e4-92fd-7b7e16fdc75d-20250627T191057Z-1-001.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8c1b16f6503fdcc91139b8f561ea5edb0f25f78792eef4a4494bbbda268491e
+size 18001

src/chroma_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dafcd27d44df3c09c89f40192f451c2eb38919e401aeaf0d9a2a055a165bb520
+size 4546560

src/chroma_storage.py

@@ -0,0 +1,33 @@
+import os
+import chromadb
+from chromadb.utils import embedding_functions
+
+class ChromaStorage:
+    def __init__(self, db_path: str = './src/chroma_db', collection_name: str = 'my_collection'):
+       
+        self.client = chromadb.PersistentClient(path=db_path)
+        self.collection = self._get_or_create_collection(collection_name)
+
+    def _get_or_create_collection(self, name: str):
+        try:
+            return self.client.get_collection(
+                name=name
+            )
+        except Exception:
+            return self.client.create_collection(
+                name=name,
+                embedding_function=self.openai_ef
+            )
+
+    def add_batch(self, documents: list, metadatas: list, ids: list, batch_size: int = 200):
+        for i in range(0, len(documents), batch_size):
+            docs = documents[i:i + batch_size]
+            metas = metadatas[i:i + batch_size]
+            batch_ids = ids[i:i + batch_size]
+            self.collection.add(documents=docs, metadatas=metas, ids=batch_ids)
+
+    def query(self, query_text: str, k: int = 1) -> list:
+        return self.collection.query(query_texts=[query_text], n_results=k)
+
+    def delete_all(self):
+        self.client.reset()

src/constant.py

@@ -0,0 +1 @@
+categorie =["Activation", "Certification","Connectivity", "Corruption", "Deploiement", "Developement", "Dump", "Error message", "Instability","Installation","Mauvais fonctionnement", "Network", "Performances", "Qodly", "Qodly Studio", "Sauvegarde", "TAOW Issue", "Usability"]

src/generation.py

@@ -0,0 +1,99 @@
+from langchain_groq import ChatGroq
+from langchain.schema import AIMessage, HumanMessage, SystemMessage
+import json
+from langchain.schema import SystemMessage, HumanMessage
+from langchain_groq import ChatGroq
+class RAGGenerator:
+     
+
+
+
+    def generate_answer( messages: list, solution: str, lang: str) -> str:
+        
+        consigne = (
+        "Te egy műszaki asszisztens vagy. A megadott végleges megoldás alapján "
+        "csak azokat a releváns elemeket emeld ki, amelyek válaszolnak a felhasználó kérdésére. "
+        "Ezután fogalmazz meg egy tiszta, tömör, szigorúan tényszerű, pontos és részletes választ. "
+        "NE tartalmazzon SEMMILYEN személyesítést (sem nevet, sem keresztnév, sem köszönést, sem köszönetet, sem aláírást). "
+        "NE tartalmazzon SEMMILYEN kérést fájlok, külső objektumok, képernyőképek küldésére "
+        "vagy bármilyen más, külső átvitelhez szükséges segítségnyújtást."
+    )
+
+
+        langue_instruction = {
+            'fr': 'Réponds en français.',
+            'en': 'Respond in English.',
+            'es': 'Responde en español.',
+            'de': 'Antworte auf Deutsch.',
+            'zh': '请用中文回答。' 
+        }
+
+        sys_msg = consigne + ' ' + langue_instruction.get(lang, '')
+
+        content = [SystemMessage(content = f"A következő megoldás felhasználásával készítsd el az átfogalmazást: Végleges megoldás:\n{solution}")]
+
+        chat_history = [SystemMessage(content=sys_msg)] + [
+            AIMessage(content=msg["content"]) if msg["role"] == "assistant" else HumanMessage(content=msg["content"]) 
+            for msg in messages
+        ] + content
+
+        chat = ChatGroq(
+            model_name="meta-llama/llama-4-scout-17b-16e-instruct",  
+            temperature=0.2
+        )
+
+        reply = chat.invoke(chat_history)
+
+        return reply.content
+
+
+def retrieve_qa( query: str, docs_scores: list = []) -> tuple:
+    if not docs_scores:
+       raise LookupError("Nem található dokumentum a lekérdezéshez.")
+
+    chat = ChatGroq(
+        model_name="LLaMA3-8b-8192",
+        temperature=0
+    )
+    try:
+        lang_detect_messages = [
+            SystemMessage(content="Határozd meg a következő mondat nyelvét. Csak az ISO 639-1 nyelvkódot válaszold meg (pl. 'fr', 'en', 'es' stb.):"),
+
+            HumanMessage(content=query)
+        ]
+        lang_response = chat.invoke(lang_detect_messages)
+        lang = lang_response.content.strip().lower()
+    except Exception:
+        lang = "en"  
+
+    first_doc, _ = docs_scores[0]
+    try:
+        qa_pairs = json.loads(first_doc.metadata.get('qa_pairs', '[]'))
+        qa_pairs = [q for q in qa_pairs if q.get('question')]
+    except json.JSONDecodeError:
+        raise ValueError("Érvénytelen a 'qa_pairs' metaadat.")
+
+
+    if lang and qa_pairs:
+        for pair in qa_pairs:
+            original_question = pair.get("question", "")
+            if original_question:
+                try:
+                    messages = [
+                        SystemMessage(content=f"Fordítsd le a következő kérdést {lang} nyelvre, csak a lefordított mondatot add meg:"),
+
+                        HumanMessage(content=original_question)
+                    ]
+                    response = chat.invoke(messages)
+                    pair["question"] = response.content.strip()
+                except Exception:
+                    pass
+
+    solutions = []
+    for doc, _ in docs_scores:
+        sol = doc.metadata.get('solution_finale', '')
+        if sol:
+            solutions.append(sol)
+    full_solution = "\n\n".join(solutions)
+
+    return qa_pairs, full_solution, lang

src/retrieval.py

@@ -0,0 +1,32 @@
+
+# hybrid_retrieval.py
+from langchain_community.vectorstores import Chroma
+from langchain_core.documents import Document
+from langchain_huggingface import HuggingFaceEmbeddings
+import os
+
+print(os.listdir())
+class Retriever:
+    
+    def __init__(self,
+                 chroma_dir: str = 'src/chroma_db',
+                 collection_name: str = 'my_collection'
+                
+    ):
+        
+       
+        self.vectorstore = Chroma(
+            collection_name=collection_name,
+            persist_directory=chroma_dir
+        
+        )
+    
+    def retrieve(self, query: str):
+        """
+        Récupère les documents via le retriever Embedding.
+        """
+        return self.vectorstore.similarity_search_with_score(query,k=3)
+
+    
+    def add_document(self, doc: str, metadata: dict, id: str):
+        self.vectorstore.collection.add(documents=[doc], metadatas=[metadata], ids=[id])

src/style.css

@@ -0,0 +1,88 @@
+/* Conteneur principal du chat */
+.chat-container {
+    position: fixed;
+    bottom: 20px;
+    right: 20px;
+    width: 360px;
+    max-height: 600px;
+    display: flex;
+    flex-direction: column;
+    box-shadow: 0 4px 20px rgba(0,0,0,0.2);
+    border-radius: 1rem;
+    overflow: hidden;
+    background: #ffffff;
+    font-family: sans-serif;
+    z-index: 1000;
+  }
+  
+  /* En‑tête en forme de vague */
+  .chat-header {
+    position: relative;
+    height: 120px;
+    background: #333835;
+  }
+  .chat-header svg {
+    position: absolute;
+    bottom: 0;
+    width: 100%;
+    height: auto;
+    display: block;
+  }
+  
+  /* Corps du chat */
+  .chat-body {
+    flex: 1;
+    padding: 0.5rem;
+    overflow-y: auto;
+    display: flex;
+    flex-direction: column;
+  }
+  
+  /* Bulles de message */
+  .user-bubble,
+  .assistant-bubble {
+    padding: 0.5rem 1rem;
+    border-radius: 0.75rem;
+    margin: 0.25rem 0;
+    max-width: 80%;
+    word-wrap: break-word;
+    line-height: 1.4;
+  }
+  .user-bubble {
+    background: #f1f0f0;
+    align-self: flex-end;
+  }
+  .assistant-bubble {
+    background: #e0f7fa;
+    align-self: flex-start;
+  }
+  
+  /* Zone de saisie */
+  .chat-input-container {
+    padding: 0.5rem;
+    border-top: 1px solid #ececec;
+  }
+  .chat-input {
+    width: 100%;
+    padding: 0.5rem;
+    border: 1px solid #d1d1d1;
+    border-radius: 0.5rem;
+    outline: none;
+    font-size: 1rem;
+  }
+  
+  /* Bouton flottant */
+  .chat-toggle {
+    position: fixed;
+    bottom: 20px;
+    right: 20px;
+    background: linear-gradient(135deg, #6e8efb, #a777e3);
+    color: #fff;
+    padding: 0.75rem 1rem;
+    border: none;
+    border-radius: 999px;
+    cursor: pointer;
+    box-shadow: 0 4px 12px rgba(0,0,0,0.3);
+    z-index: 1001;
+  }
+