ajout et big update

.streamlit/.env

@@ -1,2 +1,3 @@
 API_TOKEN_PERPLEXITYAI = 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
 OPENAI_API_KEY = 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
+FIRECRAWL_API_KEY = fc-381ecdb1175147aab5d2b48023961491

chat_te.py

@@ -0,0 +1,90 @@
+import streamlit as st
+from langchain_core.messages import AIMessage, HumanMessage
+from langchain_community.chat_models import ChatOpenAI
+from dotenv import load_dotenv
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from download_chart import construct_plot
+from langchain_core.runnables import RunnablePassthrough
+from langchain import hub
+from langchain_core.prompts.prompt import PromptTemplate
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OpenAIEmbeddings
+load_dotenv()  
+
+def get_conversation_chain(vectorstore):
+    llm = ChatOpenAI(model="gpt-4o",temperature=0.5, max_tokens=2048)
+    retriever=vectorstore.as_retriever()
+
+    prompt = hub.pull("rlm/rag-prompt")
+    # Chain
+    rag_chain = (
+        {"context": retriever , "question": RunnablePassthrough()}
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+    return rag_chain
+
+def get_response(user_query, chat_history):
+
+    template = """
+    Chat history: {chat_history}
+    User question: {user_question}
+    """
+
+    embeddings = OpenAIEmbeddings()
+    db = FAISS.load_local("vectorstore_op", embeddings)
+
+    question = ChatPromptTemplate.from_template(template)
+    question = question.format(chat_history=chat_history, user_question=user_query)
+
+    chain = get_conversation_chain(db)
+
+    return chain.stream(question)
+
+def display_chart():
+    if "pp_grouped" not in st.session_state or st.session_state['pp_grouped'] is None or len(st.session_state['pp_grouped']) == 0:
+        st.warning("Aucune partie prenante n'a été définie")
+        return None
+    plot = construct_plot()
+    st.plotly_chart(plot)
+
+
+def display_chat():
+    # app config
+    st.title("Chatbot")
+
+    # session state
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = [
+            AIMessage(content="Salut, voici votre cartographie des parties prenantes. Que puis-je faire pour vous?"),
+        ]
+
+        
+    # conversation
+    for message in st.session_state.chat_history:
+        if isinstance(message, AIMessage):
+            with st.chat_message("AI"):
+                st.write(message.content)
+                if "cartographie des parties prenantes" in message.content:
+                    display_chart()
+        elif isinstance(message, HumanMessage):
+            with st.chat_message("Moi"):
+                st.write(message.content)
+
+    # user input
+    user_query = st.chat_input("Par ici...")
+    if user_query is not None and user_query != "":
+        st.session_state.chat_history.append(HumanMessage(content=user_query))
+
+        with st.chat_message("Moi"):
+            st.markdown(user_query)
+
+        with st.chat_message("AI"):
+
+            response = st.write_stream(get_response(user_query, st.session_state.chat_history,format_context(st.session_state['pp_grouped'],st.session_state['Nom de la marque'])))
+            if "cartographie des parties prenantes" in message.content:
+                    display_chart()
+
+        st.session_state.chat_history.append(AIMessage(content=response))

chat_with_pps.py

@@ -25,8 +25,6 @@ def format_context(partie_prenante_grouped,marque):
 '''
     context += segmentation
     return context
-    
-
 
 
 def get_response(user_query, chat_history, context):

high_chart.py

@@ -151,7 +151,8 @@ cd2 = {
             "dragSensitivity":0
          },
          "data":[],
-         "colorByPoint":True
+         "colorByPoint":True,
+         
       }
    ],
    "exporting": {
@@ -191,7 +192,7 @@ def test_chart():
    #    st.session_state['pp_grouped'] = chart
 
    
-   
+   st.write(chart)
    if st.session_state['save']:
       st.session_state['save'] = False
       st.session_state['pp_grouped'] = chart.copy()

partie_prenante_carte.py

@@ -15,13 +15,11 @@ from langchain.llms import HuggingFaceHub
 from langchain import hub
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnablePassthrough
-from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.document_loaders import WebBaseLoader,FireCrawlLoader,PDFLoader
 from langchain_core.prompts.prompt import PromptTemplate
-import altair as alt
 from session import set_partie_prenante
 import os
 from streamlit_vertical_slider import vertical_slider
-from pp_viz import display_viz
 from high_chart import test_chart
 
 load_dotenv()
@@ -35,7 +33,18 @@ def get_docs_from_website(urls):
         return docs
     except Exception as e:
         return None
-    
+  
+
+def get_docs_from_website_fc(urls,firecrawl_api_key):
+    docs = []
+    try:
+        for url in urls:
+            loader = FireCrawlLoader(api_key=firecrawl_api_key, url = url,mode="scrape")
+            docs+=loader.load()
+        return docs
+    except Exception as e:
+        return None
+  
 
 def get_doc_chunks(docs):
     # Split the loaded data
@@ -43,17 +52,35 @@ def get_doc_chunks(docs):
     #                             chunk_size=500, 
     #                             chunk_overlap=100)
 
-    text_splitter = SemanticChunker(OpenAIEmbeddings())
+    text_splitter = SemanticChunker(OpenAIEmbeddings(model="text-embedding-3-small"))
     
     docs = text_splitter.split_documents(docs)
     return docs
+
+def get_doc_chunks_fc(docs):
+    # Split the loaded data
+    # text_splitter = RecursiveCharacterTextSplitter(
+    #                             chunk_size=500, 
+    #                             chunk_overlap=100)
+
+    text_splitter = SemanticChunker(OpenAIEmbeddings(model="text-embedding-3-small"))
+    docs_splitted = []
+    for text in docs:
+        text_splitted = text_splitter.split_text(text)
+        docs_splitted+=text_splitted
+    return docs_splitted
     
 
 def get_vectorstore_from_docs(doc_chunks):
-    embedding = OpenAIEmbeddings(model="text-embedding-3-large")
+    embedding = OpenAIEmbeddings(model="text-embedding-3-small")
     vectorstore = FAISS.from_documents(documents=doc_chunks, embedding=embedding)
     return vectorstore
 
+def get_vectorstore_from_text(texts):
+    embedding = OpenAIEmbeddings(model="text-embedding-3-small")
+    vectorstore = FAISS.from_texts(texts=texts, embedding=embedding)
+    return vectorstore
+
 def get_conversation_chain(vectorstore):
     llm = ChatOpenAI(model="gpt-4o",temperature=0.5, max_tokens=2048)
     retriever=vectorstore.as_retriever()
@@ -107,12 +134,15 @@ def display_list_urls():
 
         if len(st.session_state.urls) > index:
             # Instead of using markdown, use an expander in the first column
-            with col1.expander(f"URL {index}: {item}"):
+            with col1.expander(f"Source {index+1}: {item}"):
                 pp = st.session_state["parties_prenantes"][index]
                 st.write(pd.DataFrame(pp, columns=["Partie prenante"]))
         else:
             emp.empty()  # Clear the placeholder if the index exceeds the list
     
+def colored_circle(color):
+    return f'<span style="display: inline-block; width: 15px; height: 15px; border-radius: 50%; background-color: {color};"></span>'
+
 def display_list_pps():
     for index, item in enumerate(st.session_state["pp_grouped"]):
         emp = st.empty()
@@ -125,27 +155,24 @@ def display_list_pps():
 
         if len(st.session_state["pp_grouped"]) > index:
             name = st.session_state["pp_grouped"][index]["name"]
-            col1.markdown(f"{name}")
+            col1.markdown(f'<p>{colored_circle(st.session_state["pp_grouped"][index]["color"])} {st.session_state["pp_grouped"][index]["name"]}</p>',
+        unsafe_allow_html=True
+    )
         else:
             emp.empty()
 
 
                 
-def extract_pp(urls,input_variables):
+def extract_pp(docs,input_variables):
     template_extraction_PP = '''
-    Objectif : identifiez tout les noms de marques qui sont des parties prenantes de la marque suivante pour développer un marketing de coopération (co-op marketing)
-
-    Le nom de la marque de référence est le suivant : {BRAND_NAME}
-    Son activité est la suivante : {BRAND_DESCRIPTION}
+        Objectif : identifiez tout les parties prenantes de la marque suivante:
 
-    TA REPONSE DOIT ETRE SOUS FORME DE LISTE DE NOMS DE MARQUES SANS NUMEROTATION ET SEPARES PAR DES SAUTS DE LIGNE
+        Le nom de la marque de référence est le suivant : {BRAND_NAME}
 
-    SI TU NE TROUVES PAS DE NOM DE MARQUE, REPONDS "444"
-    '''
+        TA REPONSE DOIT ETRE SOUS FORME DE LISTE DE NOMS DE MARQUES SANS INCLURE LE NOM DE LA MARQUE DE REFERENCE SANS NUMEROTATION ET SEPARES PAR DES RETOURS A LA LIGNE
+        '''
     #don't forget to add the input variables from the maim function
 
-    docs = get_docs_from_website(urls)
-
     if docs == None:
         return "445"
 
@@ -167,9 +194,22 @@ def extract_pp(urls,input_variables):
 
     #version simple
     partie_prenante = response.content.replace("- ","").split('\n')
+    partie_prenante = [item.strip() for item in partie_prenante]
 
     return partie_prenante
 
+def generate_random_color():
+        # Generate random RGB values
+        r = random.randint(0, 255)
+        g = random.randint(0, 255)
+        b = random.randint(0, 255)
+
+        # Convert RGB to hexadecimal
+        color_hex = '#{:02x}{:02x}{:02x}'.format(r, g, b)
+
+        return color_hex
+
+
 def format_pp_add_viz(pp):
     y = 50
     x = 50
@@ -182,11 +222,11 @@ def format_pp_add_viz(pp):
         if st.session_state['pp_grouped'][i]['name'] == pp:
             return None
     else:
-        st.session_state['pp_grouped'].append({'name':pp, 'x':x,'y':y})
+        st.session_state['pp_grouped'].append({'name':pp, 'x':x,'y':y, 'color':generate_random_color()})
 
 def add_pp(new_pp, default_value=50):
     new_pp = sorted(new_pp)
-    new_pp = [item.lower().capitalize() for item in new_pp]
+    new_pp = [item.lower().capitalize().strip() for item in new_pp]
     st.session_state['parties_prenantes'].append(new_pp)
     for pp in new_pp:
         format_pp_add_viz(pp)
@@ -198,6 +238,7 @@ def add_pp_input_text():
         format_pp_add_viz(new_pp)
 
 import re
+import random
 
 def complete_and_verify_url(partial_url):
     # Regex pattern for validating a URL
@@ -232,7 +273,7 @@ def complete_and_verify_url(partial_url):
 def display_pp():
 
     load_dotenv()
-    
+    fire_crawl_api_key = os.getenv("FIRECRAWL_API_KEY")
     #check if brand name and description are already set
     if "Nom de la marque" not in st.session_state:
         st.session_state["Nom de la marque"] = ""
@@ -260,6 +301,7 @@ def display_pp():
 
         url = st.text_input("Ajouter une URL")
 
+        scraping_option = st.radio("Mode", ("Analyse rapide", "Analyse profonde"),horizontal=True)
         #if the user clicks on the button
         if st.button("ajouter"):
             st.session_state["save"] = True
@@ -271,9 +313,20 @@ def display_pp():
                 st.error("URL déjà ajoutée")
             
             else:
-                docs = get_docs_from_website(url)
+                if scraping_option == "Analyse profonde":
+                    with st.spinner("Collecte des données..."):
+                        docs = get_docs_from_website_fc([url],fire_crawl_api_key)
+                    if docs is None:
+                        st.warning("Erreur lors de la collecte des données, 2eme essai avec collecte rapide...")
+                        with st.spinner("2eme essai, collecte rapide..."):  
+                            docs = get_docs_from_website([url])
+
+                if scraping_option == "Analyse rapide":
+                    with st.spinner("Collecte des données..."):
+                        docs = get_docs_from_website([url])
+                
                 if docs is None:
-                    st.error("Aucune url trouvée ou erreur lors de la récupération du contenu")
+                    st.error("Erreur lors de la collecte des données")
                 else:
                 # Création de l'expander
                     with st.expander("Cliquez ici pour éditer et voir le document"):
@@ -286,7 +339,7 @@ def display_pp():
 
                         #handle the extraction
                         input_variables = {"BRAND_NAME": brand_name, "BRAND_DESCRIPTION": ""}
-                        partie_prenante = extract_pp([url], input_variables)
+                        partie_prenante = extract_pp(docs, input_variables)
 
                         if "444" in partie_prenante: #444 is the code for no brand found , chosen
                             st.error("Aucune partie prenante trouvée")

pp_viz.py

@@ -1,51 +0,0 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-import re
-
-import altair as alt
-from session import get_parties_prenantes
-import os
-from streamlit_vertical_slider import vertical_slider
-from st_draggable_list import DraggableList
-
-def display_viz():
-
-
-    parties_prenantes = get_parties_prenantes()
-
-    if parties_prenantes is None or len(parties_prenantes) == 0:
-        st.write("aucune partie prenante n'a été définie")
-    else:
-        partie_prenante_non_filtre = [item.lower().capitalize() for sublist in parties_prenantes for item in sublist]
-        partie_prenante = sorted(list(set(partie_prenante_non_filtre)))
-        pouvoir = [ 50 for _ in range(len(partie_prenante))]
-
-        c = (
-        alt.Chart(st.session_state['partie_prenante_grouped'])
-        .mark_circle(size=800)
-        .encode(x="partie_prenante", y=alt.Y("pouvoir",scale=alt.Scale(domain=[0,100])), color="Code couleur",tooltip=["partie_prenante","pouvoir"])
-        ).configure_legend(orient='bottom',direction="vertical").properties(height=600)
-
-        number_of_sliders = len(partie_prenante)
-        st.write("Modifiez le pouvoir des parties prenantes en utilisant les sliders ci-dessous")
-        
-       
-        bar = st.columns(number_of_sliders)
-        for i in range(number_of_sliders):
-            with bar[i]:
-                st.session_state['partie_prenante_grouped']['pouvoir'][i] = vertical_slider(
-                    label=partie_prenante[i],
-                    height=100,
-                    key=partie_prenante[i],
-                    default_value=int(st.session_state['partie_prenante_grouped']['pouvoir'][i]),
-                    thumb_color= "orange", #Optional - Defaults to Streamlit Red
-                    step=1,
-                    min_value=0,
-                    max_value=100,
-                    value_always_visible=False,
-                )
-        st.altair_chart(c, use_container_width=True)
-        # data = [{'id':partie_prenante[i], 'name':partie_prenante[i],'pouvoir':int(df["pouvoir"][i])} for i in range(len(partie_prenante))]
-        # slist = DraggableList(data)
-        # st.write(slist)

rag_funcs.py

@@ -0,0 +1,14 @@
+from firecrawl import FireCrawl
+
+
+def get_docs_from_website_fc(urls):
+    app = FireCrawl()
+    docs = []
+    try:
+        for url in urls:
+            content = app.scrape_url(url)
+            docs.append(content["markdown"])
+        return docs
+    except Exception as e:
+        return None
+

requirements.txt

@@ -32,4 +32,4 @@ langchain_experimental
 streamlit_draggable_list
 streamlit-highcharts
 pdfkit
-kaleido
+kaleido

st_hc/frontend/main.js

@@ -13,6 +13,7 @@ function onRender(event) {
     let points = c.series[0].data.map((p) => 
       ({ x: Math.round(p.x),
          y: Math.round(p.y),
+          color:p.color,
          name:p.name} ));
     sendValue(points);
 
@@ -25,6 +26,7 @@ function onRender(event) {
         let points = c.series[0].data.map((p) => 
          ({ x: Math.round(p.x),
             y: Math.round(p.y),
+            color:p.color,
             name:p.name} ));
 
         console.log(points);

vectorstore_op/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b492225278bd4ba23d11fe72fa16f8abd9a023babcc6734901740ba34fd0ba7
+size 106874