add new files

.gitignore

@@ -0,0 +1,6 @@
+/.streamlit
+*.env
+.env
+venv
+.streamlit/secrets.toml
+

README.md

@@ -0,0 +1,15 @@
+---
+title: New LS Chatbot App
+emoji: 🌍
+colorFrom: blue
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.42.0
+app_file: app.py
+pinned: false
+short_description: It is a chat built with an AI model about www.Status.law
+---
+
+# LS Chatbot App
+
+It is a chat app built using Streamlit that allows users to interact with an AI model to communicate about www.Status.law

app.py

@@ -0,0 +1,210 @@
+import os
+import time
+import streamlit as st
+from dotenv import load_dotenv
+from langchain_groq import ChatGroq
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnableLambda
+import requests
+import json
+
+# Page configuration
+st.set_page_config(page_title="Status Law Assistant", page_icon="⚖️")
+
+# Knowledge base info in session_state
+if 'kb_info' not in st.session_state:
+    st.session_state.kb_info = {
+        'build_time': None,
+        'size': None
+    }
+
+# Display title and knowledge base info
+# st.title("www.Status.Law Legal Assistant")
+
+st.markdown(
+    '''
+    <h1>
+        ⚖️ 
+        <a href="https://status.law/" style="text-decoration: underline; color: blue; font-size: inherit;">
+            Status.Law
+        </a> 
+        Legal Assistant
+    </h1>
+    ''',
+    unsafe_allow_html=True
+)
+
+if st.session_state.kb_info['build_time'] and st.session_state.kb_info['size']:
+    st.caption(f"(Knowledge base build time: {st.session_state.kb_info['build_time']:.2f} seconds, "
+               f"size: {st.session_state.kb_info['size']:.2f} MB)")
+
+# Path to store vector database
+VECTOR_STORE_PATH = "vector_store"
+
+# Создание папки истории, если она не существует
+if not os.path.exists("chat_history"):
+    os.makedirs("chat_history")
+
+# Website URLs
+urls = [
+    "https://status.law",  
+    "https://status.law/about",
+    "https://status.law/careers",
+    "https://status.law/challenging-sanctions",
+    "https://status.law/law-firm-contact-legal-protection"
+    "https://status.law/cross-border-banking-legal-issues", 
+    "https://status.law/extradition-defense", 
+    "https://status.law/international-prosecution-protection", 
+    "https://status.law/interpol-red-notice-removal",  
+    "https://status.law/practice-areas",  
+    "https://status.law/reputation-protection",
+    "https://status.law/faq"
+]
+
+# Load secrets
+try:
+    GROQ_API_KEY = st.secrets["GROQ_API_KEY"]
+except Exception as e:
+    st.error("Error loading secrets. Please check your configuration.")
+    st.stop()
+
+# Initialize models
+@st.cache_resource
+def init_models():
+    llm = ChatGroq(
+        model_name="llama-3.3-70b-versatile",
+        temperature=0.6,
+        api_key=GROQ_API_KEY
+    )
+    embeddings = HuggingFaceEmbeddings(
+        model_name="intfloat/multilingual-e5-large-instruct"
+    )
+    return llm, embeddings
+
+# Build knowledge base
+def build_knowledge_base(embeddings):
+    start_time = time.time()
+    
+    documents = []
+    with st.status("Loading website content...") as status:
+        for url in urls:
+            try:
+                loader = WebBaseLoader(url)
+                docs = loader.load()
+                documents.extend(docs)
+                status.update(label=f"Loaded {url}")
+            except Exception as e:
+                st.error(f"Error loading {url}: {str(e)}")
+                
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=100
+    )
+    chunks = text_splitter.split_documents(documents)
+    
+    vector_store = FAISS.from_documents(chunks, embeddings)
+    vector_store.save_local(VECTOR_STORE_PATH)
+    
+    end_time = time.time()
+    build_time = end_time - start_time
+    
+    # Calculate knowledge base size
+    total_size = 0
+    for path, dirs, files in os.walk(VECTOR_STORE_PATH):
+        for f in files:
+            fp = os.path.join(path, f)
+            total_size += os.path.getsize(fp)
+    size_mb = total_size / (1024 * 1024)
+    
+    # Save knowledge base info
+    st.session_state.kb_info['build_time'] = build_time
+    st.session_state.kb_info['size'] = size_mb
+    
+    st.success(f"""
+    Knowledge base created successfully:
+    - Time taken: {build_time:.2f} seconds
+    - Size: {size_mb:.2f} MB
+    - Number of chunks: {len(chunks)}
+    """)
+    
+    return vector_store
+
+# Main function
+def main():
+    # Initialize models
+    llm, embeddings = init_models()
+    
+    # Check if knowledge base exists
+    if not os.path.exists(VECTOR_STORE_PATH):
+        st.warning("Knowledge base not found.")
+        if st.button("Create Knowledge Base"):
+            vector_store = build_knowledge_base(embeddings)
+            st.session_state.vector_store = vector_store
+            st.rerun()
+    else:
+        if 'vector_store' not in st.session_state:
+            st.session_state.vector_store = FAISS.load_local(
+                VECTOR_STORE_PATH,
+                embeddings,
+                allow_dangerous_deserialization=True
+            )
+    
+    # Chat mode
+    if 'vector_store' in st.session_state:
+        if 'messages' not in st.session_state:
+            st.session_state.messages = []
+            
+        # Display chat history
+        for message in st.session_state.messages:
+            st.chat_message("user").write(message["question"])
+            st.chat_message("assistant").write(message["answer"])
+            
+        # User input
+        if question := st.chat_input("Ask your question"):
+            st.chat_message("user").write(question)
+            
+            # Retrieve context and generate response
+            with st.chat_message("assistant"):
+                with st.spinner("Thinking..."):
+                    context = st.session_state.vector_store.similarity_search(question)
+                    context_text = "\n".join([doc.page_content for doc in context])
+                    
+                    prompt = PromptTemplate.from_template("""
+                    You are a helpful and polite legal assistant at Status Law.
+                    You answer in the language in which the question was asked.
+                    Answer the question based on the context provided.
+                    If you cannot answer based on the context, say so politely and offer to contact Status Law directly via the following channels:
+                    - For all users: +32465594521 (landline phone).
+                    - For English and Swedish speakers only: +46728495129 (available on WhatsApp, Telegram, Signal, IMO).
+                    - Provide a link to the contact form: [Contact Form](https://status.law/law-firm-contact-legal-protection/).
+                    Answer professionally but in a friendly manner.
+
+                    Example:
+                    Q: How can I challenge the sanctions?
+                    A: To challenge the sanctions, you should consult with our legal team, who specialize in this area. Please contact us directly for detailed advice. You can fill out our contact form here: [Contact Form](https://status.law/law-firm-contact-legal-protection/).
+
+                    Context: {context}
+                    Question: {question}
+                    """)
+                    
+                    chain = prompt | llm | StrOutputParser()
+                    response = chain.invoke({
+                        "context": context_text,
+                        "question": question
+                    })
+                    
+                    st.write(response)
+                    
+                    # Save chat history
+                    st.session_state.messages.append({
+                        "question": question,
+                        "answer": response
+                    })
+
+if __name__ == "__main__":
+    main()

gitattributes

@@ -0,0 +1,44 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+
+
+* text=auto eol=crlf
+
+*.bin binary
+
+.gitignore text eol=lf
+.gitattributes text eol=lf
+

gitignore

@@ -0,0 +1,4 @@
+*.env
+
+venv
+.streamlit/secrets.toml

requirements.txt

@@ -0,0 +1,9 @@
+streamlit
+langchain-community
+langchain-core
+langchain-huggingface
+langchain-groq
+python-dotenv
+beautifulsoup4
+faiss-cpu
+requests

two-in-one.py

@@ -0,0 +1,150 @@
+import os
+import streamlit as st
+from langchain_groq import ChatGroq
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough, RunnableLambda
+from requests.exceptions import RequestException, Timeout
+
+# Загрузка переменных окружения
+if os.path.exists(".env"):
+    load_dotenv(verbose=True)
+
+# Загрузка API-ключей
+try:
+    GROQ_API_KEY = st.secrets["GROQ_API_KEY"]
+    USER_AGENT = st.secrets["USER_AGENT"]
+    LANGSMITH_TRACING = st.secrets["LANGSMITH_TRACING"] 
+    LANGSMITH_ENDPOINT = st.secrets["LANGSMITH_ENDPOINT"]
+    LANGSMITH_API_KEY = st.secrets["LANGSMITH_API_KEY"]
+    LANGSMITH_PROJECT = st.secrets["LANGSMITH_PROJECT"]
+    OPENAI_API_KEY = st.secrets["OPENAI_API_KEY"]
+except FileNotFoundError:
+    GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+    USER_AGENT = os.getenv("USER_AGENT")
+    LANGSMITH_TRACING = os.getenv("LANGSMITH_TRACING") 
+    LANGSMITH_ENDPOINT = os.getenv("LANGSMITH_ENDPOINT")
+    LANGSMITH_API_KEY = os.getenv("LANGSMITH_API_KEY")
+    LANGSMITH_PROJECT = os.getenv("LANGSMITH_PROJECT")
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+# Проверка API-ключей
+if not all([GROQ_API_KEY, USER_AGENT, LANGSMITH_TRACING, LANGSMITH_ENDPOINT, LANGSMITH_API_KEY, LANGSMITH_PROJECT, OPENAI_API_KEY]):
+    st.error("Ошибка: Не все переменные окружения заданы.")
+    st.stop()
+
+# Инициализация LLM
+try:
+    llm = ChatGroq(model_name="llama-3.3-70b-versatile", temperature=0.6, api_key=GROQ_API_KEY)
+    print("[DEBUG] LLM успешно инициализирован")
+except Exception as e:
+    st.error(f"Ошибка инициализации LLM: {e}")
+    st.stop()
+
+# Инициализация эмбеддингов
+embeddings_model = HuggingFaceEmbeddings(model_name="intfloat/multilingual-e5-large-instruct")
+print("[DEBUG] Модель эмбеддингов загружена")
+
+# Список страниц для анализа
+urls = [
+    "https://status.law",  
+    "https://status.law/about",
+    "https://status.law/careers",
+    "https://status.law/challenging-sanctions",
+    "https://status.law/contact", 
+    "https://status.law/cross-border-banking-legal-issues", 
+    "https://status.law/extradition-defense", 
+    "https://status.law/international-prosecution-protection", 
+    "https://status.law/interpol-red-notice-removal",  
+    "https://status.law/practice-areas",  
+    "https://status.law/reputation-protection",
+    "https://status.law/faq"
+]
+
+# Путь к файлу векторного хранилища
+VECTOR_STORE_PATH = "vector_store"
+
+# Функция для создания базы знаний
+def build_knowledge_base():
+    documents = []
+    for url in urls:
+        try:
+            loader = WebBaseLoader(url)
+            documents.extend(loader.load(timeout=10))
+            st.write(f"[DEBUG] Загружен контент с {url}")
+        except (RequestException, Timeout) as e:
+            st.write(f"[ERROR] Ошибка загрузки страницы {url}: {e}")
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+    chunks = text_splitter.split_documents(documents)
+    st.write(f"[DEBUG] Разбито на {len(chunks)} фрагментов")
+    vector_store = FAISS.from_documents(chunks, embeddings_model)
+    vector_store.save_local(VECTOR_STORE_PATH)
+    st.write("[DEBUG] Векторное хранилище создано и сохранено")
+    return vector_store
+
+# Функция для загрузки базы знаний
+@st.cache_resource
+def load_knowledge_base():
+    if os.path.exists(VECTOR_STORE_PATH):
+        st.write("[DEBUG] Загрузка существующего векторного хранилища")
+        return FAISS.load_local(VECTOR_STORE_PATH, embeddings_model)
+    else:
+        st.write("[DEBUG] Векторное хранилище не найдено, создание нового")
+        return build_knowledge_base()
+
+# Загрузка или создание базы знаний
+vector_store = load_knowledge_base()
+
+# Промпт для бота
+template = """
+You are a helpful legal assistant that answers questions based on information from status.law.
+Answer accurately and concisely.
+Question: {question}
+Only use the provided context to answer the question.
+Context: {context}
+"""
+prompt = PromptTemplate.from_template(template)
+
+# Инициализация цепочки обработки запроса
+if "chain" not in st.session_state:
+    st.session_state.chain = (
+        RunnableLambda(lambda x: {"context": x["context"], "question": x["question"]}) 
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+chain = st.session_state.chain
+
+# Интерфейс Streamlit
+st.set_page_config(page_title="Legal Chatbot", page_icon="🤖")
+st.title("🤖 Legal Chatbot")
+st.write("Этот бот отвечает на юридические вопросы, используя информацию с сайта status.law.")
+
+# Поле для ввода вопроса
+user_input = st.text_input("Введите ваш вопрос:")
+if st.button("Отправить") and user_input:
+    # Поиск релевантных документов
+    retrieved_docs = vector_store.similarity_search(user_input)
+    context_text = "\n\n".join([doc.page_content for doc in retrieved_docs])
+
+    # Генерация ответа
+    response = chain.invoke({"question": user_input, "context": context_text})
+    
+    # Сохранение истории сообщений
+    if "message_history" not in st.session_state:
+        st.session_state.message_history = []
+    st.session_state.message_history.append({"question": user_input, "answer": response})
+    
+    # Вывод ответа
+    st.write(response)
+
+# Вывод истории сообщений
+if "message_history" in st.session_state:
+    st.write("### История сообщений")
+    for msg in st.session_state.message_history:
+        st.write(f"**User:** {msg['question']}")
+        st.write(f"**Bot:** {msg['answer']}")