Upload 23 files

.gitattributes

@@ -1,35 +1,41 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_indexes/2021_CDU-CSU_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_indexes/2021_FDP_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_indexes/2021_Freie[[:space:]]wah_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_indexes/2021_Greens_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_indexes/2021_The[[:space:]]Left_index/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,13 @@
----
-title: German Political Chatbot
-emoji: 📊
-colorFrom: yellow
-colorTo: pink
-sdk: streamlit
-sdk_version: 1.41.1
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: German Political Chatbot
+emoji: 📊
+colorFrom: yellow
+colorTo: pink
+sdk: streamlit
+sdk_version: 1.41.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,110 @@
+import streamlit as st
+from llm import GeminiModel, api_key as SECRET_KEY
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_community.vectorstores import FAISS
+
+
+
+class RAGEnabledModel:
+    def __init__(self):
+        self.prompt = """
+You are a helpful chat assistant who provides information about the public statements 
+and policy positions of different political parties in Germany for the upcoming 2025 elections. 
+When the user asks a question, you should respond in the same language they used 
+(e.g., if they ask in German, respond in German; if in English, respond in English).
+
+Focus on factual information regarding each party’s stance, referencing relevant 
+policy areas such as economy, immigration, healthcare, the environment, and so on. 
+Stay neutral and objective, providing factual information without bias or personal 
+political opinions. Search online to find up-to-date latest information.
+"""
+
+        # Load FAISS vector store
+        self.vector_db = FAISS.load_local("./faiss_index", GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=SECRET_KEY),allow_dangerous_deserialization=True)
+
+        # Instantiate the GeminiModel (replace with actual import or code)
+        self.model = GeminiModel()
+
+    def retrieve_documents(self, query):
+        """Retrieve relevant documents from the FAISS vector database."""
+        results = self.vector_db.similarity_search(query, k=5)
+        return results
+
+    def predict(self, text, history):
+        """Perform RAG-enabled prediction."""
+        # Step 1: Retrieve relevant documents
+        documents = self.retrieve_documents(text)
+
+        # Step 2: Incorporate retrieved documents into the prompt
+        context = "\n\n".join([doc.page_content for doc in documents])
+        augmented_prompt = f"{self.prompt}\n\nRelevant Context:\n{context}\n\nUser Query: {text}"
+
+        # Step 3: Use the model for prediction
+        outp, pricing = self.model.predict(
+            augmented_prompt,
+            history=history,
+            grounding_threshold=0.15
+        )
+        return outp
+
+####################################
+# 2) Streamlit application layout  #
+####################################
+def main():
+    st.set_page_config(page_title="German 2025 Elections - Political Parties", layout="centered")
+    st.title("German Political Parties' Statements for the 2025 Elections")
+
+    # Initialize the conversation history
+    if "history" not in st.session_state:
+        # We'll store (speaker, message) tuples in this list
+        st.session_state.history = []
+
+    # Create an instance of our RegularModel
+    model = RAGEnabledModel()
+
+    #############################
+    # 3) Chat-style input form  #
+    #############################
+    with st.form(key="user_form"):
+        user_input = st.text_input(
+            "You:",
+            placeholder="Ask about a political party's stance on any policy in Germany (2025 elections)..."
+        )
+        submitted = st.form_submit_button("Send")
+
+    ##########################
+    # 4) Handle user submit  #
+    ##########################
+    if submitted and user_input:
+        # Save the user message
+        st.session_state.history.append(("user", user_input))
+
+        # Instruct or rely on your model to answer in the user's language
+        # Optionally, you could do some language detection and pass it along:
+        # For example, using langdetect:
+        # from langdetect import detect
+        # user_lang = detect(user_input)
+        # instruction = f"Please reply in {user_lang}."
+        # combined_input = f"{instruction}\n\nUser asked: {user_input}"
+
+        # But if your model can automatically detect & respond in the same language,
+        # you can simply pass the user_input.
+        response = model.predict(user_input, st.session_state.history)
+
+        # Save the model's response
+        st.session_state.history.append(("bot", response))
+
+    ##################################
+    # 5) Display the chat messages   #
+    ##################################
+    for speaker, message in st.session_state.history:
+        if speaker == "user":
+            st.markdown(f"**You**: {message}")
+        else:
+            st.markdown(f"**Assistant**: {message}")
+
+###################################
+# 6) Entry point for the app      #
+###################################
+if __name__ == "__main__":
+    main()

faiss_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa51ba1ade1dd3bc411db82620130ef35562859ca0eed117b55b336db73a605
+size 7154733

faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3a05204ad5b9b86d755572f37a2c26d9cbd8c687c7c53f4ce50723687d1465f
+size 1921037

faiss_indexes/2021_AfD_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c943d033af8314989ee741837cc227585149fb3958814364b2c5f29ad50e6fed
+size 235017

faiss_indexes/2021_CDU-CSU_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad8c886f5d11a244df07c3b7dab89786beb3c721dd2eedee7bb0d3537f07b9b4
+size 1376301

faiss_indexes/2021_CDU-CSU_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73152129ed656981ba080fbeea3f206cb95d8821a582333255277204154cfade
+size 425199

faiss_indexes/2021_FDP_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7143eb738429d45918dde851b06109be81924f8c38c1664416c270f564522de
+size 1342509

faiss_indexes/2021_FDP_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d2bd7c2b4ee34fafc9dc31346ee6008a726dbcd7722bfdd2b565642d431a0d4
+size 362556

faiss_indexes/2021_Freie wah_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0187492eef61b71459e92b256e78971536ae31428ee5c08e0d771325795f8738
+size 1324077

faiss_indexes/2021_Freie wah_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dbdb92ab9516ed45e851e7644b9f19ee65b583ab7285abff9ef0d43c74a41cd
+size 327166

faiss_indexes/2021_Greens_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f4153203b45222ec5e7b76bcd5be285e6483d5bc8d388ddaa8c85edffcdfbf1
+size 2734125

faiss_indexes/2021_Greens_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6913c5fd21fa78a6267d37bc1e931ea4dfa46081a8fe2c352c75c80509dfb26e
+size 700354

faiss_indexes/2021_SPD_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3563521322671ca3c9e2fc0902e14f43946f7187dec1688e34240632490d59e
+size 221543

faiss_indexes/2021_The Left_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1995822bec2972c5cdbbf4087a5386e61cd5281a8aa3df8c954ba9687e2bd68
+size 2279469

faiss_indexes/2021_The Left_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3708e81986319babbf10c0affb07c71b485360cffcd2d21242f5020161631d22
+size 667322

llm.py

@@ -0,0 +1,74 @@
+
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+import google.generativeai as genai
+import numpy as np
+import os
+from langchain.vectorstores import FAISS
+api_key = os.environ["GOOGLE_GEMINI_API"]
+
+def format_chat_history(chat_history):
+    """Converts chat history from the provided format to the Gemini format."""
+    formatted_messages = []
+    for message in chat_history:
+        role = message[0]
+        if role == "user":
+          role =  "user"
+        # For simplicity, assuming anything not "user" is the assistant
+        elif role =="bot": #You can expand this logic if you have other roles.
+          role ="model"
+
+        formatted_messages.append({"role": role, "parts": message[1]})
+    return formatted_messages
+
+genai.configure(api_key=api_key)
+class GeminiModel:
+    def __init__(self) -> None:
+        self.model = genai.GenerativeModel('gemini-1.5-pro-latest')
+    
+    def predict(self, inp, history, grounding_threshold = 1.0):
+        chat  = self.model.start_chat(history=format_chat_history(history))
+        response = chat.send_message(inp, tools ={"google_search_retrieval": {
+        "dynamic_retrieval_config": {
+            "mode": "unspecified",
+            "dynamic_threshold": grounding_threshold}}})
+        
+        cost = (response.usage_metadata.total_token_count / 1_000_000) * 10
+        txt = response.text.replace('`', '').replace("\n","")
+        if "json" in txt[:4]:
+            txt = txt[4:]
+        return txt, cost
+    
+    def generate_title(self, initial_message):
+        prompt = f"Generate a concise and descriptive title for the following conversation:\n\n{initial_message}\n\nTitle:"
+        response = self.model.generate_content(prompt)
+        title = response.text.strip()
+        return title
+
+class GeminiEmbeddings:
+    def __init__(self) -> None:
+        self.model = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=api_key)
+
+    def predict(self, input):
+        embedding = self.model.embed_query(input)
+        embedding = np.array(embedding).reshape(1, -1).astype('float32')
+        return embedding
+embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=api_key)  # Ensure your OpenAI API key is set in the environment
+
+def query_all_indexes(query):
+    indexes_path = "faiss_indexes"
+    results = []
+
+    for index_dir in os.listdir(indexes_path):
+        index_path = os.path.join(indexes_path, index_dir)
+        if os.path.isdir(index_path):
+            # Load the FAISS vectorstore
+            faiss_vectorstore = FAISS.load_local(index_path, embeddings,allow_dangerous_deserialization=True)
+
+            # Perform the search query
+            search_results = faiss_vectorstore.similarity_search(query, k=2)  # Adjust k for number of results
+            results.extend([(res.page_content, index_dir) for res in search_results])
+
+    return results
+
+if __name__ == "__main__":
+    print(query_all_indexes("Was sagen Parteien zum Klimawandel"))

make_vecdb.py

@@ -0,0 +1,39 @@
+from langchain.document_loaders import TextLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
+from langchain.vectorstores import FAISS
+import os
+import random
+from llm import api_key as SECRET_KEY
+# Path to the folder containing the text files
+folder_path = "./data"
+
+# Initialize variables
+documents = []
+
+# Load all text files from the folder
+for filename in os.listdir(folder_path):
+    if filename.endswith(".txt"):
+        file_path = os.path.join(folder_path, filename)
+        loader = TextLoader(file_path, encoding="utf-8")
+        documents.extend(loader.load())
+
+# Split the documents into chunks for better vectorization
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=1000,  # Size of each chunk
+    chunk_overlap=200  # Overlap between chunks
+)
+random.shuffle(documents)
+split_docs = text_splitter.split_documents(documents)
+
+# Initialize embeddings (using OpenAIEmbeddings as an example)
+embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=SECRET_KEY)  # Ensure your OpenAI API key is set in the environment
+
+# Create the FAISS vectorstore
+faiss_vectorstore = FAISS.from_documents(split_docs, embeddings)
+
+# Save the FAISS vectorstore to disk
+output_path = "faiss_index"
+faiss_vectorstore.save_local(output_path)
+
+print(f"FAISS vector database created and saved to: {output_path}")

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+google-generativeai
+langchain-community
+langchain
+langchain-core
+langchain-google-genai==2.0.7