Upload 5 files

app.py

@@ -1,4 +1,5 @@
 import streamlit as st
+from st_copy_to_clipboard import st_copy_to_clipboard
 import re
 import numpy as np
 from doc_preprocessing import process_files, get_embeddings
@@ -41,23 +42,20 @@ def process_query(query):
     return results
 
 def normalize_line_breaks(text):
-    # Replace all \r\n (Windows line endings) with \n
-    text = text.replace('\r\n', '\n')
-    # Replace multiple \n with a single token to preserve paragraph breaks
-    text = re.sub(r'\n{2,}', '<PARA_BREAK>', text)
-    # Replace remaining single \n (i.e. line breaks) with double \n
-    text = text.replace('\n', '  \n\n')
-    # Restore original paragraph breaks
-    text = text.replace('<PARA_BREAK>', '  \n\n')
+    # text = text.replace("\n", "  \n ")
+    # text = text.replace('\n', '  \n ')
+    text = text.replace("\\n", "  \n ")
+
     return text
 
 def display_results(results):
-    for result in results:
-        st.subheader("Answer")
-        st.subheader("Source")
-        st.write(f"File: {result['file_name']}, Chunk: {result['chunk_index']}")
+       for i, result in enumerate(results):
+        st.subheader(f"Réponse {i} :")
+        st.write(f"Source File: {result['file_name']}, Chunk: {result['chunk_index']}")
         st.subheader("Citations depuis le document :")
-        st.markdown(normalize_line_breaks(result["chunk_text"]))
-        print(normalize_line_breaks(result["chunk_text"]))
+        st.write(normalize_line_breaks(result["chunk_text"]))
+        st_copy_to_clipboard(normalize_line_breaks(result["chunk_text"]))
+
+
 if __name__ == "__main__":
     main()

doc_preprocessing.py

@@ -1,3 +1,70 @@
+# from pypdf import PdfReader
+# import docx
+# from transformers.pipelines import pipeline
+# import streamlit as st
+
+# def extract_text(file):
+#     text = ""
+#     if file.name.endswith(".pdf"):
+#         try:
+#             reader = PdfReader(file)
+#             for page in reader.pages:
+#                 text += page.extract_text() + "\n"
+#         except Exception as e:
+#             st.error(f"Error reading PDF {file.name}: {e}")
+#             return ""
+#     elif file.name.endswith(".docx"):
+#         try:
+#             document = docx.Document(file)
+#             for paragraph in document.paragraphs:
+#                 text += paragraph.text + "\n"
+#         except Exception as e:
+#             st.error(f"Error reading DOCX {file.name}: {e}")
+#             return ""
+#     return text
+
+# def chunk_text(text, chunk_size=500, overlap=50):
+#     chunks = []
+#     start = 0
+#     while start < len(text):
+#         end = start + chunk_size
+#         chunk = text[start:end]
+#         chunks.append(chunk)
+#         start = end - overlap
+#     return chunks
+
+# def get_embeddings(texts):
+#     try:
+#         embedding_model = pipeline(
+#             'document-question-answering',
+#             "sentence-transformers/all-MiniLM-L6-v2"
+#         )  # Example model
+#         embeddings = embedding_model(texts)
+#         return embeddings
+#     except Exception as e:
+#         st.error(f"Error generating embeddings: {e}")
+#         return []
+
+# def process_files(files):
+#     all_chunks = []
+#     all_embeddings = []
+#     chunks_metadata = []
+
+#     for file in files:
+#         text = extract_text(file)
+#         if not text:  # Skip files that failed to process
+#             continue
+#         chunks = chunk_text(text)
+#         embeddings = get_embeddings(chunks)
+#         if not embeddings: # Skip files that failed to embed
+#             continue
+
+#         all_chunks.extend(chunks)
+#         all_embeddings.extend(embeddings)
+#         for i, chunk in enumerate(chunks):
+#             chunks_metadata.append({"file_name": file.name, "chunk_index": i})
+#     print(f"Processed {len(files)} files, {len(all_chunks)} chunks generated.")
+#     return all_chunks, all_embeddings, chunks_metadata
 import pypdf
 from docx import Document
 from transformers.pipelines import pipeline
@@ -73,50 +140,26 @@ def get_embeddings(texts)-> np.ndarray:
         st.error(f"Error generating embeddings: {e}")
         return []
 
-def process_files(uploaded_files):
+def process_files(files):
     all_chunks = []
     all_embeddings = []
     chunks_metadata = []
 
-    for uploaded_file in uploaded_files:
-        # Create a temporary file within a writable directory
-        # Hugging Face Spaces usually allows writing to /tmp/ or your app's directory
-        with tempfile.NamedTemporaryFile(delete=False, suffix=f".{uploaded_file.type.split('/')[-1]}") as temp_file:
-            temp_file.write(uploaded_file.getvalue())
-            temp_file_path = temp_file.name
-
-        try:
-            # Now, use temp_file_path to process the file
-            # Your existing process_files logic would go here,
-            # reading from temp_file_path
-            st.write(f"Processing file: {temp_file_path}")
-            # Example: Replace this with your actual processing logic
-            # For PDF/Word, you'd likely use a library like pypdf, python-docx, or langchain loaders
-            if uploaded_file.type == "application/pdf":
-                # Example for PDF:
-                # from pypdf import PdfReader
-                # reader = PdfReader(temp_file_path)
-                # text = ""
-                # for page in reader.pages:
-                #     text += page.extract_text() + "\n"
-                pass # Replace with actual PDF processing
-            elif uploaded_file.type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
-                # Example for DOCX:
-                # from docx import Document
-                # document = Document(temp_file_path)
-                # text = ""
-                # for paragraph in document.paragraphs:
-                #     text += paragraph.text + "\n"
-                pass # Replace with actual DOCX processing
-
-            # Placeholder for chunking, embedding, and metadata extraction
-            # (You'd replace this with your actual logic based on the processed file content)
-            all_chunks.append(f"Content from {uploaded_file.name}")
-            all_embeddings.append([0.1, 0.2, 0.3]) # Replace with actual embeddings
-            chunks_metadata.append({"filename": uploaded_file.name})
-
-        finally:
-            # Clean up the temporary file
-            os.remove(temp_file_path)
+    for file in files:
+        print(f"Processing file: {file.name if hasattr(file, 'name') else os.path.basename(file)}")
+        text = extract_text(file)
+        if not text:  # Skip files that failed to process
+            print(f"Skipping file {file.name if hasattr(file, 'name') else os.path.basename(file)} due to extraction error.")   
+            continue
+        print(f"Chunking text...{file.name if hasattr(file, 'name') else os.path.basename(file)}\n")
+        chunks = chunk_text(text)
+        embeddings = get_embeddings(chunks)
+        # if not embeddings: # Skip files that failed to embed
+        #     continue
 
+        all_chunks.extend(chunks)
+        all_embeddings.extend(embeddings)
+        for i, chunk in enumerate(chunks):
+            chunks_metadata.append({"file_name": file.name if hasattr(file, 'name') else os.path.basename(file), "chunk_index": i})
     return all_chunks, all_embeddings, chunks_metadata
+

pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "pypdf>=5.5.0",
     "python-docx>=1.1.2",
     "sentence-transformers>=4.1.0",
+    "st-copy-to-clipboard>=0.1.6",
     "streamlit>=1.45.1",
     "torch==2.2.0",
     "transformers>=4.51.3",

uv.lock

@@ -477,6 +477,7 @@ dependencies = [
     { name = "pypdf" },
     { name = "python-docx" },
     { name = "sentence-transformers" },
+    { name = "st-copy-to-clipboard" },
     { name = "streamlit" },
     { name = "torch" },
     { name = "transformers" },
@@ -491,6 +492,7 @@ requires-dist = [
     { name = "pypdf", specifier = ">=5.5.0" },
     { name = "python-docx", specifier = ">=1.1.2" },
     { name = "sentence-transformers", specifier = ">=4.1.0" },
+    { name = "st-copy-to-clipboard", specifier = ">=0.1.6" },
     { name = "streamlit", specifier = ">=1.45.1" },
     { name = "torch", specifier = "==2.2.0" },
     { name = "transformers", specifier = ">=4.51.3" },
@@ -1228,6 +1230,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" },
 ]
 
+[[package]]
+name = "st-copy-to-clipboard"
+version = "0.1.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "jinja2" },
+    { name = "streamlit" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/6c/bb8ba23b226259974c3a57d122ceccdfd298e6fc3dd9da193716577c8042/st-copy-to-clipboard-0.1.6.tar.gz", hash = "sha256:76634e74384335f64d80469bfe2ca31d15806a140cb89490321764b235acaae6", size = 5046, upload-time = "2024-03-30T16:41:11.581Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/40/9b/aca2dfca2bebe2850f83bb0536bfb8836ad39ec7f8086299ae8a60c41558/st_copy_to_clipboard-0.1.6-py3-none-any.whl", hash = "sha256:2a3eed0beb550548b04ac8a4e12c0fb09726654a15c302acb035454948b01cd1", size = 6024, upload-time = "2024-03-30T16:41:09.414Z" },
+]
+
 [[package]]
 name = "streamlit"
 version = "1.45.1"