Spaces:
Sleeping
Sleeping
File size: 5,963 Bytes
35bda59 9ff2eb6 35bda59 9ff2eb6 35bda59 9ff2eb6 35bda59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | # from pypdf import PdfReader
# import docx
# from transformers.pipelines import pipeline
# import streamlit as st
# def extract_text(file):
# text = ""
# if file.name.endswith(".pdf"):
# try:
# reader = PdfReader(file)
# for page in reader.pages:
# text += page.extract_text() + "\n"
# except Exception as e:
# st.error(f"Error reading PDF {file.name}: {e}")
# return ""
# elif file.name.endswith(".docx"):
# try:
# document = docx.Document(file)
# for paragraph in document.paragraphs:
# text += paragraph.text + "\n"
# except Exception as e:
# st.error(f"Error reading DOCX {file.name}: {e}")
# return ""
# return text
# def chunk_text(text, chunk_size=500, overlap=50):
# chunks = []
# start = 0
# while start < len(text):
# end = start + chunk_size
# chunk = text[start:end]
# chunks.append(chunk)
# start = end - overlap
# return chunks
# def get_embeddings(texts):
# try:
# embedding_model = pipeline(
# 'document-question-answering',
# "sentence-transformers/all-MiniLM-L6-v2"
# ) # Example model
# embeddings = embedding_model(texts)
# return embeddings
# except Exception as e:
# st.error(f"Error generating embeddings: {e}")
# return []
# def process_files(files):
# all_chunks = []
# all_embeddings = []
# chunks_metadata = []
# for file in files:
# text = extract_text(file)
# if not text: # Skip files that failed to process
# continue
# chunks = chunk_text(text)
# embeddings = get_embeddings(chunks)
# if not embeddings: # Skip files that failed to embed
# continue
# all_chunks.extend(chunks)
# all_embeddings.extend(embeddings)
# for i, chunk in enumerate(chunks):
# chunks_metadata.append({"file_name": file.name, "chunk_index": i})
# print(f"Processed {len(files)} files, {len(all_chunks)} chunks generated.")
# return all_chunks, all_embeddings, chunks_metadata
import pypdf
from docx import Document
from transformers.pipelines import pipeline
from sentence_transformers import SentenceTransformer
import streamlit as st
import numpy as np
import os
def extract_text(file):
text = ""
# Check if the input is a file path (string) or a file-like object
if isinstance(file, str):
file_name = os.path.basename(file)
try:
with open(file, 'rb') as f: # Open in binary mode
if file_name.endswith(".pdf"):
print('Processing pdf file.................\n')
reader = pypdf.PdfReader(f)
for page in reader.pages:
text += page.extract_text() + "\\n"
elif file_name.endswith(".docx"):
document = Document(f)
print('Processing DOCX file.................\n')
for paragraph in document.paragraphs:
if paragraph.text.strip(): # Check if the paragraph is not empty
text += paragraph.text + "\\n"
except FileNotFoundError:
st.error(f"Error: File not found at {file}")
return ""
except Exception as e:
st.error(f"Error reading {file_name}: {e}")
return ""
else: # Assume it's a file-like object (e.g., from Streamlit file_uploader)
file_name = file.name
try:
if file_name.endswith(".pdf"):
reader = pypdf.PdfReader(file)
for page in reader.pages:
text += page.extract_text() + "\\n"
elif file_name.endswith(".docx"):
document = Document(file)
for paragraph in document.paragraphs:
text += paragraph.text + "\\n"
except Exception as e:
st.error(f"Error reading {file_name}: {e}")
return ""
return text
def chunk_text(text, chunk_size=500, overlap=50):
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
chunk = text[start:end]
chunks.append(chunk)
start = end - overlap
return chunks
def get_embeddings(texts)-> np.ndarray:
try:
# embedding_model = pipeline(
# "sentence-transformers/all-MiniLM-L6-v2"
# ) # Example model
# embeddings = embedding_model(texts)
model = SentenceTransformer("sujet-ai/Marsilia-Embeddings-FR-Base")
embeddings = model.encode(texts)
print(f"Generated {len(embeddings)} embeddings.")
return embeddings
except Exception as e:
st.error(f"Error generating embeddings: {e}")
return []
def process_files(files):
all_chunks = []
all_embeddings = []
chunks_metadata = []
for file in files:
print(f"Processing file: {file.name if hasattr(file, 'name') else os.path.basename(file)}")
text = extract_text(file)
if not text: # Skip files that failed to process
print(f"Skipping file {file.name if hasattr(file, 'name') else os.path.basename(file)} due to extraction error.")
continue
print(f"Chunking text...{file.name if hasattr(file, 'name') else os.path.basename(file)}\n")
chunks = chunk_text(text)
embeddings = get_embeddings(chunks)
# if not embeddings: # Skip files that failed to embed
# continue
all_chunks.extend(chunks)
all_embeddings.extend(embeddings)
for i, chunk in enumerate(chunks):
chunks_metadata.append({"file_name": file.name if hasattr(file, 'name') else os.path.basename(file), "chunk_index": i})
return all_chunks, all_embeddings, chunks_metadata
|