Update utils.py

utils.py

@@ -1,7 +1,7 @@
 import logging
+import os
+import fitz  
 from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration
-from pdfminer.high_level import extract_text
-from fine_tuning import fine_tune_model
 
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -15,13 +15,52 @@ fine_tuned_model = T5ForConditionalGeneration.from_pretrained(fine_tuned_model_p
 fine_tuned_tokenizer = T5Tokenizer.from_pretrained(fine_tuned_model_path)
 
 
-def pdf_to_text(pdf_path):
+def extract_text_from_pdf(pdf_path):
     try:
-        logger.info(f"Extracting text from PDF: {pdf_path}")
-        return extract_text(pdf_path)
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"PDF file '{pdf_path}' does not exist.")
+        
+        # PDF dosyasından metni çıkar
+        document = fitz.open(pdf_path)
+        text = ""
+        for page_num in range(document.page_count):
+            page = document.load_page(page_num)
+            text += page.get_text("text")
+        
+        print(f"Text extraction successful from {pdf_path}.")
+        return text
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+        raise e
     except Exception as e:
-        logger.error(f"Error while extracting text from PDF: {str(e)}")
-        raise ValueError(f"PDF extraction error: {str(e)}")
+        print(f"An error occurred while extracting text from PDF: {e}")
+        raise e
+
+def split_text_into_chunks(text, chunk_size=1000):
+    words = text.split()
+    chunks = []
+    for i in range(0, len(words), chunk_size):
+        chunk = ' '.join(words[i:i+chunk_size])
+        chunks.append(chunk)
+    return chunks
+
+def batch_process_texts(texts, batch_size=2):
+    batched_results = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i+batch_size]
+        try:
+            combined_text = " ".join(batch)
+            processed_text = some_processing_function(combined_text)
+            batched_results.append(processed_text)
+        except Exception as e:
+            print(f"Error processing batch {i // batch_size + 1}: {e}")
+            continue
+
+    return batched_results
+
+def generate_lesson_from_chunks(chunks):
+    generated_texts = batch_process_texts(chunks)  
+    return ' '.join(generated_texts)
 
 def generate_lesson_from_transcript(doc_text):
     try:
@@ -55,27 +94,3 @@ def refine_with_fine_tuned_model(general_output):
     except Exception as e:
         logger.error(f"Error during refinement with fine-tuned model: {str(e)}")
         return "An error occurred during refinement."
-
-def split_text_into_chunks(text, chunk_size=1000):
-    words = text.split()
-    chunks = []
-    for i in range(0, len(words), chunk_size):
-        chunk = ' '.join(words[i:i+chunk_size])
-        chunks.append(chunk)
-    return chunks
-
-def generate_lesson_from_chunks(chunks):
-    generated_texts = []
-    for chunk in chunks:
-        try:
-            generated_text = pipe(chunk, max_length=500, truncation=True)[0]['generated_text']
-            generated_texts.append(generated_text)
-        except Exception as e:
-            print(f"Error in chunk processing: {str(e)}")
-            continue
-    return ' '.join(generated_texts)
-
-def process_large_text(text):
-    chunks = split_text_into_chunks(text, chunk_size=1000)
-    generated_text = generate_lesson_from_chunks(chunks)
-    return generated_text