first app

app.py

@@ -0,0 +1,95 @@
+import streamlit as st
+import pymupdf as fitz
+import pyperclip
+from utils.audit.audit_doc import audit_descriptif
+
+# Function to extract text from PDF
+def extract_text_from_pdf(file):
+    document = fitz.open(stream=file.read(), filetype="pdf")
+    full_text = ""
+    for page_num in range(len(document)):
+        page = document.load_page(page_num)
+        text = page.get_text("text")
+        full_text += text
+    return full_text
+
+# Function to classify file type
+def classify_file(file):
+    if file.type.startswith("image/"):
+        return "image"
+    elif file.type == "application/pdf":
+        return "pdf"
+    elif file.type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        return "word"
+    elif file.type.startswith("audio/"):
+        return "audio"
+    elif file.type.startswith("text/"):
+        return "text"
+    else:
+        return "unknown"
+
+def main():
+    # Streamlit app
+    st.title("AUDIT DES DOCUMENTS")
+
+    if "audit" not in st.session_state:
+        st.session_state.audit = {}
+    if "name_file" not in st.session_state:
+        st.session_state.name_file = ""
+
+    # File uploader
+    uploaded_file = st.file_uploader("Télécharger un documents")
+
+    if uploaded_file is not None:
+        type = classify_file(uploaded_file)
+        if type == "pdf":
+
+            if st.session_state.name_file != uploaded_file.name:
+                st.session_state.name_file = uploaded_file.name
+                with st.spinner("Analyse du document..."):
+                    st.session_state.audit = audit_descriptif(uploaded_file)
+            audit = st.session_state.audit
+
+            #global audit
+            audit_simplified = {
+                "Nombre de pages": audit["number_of_pages"],
+                "Nombre d'images": audit["number_of_images"],
+                "Nombre de liens": audit["number_of_links"],
+                "Nombre de tableaux": audit["number_of_tables"],
+                "Nombre de tokens": audit["number_of_tokens"],
+                "Nombre de mots": audit["number_of_words"]
+            }
+
+            well_formatted_audit = "Audit descriptif\n"
+            for key, value in audit_simplified.items():
+                well_formatted_audit += f"- {key}: {value}\n"
+
+            st.write("### Audit de tout le document")
+            st.code(well_formatted_audit)
+
+            #audit par page
+            with st.expander("Audit par page"):
+                number = st.number_input("Numéro de page", min_value=1, max_value=audit["number_of_pages"], value=1)
+                audit_page = audit[f"page_{number-1}"]
+                audit_page = {
+                    "Nombre d'images": audit_page["number_of_images"],
+                    "Nombre de liens": audit_page["number_of_links"],
+                    "Nombre de tableaux": audit_page["number_of_tables"],
+                    "Nombre de tokens": audit_page["number_of_tokens"],
+                    "Nombre de mots": audit_page["number_of_words"]
+                }
+                well_formatted_audit_page = "Audit descriptif\n"
+                for key, value in audit_page.items():
+                    well_formatted_audit_page += f"- {key}: {value}\n"
+                
+                st.code(well_formatted_audit_page)
+
+            # # Button to copy text to clipboard
+            # if st.button("Copy to Clipboard"):
+            #     pyperclip.copy(audit)
+            #     st.success("Text copied to clipboard successfully!")
+            # else:
+            #     st.info("Please upload a PDF file to extract text.")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+pyperclip
+tiktoken
+pydub 
+numpy 
+scipy
+textstat

utils/audit/audit_audio.py

@@ -0,0 +1,28 @@
+import streamlit as st
+import numpy as np
+import scipy.io.wavfile as wavfile
+from pydub import AudioSegment
+import io
+
+# Function to calculate SNR
+def calculate_snr(audio_data):
+    signal = audio_data
+    noise = audio_data - np.mean(audio_data)
+    signal_power = np.mean(signal ** 2)
+    noise_power = np.mean(noise ** 2)
+    snr = 10 * np.log10(signal_power / noise_power)
+    return snr
+
+# Function to evaluate audio quality
+def evaluate_audio_quality(file):
+    audio = AudioSegment.from_file(file)
+    audio_data = np.array(audio.get_array_of_samples())
+    
+    # Calculate volume
+    volume = audio.dBFS
+    
+    # Calculate SNR
+    snr = calculate_snr(audio_data)
+    
+    return volume, snr
+

utils/audit/audit_doc.py

@@ -0,0 +1,96 @@
+
+import pymupdf
+import tiktoken
+import textstat
+
+
+def evaluate_text_quality(text: str) -> dict:
+    # Calculate readability metrics
+    flesch_reading_ease = textstat.flesch_reading_ease(text)
+    flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
+    gunning_fog = textstat.gunning_fog(text)
+    smog_index = textstat.smog_index(text)
+    automated_readability_index = textstat.automated_readability_index(text)
+
+    # Normalize readability scores to a 0-1 scale
+    def normalize_score(score, min_score, max_score):
+        return (score - min_score) / (max_score - min_score)
+
+    # Normalize each readability score
+    n_flesch_reading_ease = normalize_score(flesch_reading_ease, 0, 100)
+    n_flesch_kincaid_grade = 1 - normalize_score(flesch_kincaid_grade, 0, 18)  # Higher is more difficult
+    n_gunning_fog = 1 - normalize_score(gunning_fog, 0, 18)  # Higher is more difficult
+    n_smog_index = 1 - normalize_score(smog_index, 0, 18)  # Higher is more difficult
+    n_automated_readability_index = 1 - normalize_score(automated_readability_index, 0, 18)  # Higher is more difficult
+
+    # Weights for each metric (adjust these as needed)
+    weights = {
+        "flesch_reading_ease": 0.25,
+        "flesch_kincaid_grade": 0.25,
+        "gunning_fog": 0.2,
+        "smog_index": 0.15,
+        "automated_readability_index": 0.15
+    }
+
+    # Calculate the global readability score
+    global_score = (
+        n_flesch_reading_ease * weights["flesch_reading_ease"] +
+        n_flesch_kincaid_grade * weights["flesch_kincaid_grade"] +
+        n_gunning_fog * weights["gunning_fog"] +
+        n_smog_index * weights["smog_index"] +
+        n_automated_readability_index * weights["automated_readability_index"]
+    )
+
+    # Scale the global score to 0-5
+    global_score_0_5 = global_score * 5
+
+def count_tokens(input_string: str) -> int:
+    tokenizer = tiktoken.get_encoding("cl100k_base")
+    tokens = tokenizer.encode(input_string)
+    return len(tokens)
+
+def audit_descriptif(file) -> dict:
+    document = pymupdf.open(stream=file.read())
+
+    audit_dict_doc = {
+        "number_of_pages": len(document),
+        "number_of_images": 0,
+        "number_of_links": 0,
+        "number_of_tables": 0,
+        "number_of_tokens": 0,
+        "number_of_words": 0
+    }
+
+    for page in document:
+
+        audit_dict_page = {}
+        #number of images
+        number_images = len(page.get_images())
+        audit_dict_page["number_of_images"] = number_images
+        audit_dict_doc["number_of_images"] += number_images
+
+
+        #number of links
+        number_links = len(page.get_links())
+        audit_dict_page["number_of_links"] = number_links
+        audit_dict_doc["number_of_links"] += number_links
+
+        #number of tables
+        number_tables = len(page.find_tables().tables)
+        audit_dict_page["number_of_tables"] = number_tables
+        audit_dict_doc["number_of_tables"] += number_tables
+
+        #number of tokens and words
+        text = page.get_text("text")
+        number_tokens = count_tokens(text)
+        number_words = len(text.split())
+
+        audit_dict_page["number_of_tokens"] = count_tokens(text)
+        audit_dict_page["number_of_words"] = len(text.split())
+
+        audit_dict_doc["number_of_tokens"] += number_tokens
+        audit_dict_doc["number_of_words"] += number_words
+
+        audit_dict_doc[f"page_{page.number}"] = audit_dict_page
+
+    return audit_dict_doc