made directory changes, after findind the optimal data generation script

experiments/csv_file_generator_iterations/generate_emr_csv_final.py

@@ -0,0 +1,132 @@
+import random
+import csv
+import string
+from pathlib import Path
+
+# Paths
+CURRENT_DIR = Path(__file__).resolve().parent
+IMAGES_DIR = CURRENT_DIR.parent / "data" / "images"
+OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_softlabels.csv"
+
+# Label to triage
+triage_map = {"COVID": "high", "NORMAL": "low", "VIRAL PNEUMONIA": "medium"}
+SAMPLES_PER_CLASS = 300
+
+# Folders
+categories = {
+    "COVID": IMAGES_DIR / "COVID",
+    "NORMAL": IMAGES_DIR / "NORMAL",
+    "VIRAL PNEUMONIA": IMAGES_DIR / "VIRAL PNEUMONIA"
+}
+
+# Shared ambiguous templates
+shared_symptoms = [
+    "Mild cough and slight fever reported.",
+    "General fatigue and throat irritation present.",
+    "Breathing mildly labored during physical exertion.",
+    "No major respiratory distress; mild wheezing noted.",
+    "Occasional chest tightness reported.",
+    "Vital signs mostly stable; slight variation in temperature.",
+]
+
+# Overlapping diagnosis clues
+shared_diagnosis = [
+    "Symptoms could relate to a range of viral infections.",
+    "Presentation not distinctly matching any single infection.",
+    "Further tests required to confirm diagnosis.",
+    "Findings are borderline; clinical judgment advised.",
+    "Observation warranted due to overlapping signs.",
+    "Initial assessment inconclusive."
+]
+
+# Noise sentences
+neutral_noise = [
+    "Patient is cooperative and alert.",
+    "Dietary habits unremarkable.",
+    "Hydration status normal.",
+    "Follow-up advised if symptoms persist.",
+    "No notable family medical history.",
+    "No medications currently administered.",
+]
+
+def random_token():
+    prefix = "ID"
+    letters = ''.join(random.choices(string.ascii_uppercase, k=2))
+    digits = ''.join(random.choices(string.digits, k=2))
+    return f"{prefix}-{letters}{digits}"
+
+def get_oxygen(label):
+    # Soft blur across classes
+    if label == "NORMAL":
+        return random.randint(94, 100)
+    elif label == "VIRAL PNEUMONIA":
+        return random.randint(90, 96)
+    else:
+        return random.randint(87, 94)
+
+def get_temp(label):
+    if label == "NORMAL":
+        return round(random.uniform(97.5, 99.0), 1)
+    else:
+        return round(random.uniform(98.8, 102.5), 1)
+
+def get_age():
+    return random.randint(18, 85)
+
+def get_days():
+    return random.randint(1, 10)
+
+def build_emr(label, i):
+    pid = random_token()
+    age = f"{get_age()}-year-old"
+    days = get_days()
+    temp = get_temp(label)
+    oxygen = get_oxygen(label)
+
+    intro = f"Patient {pid}, a {age}, reports symptoms for {days} days."
+    vitals = f"Temperature recorded at {temp}°F and SPO2 at {oxygen}%."
+
+    # Shared symptoms + blurred logic
+    body = [
+        intro,
+        random.choice(shared_symptoms),
+        vitals,
+        random.choice(shared_diagnosis)
+    ]
+
+    # Optionally inject a mild class-specific clue (with low probability)
+    if random.random() < 0.3:
+        if label == "COVID":
+            body.append("Patient reports recent loss of taste.")
+        elif label == "VIRAL PNEUMONIA":
+            body.append("Chest X-ray shows scattered infiltrates.")
+        elif label == "NORMAL":
+            body.append("No active complaints at this time.")
+
+    # Inject 1–2 noise sentences
+    if random.random() < 0.8:
+        body.insert(random.randint(1, len(body)), random.choice(neutral_noise))
+    if random.random() < 0.5:
+        body.insert(random.randint(1, len(body)), random.choice(neutral_noise))
+
+    random.shuffle(body[1:])  # Keep intro in position 0
+    return " ".join(body)
+
+# Generate records
+records = []
+for label, img_dir in categories.items():
+    image_files = sorted([f for f in img_dir.glob("*") if f.suffix.lower() in [".png", ".jpg", ".jpeg"]])
+    for i in range(SAMPLES_PER_CLASS):
+        image_path = str(random.choice(image_files).relative_to(IMAGES_DIR.parent.parent))
+        text = build_emr(label, i)
+        triage = triage_map[label]
+        records.append([f"{label}-{i+1}", image_path, text, triage])
+
+# Shuffle + write
+random.shuffle(records)
+with open(OUTPUT_FILE, "w", newline="") as f:
+    writer = csv.writer(f)
+    writer.writerow(["patient_id", "image_path", "emr_text", "triage_level"])
+    writer.writerows(records)
+
+print(f"✅ Softlabel EMR dataset generated at {OUTPUT_FILE}")

experiments/{generate_emr_csv_fuzzy.py → csv_file_generator_iterations/generate_emr_csv_v1.py}

@@ -5,10 +5,10 @@ from pathlib import Path
 # Setup paths
 CURRENT_DIR = Path(__file__).resolve().parent
 IMAGES_DIR = CURRENT_DIR.parent / "data" / "images"
-OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_fuzzy.csv"
+OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_extended.csv"
 
 # Sample size
-SAMPLES_PER_CLASS = 300  # 900 total
+SAMPLES_PER_CLASS = 300 # 300 * 3 = 900 total
 
 # Categories and labels
 categories = {
@@ -17,13 +17,14 @@ categories = {
     "VIRAL PNEUMONIA": IMAGES_DIR / "VIRAL PNEUMONIA"
 }
 
+# Triage mapping
 triage_map = {
     "COVID": "high",
     "NORMAL": "low",
     "VIRAL PNEUMONIA": "medium"
 }
 
-# --- Shared & ambiguous templates ---
+# --- Noise Sentences ---
 noise_sentences = [
     "Follow-up scheduled for next week.",
     "Patient advised to maintain hydration and rest.",
@@ -31,17 +32,21 @@ noise_sentences = [
     "Patient remains alert and oriented.",
     "Vitals are within acceptable ranges.",
     "No complications noted during assessment.",
-    "Doctor recommends continued observation.",
     "Patient has no known drug allergies.",
+    "Doctor recommends continued observation.",
     "Supportive care was initiated.",
     "Patient advised to avoid strenuous activity.",
+    "No complications noted during assessment",
+    "No prior history of respiratory illness.",
     "Mild discomfort reported with no severe symptoms.",
     "Symptoms are self-limiting according to patient.",
+    "Patient remains alert and cooperative.",
     "No medication administered at this stage.",
-    "Doctor recommends home rest and observation.",
+    "Doctor recommends home resr and observation.",
     "Evaluation ongoing for possible infection."
 ]
 
+# --- ambiguity sentences ---
 ambiguous_templates = [
     "Mild fever noted. No cough. Patient recently traveled.",
     "Normal oxygen levels observed. Slight wheeze on auscultation.",
@@ -50,45 +55,62 @@ ambiguous_templates = [
     "Slight fatigue without other systemic symptoms."
 ]
 
-# --- Vitals ---
+# --- Vitals & Symptoms ---
 def get_oxygen(label):
-    ranges = {
+    base_ranges = {
         "COVID": (85, 94),
         "VIRAL PNEUMONIA": (88, 95),
         "NORMAL": (96, 99)
     }
-    low, high = ranges[label]
-    oxygen = random.randint(low - 1, high + 1)
+    base_min, base_max = base_ranges[label]
+    # Apply + or - 1 blur, clamping between 80 and 100
+    oxygen = random.randint(base_min - 1, base_max + 1)
     return min(100, max(80, oxygen))
 
 def get_temp(label):
     if label == "NORMAL":
-        temp = random.uniform(96.5, 99.0)
+        base_min, base_max = 97.0, 98.6
     else:
-        temp = random.uniform(98.5, 104.0)
-    return round(temp, 1)
+        base_min, base_max = 99.0, 103.5
+
+    # Apply + or - 0.5°F blur and clamp between 95-105°F
+    temp = random.uniform(base_min - 0.5, base_max + 0.5)
+    return round(min(105.0, max(95.0, temp)), 1)
+    
+def get_days():
+    return random.randint(1, 14)
 
-def get_days(): return random.randint(1, 14)
-def get_age(): return random.randint(18, 85)
+def get_age():
+    return random.randint(18, 80)
 
-# --- Build EMR ---
+# --- Templates ---
 def build_emr(label, i):
     name = f"Patient-{label}-{i+1}"
     age = f"{get_age()}-year-old"
     days = get_days()
     temp = get_temp(label)
     oxygen = get_oxygen(label)
+    
+    # Symptoms Pool
+    symptoms = {
+        "COVID": [
+            f"{name} ({age}) reports fatigue and dry cough for {days} days.",
+            f"{name} complains of shortness of breath and fever of {temp}°F.",
+            f"{name} reports loss of taste. SPO2 at {oxygen}%.",
+        ],
+        "NORMAL": [
+            f"{name} ({age}) presents for routine check-up. Vitals stable.",
+            f"{name} shows no respiratory distress. Oxygen at {oxygen}%.",
+            f"{name} denies any recent illness. Temperature is {temp}°F.",
+        ],
+        "VIRAL PNEUMONIA": [
+            f"{name} ({age}) complains of dry cough for {days} days.",
+            f"{name} experiencing low-grade fever and SPO2 at {oxygen}%.",
+            f"{name} reports breathlessness. X-ray indicates mild infiltrates.",
+        ]
+    }
 
-    # Shared symptoms across labels
-    shared_symptoms = [
-        f"{name} ({age}) reports dry cough and fatigue for {days} days.",
-        f"{name} reports breathlessness. Temp recorded as {temp}°F.",
-        f"{name} is experiencing low oxygen levels at {oxygen}%.",
-        f"{name} complains of throat irritation and tiredness.",
-        f"{name} has fever, but vitals are otherwise stable."
-    ]
-
-    # Label-specific diagnosis
+    # Diagnosis Observations
     diagnosis = {
         "COVID": [
             "Findings suggest viral respiratory infection.",
@@ -97,48 +119,51 @@ def build_emr(label, i):
         ],
         "NORMAL": [
             "No signs of respiratory infection.",
-            "Checkup results within normal limits.",
-            "No abnormal findings detected."
+            "No abnormal findings detected.",
+            "Checkup results within normal limits."
         ],
         "VIRAL PNEUMONIA": [
             "X-ray shows patchy infiltrates.",
-            "Clinical signs indicate viral pneumonia.",
-            "Suspected viral origin of symptoms."
+            "Suspected viral origin of symptoms.",
+            "Clinical signs indicate viral pneumonia."
         ]
     }
 
-    # Build full body
-    emr = [random.choice(shared_symptoms), random.choice(diagnosis[label])]
+    # Construct sentence pool
+    body = [random.choice(symptoms[label]), random.choice(diagnosis[label])]
 
-    # Add ambiguity (~60%)
-    if random.random() < 0.6:
-        emr.insert(random.randint(0, len(emr)), random.choice(ambiguous_templates))
+    # adding ambiguous cases randomly (~70% of cases)
+    if random.random() < 0.7:
+        body.insert(random.randint(0, len(body)), random.choice(ambiguous_templates))
 
-    # Add noise (~90%)
+    # adding noise to 90% of cases
     if random.random() < 0.9:
-        for _ in range(random.randint(1, 2)):
-            emr.insert(random.randint(0, len(emr)), random.choice(noise_sentences))
-
-    random.shuffle(emr)
-    return " ".join(emr)
+        for _ in range(random.randint(1,2)):
+            body.insert(random.randint(0, len(body)), random.choice(noise_sentences))
+    
+    random.shuffle(body)
+    return " ".join(body)
 
-# --- Generate records ---
+# Generate dataset
 records = []
 for label, img_dir in categories.items():
-    files = sorted([f for f in img_dir.glob("*") if f.suffix.lower() in [".png", ".jpg", ".jpeg"]])
+    valid_exts = [".png", ".jpg", ".jpeg"]
+    image_files = sorted(
+        [f for f in img_dir.glob("*") if f.suffix.lower() in valid_exts]
+    )
     for i in range(SAMPLES_PER_CLASS):
-        image_path = str(random.choice(files).relative_to(IMAGES_DIR.parent.parent))
+        patient_id = f"{label}-{i+1}"
+        image_path = str(random.choice(image_files).relative_to(IMAGES_DIR.parent.parent))
         emr_text = build_emr(label, i)
-        triage = triage_map[label]
-        pid = f"{label}-{i+1}"
-        records.append([pid, image_path, emr_text, triage])
+        triage_level = triage_map[label]
+        records.append([patient_id, image_path, emr_text, triage_level])
 
 random.shuffle(records)
 
-# --- Save to CSV ---
+# Save to CSV
 with open(OUTPUT_FILE, "w", newline="") as f:
     writer = csv.writer(f)
     writer.writerow(["patient_id", "image_path", "emr_text", "triage_level"])
     writer.writerows(records)
 
-print(f"✅ Regenerated {len(records)} fuzzy EMR records at: {OUTPUT_FILE}")
+print(f"✅Generated {len(records)} EMR records in {OUTPUT_FILE}")

experiments/csv_file_generator_iterations/generate_emr_csv_v2.py

@@ -0,0 +1,122 @@
+import random
+import csv
+import string
+from pathlib import Path
+
+# Paths
+CURRENT_DIR = Path(__file__).resolve().parent
+IMAGES_DIR = CURRENT_DIR.parent / "data" / "images"
+OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_richfuzzy.csv"
+
+# Label to triage
+triage_map = {"COVID": "high", "NORMAL": "low", "VIRAL PNEUMONIA": "medium"}
+SAMPLES_PER_CLASS = 300
+
+# Folders
+categories = {
+    "COVID": IMAGES_DIR / "COVID",
+    "NORMAL": IMAGES_DIR / "NORMAL",
+    "VIRAL PNEUMONIA": IMAGES_DIR / "VIRAL PNEUMONIA"
+}
+
+# Shared ambiguous templates
+ambiguous_phrases = [
+    "Slight throat irritation without systemic symptoms.",
+    "Mild dyspnea but normal vitals.",
+    "Minor dry cough reported, patient stable.",
+    "Chest X-ray inconclusive.",
+    "No recent exposure or travel noted.",
+    "Intermittent headache without fever.",
+]
+
+# Noise sentences
+neutral_noise = [
+    "Patient is cooperative and alert.",
+    "Dietary habits unremarkable.",
+    "Follow-up recommended if symptoms persist.",
+    "Hydration status is normal.",
+    "No family history of chronic illness.",
+    "Patient expresses concern about possible flu.",
+]
+
+# ---Patient random token genrator ---
+def random_token():
+    prefix = "ID"
+    letters = ''.join(random.choices(string.ascii_uppercase, k=2))
+    digits = ''.join(random.choices(string.digits, k=2))
+    return f"{prefix}-{letters}{digits}"
+
+# Vitals (blurred)
+def get_oxygen(label):
+    base = {"COVID": (85, 94), "VIRAL PNEUMONIA": (89, 96), "NORMAL": (96, 99)}
+    min_, max_ = base[label]
+    return min(100, max(80, random.randint(min_-1, max_+1)))
+
+def get_temp(label):
+    if label == "NORMAL":
+        min_, max_ = 97.0, 98.5
+    else:
+        min_, max_ = 99.0, 103.0
+    return round(random.uniform(min_ - 0.6, max_ + 0.6), 1)
+
+def get_age(): return random.randint(18, 85)
+def get_days(): return random.randint(1, 10)
+
+# EMR generator
+def build_emr(label, i):
+    patient_id = random_token()
+    age = f"{get_age()}-year-old"
+    oxygen = get_oxygen(label)
+    temp = get_temp(label)
+    days = get_days()
+
+    general_intro = f"Patient {patient_id}, a {age}, presents with symptoms for {days} days."
+    vitals = f"Temperature recorded at {temp}°F, SPO2 levels at {oxygen}%."
+
+    # Label-specific (but fuzzy) symptoms
+    symptoms = {
+        "COVID": ["Complains of fatigue and shortness of breath.", "Dry cough with mild fever noted."],
+        "NORMAL": ["No major complaints; here for general checkup.", "Reports good health, no active issues."],
+        "VIRAL PNEUMONIA": ["Persistent cough and mild fever observed.", "Slight wheezing with chest tightness."]
+    }
+
+    diagnosis = {
+        "COVID": ["Viral etiology suspected.", "COVID infection not ruled out."],
+        "NORMAL": ["Unlikely presence of infection.", "Clinical impression is benign."],
+        "VIRAL PNEUMONIA": ["Signs may indicate atypical pneumonia.", "Possible viral infection of lower tract."]
+    }
+
+    body = [
+        general_intro,
+        random.choice(symptoms[label]),
+        vitals,
+        random.choice(diagnosis[label])
+    ]
+
+    # Inject 1–2 ambiguous or neutral sentences
+    if random.random() < 0.8:
+        body.insert(random.randint(1, len(body)), random.choice(ambiguous_phrases))
+    if random.random() < 0.7:
+        body.insert(random.randint(1, len(body)), random.choice(neutral_noise))
+
+    random.shuffle(body[1:])
+    return " ".join(body)
+
+# Generate records
+records = []
+for label, img_dir in categories.items():
+    image_files = sorted([f for f in img_dir.glob("*") if f.suffix.lower() in [".png", ".jpg", ".jpeg"]])
+    for i in range(SAMPLES_PER_CLASS):
+        image_path = str(random.choice(image_files).relative_to(IMAGES_DIR.parent.parent))
+        text = build_emr(label, i)
+        triage = triage_map[label]
+        records.append([f"{label}-{i+1}", image_path, text, triage])
+
+# Shuffle + Write
+random.shuffle(records)
+with open(OUTPUT_FILE, "w", newline="") as f:
+    writer = csv.writer(f)
+    writer.writerow(["patient_id", "image_path", "emr_text", "triage_level"])
+    writer.writerows(records)
+
+print(f"✅ Rich fuzzy EMR dataset saved at {OUTPUT_FILE}")

experiments/csv_file_iterations/emr_records.csv

@@ -0,0 +1,91 @@
+patient_id,image_path,emr_text,triage_level
+COVID-1,data/images/COVID/COVID-1.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-2,data/images/COVID/COVID-10.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-3,data/images/COVID/COVID-100.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-4,data/images/COVID/COVID-1000.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-5,data/images/COVID/COVID-1001.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-6,data/images/COVID/COVID-1002.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-7,data/images/COVID/COVID-1003.png,Progressive difficulty in breathing. Oxygen saturation is below the normal range.,high
+COVID-8,data/images/COVID/COVID-1004.png,Progressive difficulty in breathing. Oxygen saturation is below the normal range.,high
+COVID-9,data/images/COVID/COVID-1005.png,Progressive difficulty in breathing. Oxygen saturation is below the normal range.,high
+COVID-10,data/images/COVID/COVID-1006.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-11,data/images/COVID/COVID-1007.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-12,data/images/COVID/COVID-1008.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-13,data/images/COVID/COVID-1009.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-14,data/images/COVID/COVID-101.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-15,data/images/COVID/COVID-1010.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-16,data/images/COVID/COVID-1011.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-17,data/images/COVID/COVID-1012.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-18,data/images/COVID/COVID-1013.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-19,data/images/COVID/COVID-1014.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-20,data/images/COVID/COVID-1015.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-21,data/images/COVID/COVID-1016.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-22,data/images/COVID/COVID-1017.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-23,data/images/COVID/COVID-1018.png,Progressive difficulty in breathing. Oxygen saturation is below the normal range.,high
+COVID-24,data/images/COVID/COVID-1019.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-25,data/images/COVID/COVID-102.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-26,data/images/COVID/COVID-1020.png,Progressive difficulty in breathing. Oxygen saturation is below the normal range.,high
+COVID-27,data/images/COVID/COVID-1021.png,"The patient reports loss of taste and smell, with a persistent cough.",high
+COVID-28,data/images/COVID/COVID-1022.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-29,data/images/COVID/COVID-1023.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+COVID-30,data/images/COVID/COVID-1024.png,"The patient presents with a dry cough, fever, and shortness of breath. Symptoms began 5 days ago.",high
+NORMAL-1,data/images/NORMAL/Normal-1.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-2,data/images/NORMAL/Normal-10.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-3,data/images/NORMAL/Normal-100.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-4,data/images/NORMAL/Normal-1000.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-5,data/images/NORMAL/Normal-10000.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-6,data/images/NORMAL/Normal-10001.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-7,data/images/NORMAL/Normal-10002.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-8,data/images/NORMAL/Normal-10003.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-9,data/images/NORMAL/Normal-10004.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-10,data/images/NORMAL/Normal-10005.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-11,data/images/NORMAL/Normal-10006.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-12,data/images/NORMAL/Normal-10007.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-13,data/images/NORMAL/Normal-10008.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-14,data/images/NORMAL/Normal-10009.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-15,data/images/NORMAL/Normal-1001.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-16,data/images/NORMAL/Normal-10010.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-17,data/images/NORMAL/Normal-10011.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-18,data/images/NORMAL/Normal-10012.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-19,data/images/NORMAL/Normal-10013.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-20,data/images/NORMAL/Normal-10014.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-21,data/images/NORMAL/Normal-10015.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-22,data/images/NORMAL/Normal-10016.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-23,data/images/NORMAL/Normal-10017.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-24,data/images/NORMAL/Normal-10018.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-25,data/images/NORMAL/Normal-10019.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-26,data/images/NORMAL/Normal-1002.png,Routine checkup with no abnormal findings. The patient denies cough or chest pain.,low
+NORMAL-27,data/images/NORMAL/Normal-10020.png,No complaints. Normal vitals and physical exam.,low
+NORMAL-28,data/images/NORMAL/Normal-10021.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-29,data/images/NORMAL/Normal-10022.png,Clear lungs on auscultation. No signs of infection. Chest x-ray was unremarkable.,low
+NORMAL-30,data/images/NORMAL/Normal-10023.png,No complaints. Normal vitals and physical exam.,low
+VIRAL PNEUMONIA-1,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-2,data/images/VIRAL PNEUMONIA/Viral Pneumonia-10.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-3,data/images/VIRAL PNEUMONIA/Viral Pneumonia-100.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-4,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1000.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-5,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1001.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-6,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1002.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-7,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1003.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-8,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1004.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-9,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1005.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-10,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1006.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-11,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1007.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-12,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1008.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-13,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1009.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-14,data/images/VIRAL PNEUMONIA/Viral Pneumonia-101.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-15,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1010.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-16,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1011.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-17,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1012.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-18,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1013.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-19,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1014.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-20,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1015.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-21,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1016.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-22,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1017.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-23,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1018.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-24,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1019.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-25,data/images/VIRAL PNEUMONIA/Viral Pneumonia-102.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-26,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1020.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-27,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1021.png,Crackles are auscultated in the lower lobes. The patient presents with fatigue and mild respiratory distress.,medium
+VIRAL PNEUMONIA-28,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1022.png,The X-ray shows patchy infiltrates in the lungs. The patient is recovering from a recent viral infection.,medium
+VIRAL PNEUMONIA-29,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1023.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium
+VIRAL PNEUMONIA-30,data/images/VIRAL PNEUMONIA/Viral Pneumonia-1024.png,"Mild fever, chest tightness, and dry cough for the past 3 days. Oxygen levels are normal.",medium

src/generate_emr_csv.py

@@ -1,169 +1,132 @@
 import random
 import csv
+import string
 from pathlib import Path
 
-# Setup paths
+# Paths
 CURRENT_DIR = Path(__file__).resolve().parent
 IMAGES_DIR = CURRENT_DIR.parent / "data" / "images"
-OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_extended.csv"
+OUTPUT_FILE = CURRENT_DIR.parent / "data" / "emr_records_softlabels.csv"
 
-# Sample size
-SAMPLES_PER_CLASS = 300 # 300 * 3 = 900 total
+# Label to triage
+triage_map = {"COVID": "high", "NORMAL": "low", "VIRAL PNEUMONIA": "medium"}
+SAMPLES_PER_CLASS = 300
 
-# Categories and labels
+# Folders
 categories = {
     "COVID": IMAGES_DIR / "COVID",
     "NORMAL": IMAGES_DIR / "NORMAL",
     "VIRAL PNEUMONIA": IMAGES_DIR / "VIRAL PNEUMONIA"
 }
 
-# Triage mapping
-triage_map = {
-    "COVID": "high",
-    "NORMAL": "low",
-    "VIRAL PNEUMONIA": "medium"
-}
+# Shared ambiguous templates
+shared_symptoms = [
+    "Mild cough and slight fever reported.",
+    "General fatigue and throat irritation present.",
+    "Breathing mildly labored during physical exertion.",
+    "No major respiratory distress; mild wheezing noted.",
+    "Occasional chest tightness reported.",
+    "Vital signs mostly stable; slight variation in temperature.",
+]
 
-# --- Noise Sentences ---
-noise_sentences = [
-    "Follow-up scheduled for next week.",
-    "Patient advised to maintain hydration and rest.",
-    "No previous episodes of similar symptoms.",
-    "Patient remains alert and oriented.",
-    "Vitals are within acceptable ranges.",
-    "No complications noted during assessment.",
-    "Patient has no known drug allergies.",
-    "Doctor recommends continued observation.",
-    "Supportive care was initiated.",
-    "Patient advised to avoid strenuous activity.",
-    "No complications noted during assessment",
-    "No prior history of respiratory illness.",
-    "Mild discomfort reported with no severe symptoms.",
-    "Symptoms are self-limiting according to patient.",
-    "Patient remains alert and cooperative.",
-    "No medication administered at this stage.",
-    "Doctor recommends home resr and observation.",
-    "Evaluation ongoing for possible infection."
+# Overlapping diagnosis clues
+shared_diagnosis = [
+    "Symptoms could relate to a range of viral infections.",
+    "Presentation not distinctly matching any single infection.",
+    "Further tests required to confirm diagnosis.",
+    "Findings are borderline; clinical judgment advised.",
+    "Observation warranted due to overlapping signs.",
+    "Initial assessment inconclusive."
 ]
 
-# --- ambiguity sentences ---
-ambiguous_templates = [
-    "Mild fever noted. No cough. Patient recently traveled.",
-    "Normal oxygen levels observed. Slight wheeze on auscultation.",
-    "Patient reports chest discomfort but vitals are stable.",
-    "No known exposure. Minor throat irritation present.",
-    "Slight fatigue without other systemic symptoms."
+# Noise sentences
+neutral_noise = [
+    "Patient is cooperative and alert.",
+    "Dietary habits unremarkable.",
+    "Hydration status normal.",
+    "Follow-up advised if symptoms persist.",
+    "No notable family medical history.",
+    "No medications currently administered.",
 ]
 
-# --- Vitals & Symptoms ---
+def random_token():
+    prefix = "ID"
+    letters = ''.join(random.choices(string.ascii_uppercase, k=2))
+    digits = ''.join(random.choices(string.digits, k=2))
+    return f"{prefix}-{letters}{digits}"
+
 def get_oxygen(label):
-    base_ranges = {
-        "COVID": (85, 94),
-        "VIRAL PNEUMONIA": (88, 95),
-        "NORMAL": (96, 99)
-    }
-    base_min, base_max = base_ranges[label]
-    # Apply + or - 1 blur, clamping between 80 and 100
-    oxygen = random.randint(base_min - 1, base_max + 1)
-    return min(100, max(80, oxygen))
+    # Soft blur across classes
+    if label == "NORMAL":
+        return random.randint(94, 100)
+    elif label == "VIRAL PNEUMONIA":
+        return random.randint(90, 96)
+    else:
+        return random.randint(87, 94)
 
 def get_temp(label):
     if label == "NORMAL":
-        base_min, base_max = 97.0, 98.6
+        return round(random.uniform(97.5, 99.0), 1)
     else:
-        base_min, base_max = 99.0, 103.5
-
-    # Apply + or - 0.5°F blur and clamp between 95-105°F
-    temp = random.uniform(base_min - 0.5, base_max + 0.5)
-    return round(min(105.0, max(95.0, temp)), 1)
-    
-def get_days():
-    return random.randint(1, 14)
+        return round(random.uniform(98.8, 102.5), 1)
 
 def get_age():
-    return random.randint(18, 80)
+    return random.randint(18, 85)
+
+def get_days():
+    return random.randint(1, 10)
 
-# --- Templates ---
 def build_emr(label, i):
-    name = f"Patient-{label}-{i+1}"
+    pid = random_token()
     age = f"{get_age()}-year-old"
     days = get_days()
     temp = get_temp(label)
     oxygen = get_oxygen(label)
-    
-    # Symptoms Pool
-    symptoms = {
-        "COVID": [
-            f"{name} ({age}) reports fatigue and dry cough for {days} days.",
-            f"{name} complains of shortness of breath and fever of {temp}°F.",
-            f"{name} reports loss of taste. SPO2 at {oxygen}%.",
-        ],
-        "NORMAL": [
-            f"{name} ({age}) presents for routine check-up. Vitals stable.",
-            f"{name} shows no respiratory distress. Oxygen at {oxygen}%.",
-            f"{name} denies any recent illness. Temperature is {temp}°F.",
-        ],
-        "VIRAL PNEUMONIA": [
-            f"{name} ({age}) complains of dry cough for {days} days.",
-            f"{name} experiencing low-grade fever and SPO2 at {oxygen}%.",
-            f"{name} reports breathlessness. X-ray indicates mild infiltrates.",
-        ]
-    }
-
-    # Diagnosis Observations
-    diagnosis = {
-        "COVID": [
-            "Findings suggest viral respiratory infection.",
-            "Signs consistent with COVID-19 infection.",
-            "Clinical features align with COVID diagnosis."
-        ],
-        "NORMAL": [
-            "No signs of respiratory infection.",
-            "No abnormal findings detected.",
-            "Checkup results within normal limits."
-        ],
-        "VIRAL PNEUMONIA": [
-            "X-ray shows patchy infiltrates.",
-            "Suspected viral origin of symptoms.",
-            "Clinical signs indicate viral pneumonia."
-        ]
-    }
-
-    # Construct sentence pool
-    body = [random.choice(symptoms[label]), random.choice(diagnosis[label])]
-
-    # adding ambiguous cases randomly (~70% of cases)
-    if random.random() < 0.7:
-        body.insert(random.randint(0, len(body)), random.choice(ambiguous_templates))
-
-    # adding noise to 90% of cases
-    if random.random() < 0.9:
-        for _ in range(random.randint(1,2)):
-            body.insert(random.randint(0, len(body)), random.choice(noise_sentences))
-    
-    random.shuffle(body)
+
+    intro = f"Patient {pid}, a {age}, reports symptoms for {days} days."
+    vitals = f"Temperature recorded at {temp}°F and SPO2 at {oxygen}%."
+
+    # Shared symptoms + blurred logic
+    body = [
+        intro,
+        random.choice(shared_symptoms),
+        vitals,
+        random.choice(shared_diagnosis)
+    ]
+
+    # Optionally inject a mild class-specific clue (with low probability)
+    if random.random() < 0.3:
+        if label == "COVID":
+            body.append("Patient reports recent loss of taste.")
+        elif label == "VIRAL PNEUMONIA":
+            body.append("Chest X-ray shows scattered infiltrates.")
+        elif label == "NORMAL":
+            body.append("No active complaints at this time.")
+
+    # Inject 1–2 noise sentences
+    if random.random() < 0.8:
+        body.insert(random.randint(1, len(body)), random.choice(neutral_noise))
+    if random.random() < 0.5:
+        body.insert(random.randint(1, len(body)), random.choice(neutral_noise))
+
+    random.shuffle(body[1:])  # Keep intro in position 0
     return " ".join(body)
 
-# Generate dataset
+# Generate records
 records = []
 for label, img_dir in categories.items():
-    valid_exts = [".png", ".jpg", ".jpeg"]
-    image_files = sorted(
-        [f for f in img_dir.glob("*") if f.suffix.lower() in valid_exts]
-    )
+    image_files = sorted([f for f in img_dir.glob("*") if f.suffix.lower() in [".png", ".jpg", ".jpeg"]])
     for i in range(SAMPLES_PER_CLASS):
-        patient_id = f"{label}-{i+1}"
         image_path = str(random.choice(image_files).relative_to(IMAGES_DIR.parent.parent))
-        emr_text = build_emr(label, i)
-        triage_level = triage_map[label]
-        records.append([patient_id, image_path, emr_text, triage_level])
+        text = build_emr(label, i)
+        triage = triage_map[label]
+        records.append([f"{label}-{i+1}", image_path, text, triage])
 
+# Shuffle + write
 random.shuffle(records)
-
-# Save to CSV
 with open(OUTPUT_FILE, "w", newline="") as f:
     writer = csv.writer(f)
     writer.writerow(["patient_id", "image_path", "emr_text", "triage_level"])
     writer.writerows(records)
 
-print(f"✅Generated {len(records)} EMR records in {OUTPUT_FILE}")
+print(f"✅ Softlabel EMR dataset generated at {OUTPUT_FILE}")

src/train.py

@@ -56,7 +56,7 @@ def train_model(mode="multimodal"): # Function to instantiate model and data, tr
     config = load_config()
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Use GPU if available or else use CPU
     
-    dataset_dir = os.path.join(base_dir, "data", "emr_records_fuzzy.csv")
+    dataset_dir = os.path.join(base_dir, "data", "emr_records_softlabels.csv")
     dataset = TriageDataset(
         csv_file=dataset_dir,
         mode=mode
@@ -152,11 +152,11 @@ def train_model(mode="multimodal"): # Function to instantiate model and data, tr
         print(f"Val Accuracy: {val_acc_epoch:.4f}, F1 Score: {val_f1:.4f}")
 
     # Save model
-    model_path = os.path.join(base_dir, f"medi_llm_model_fuzzy{mode}.pth")
+    model_path = os.path.join(base_dir, f"medi_llm_model_softlabels{mode}.pth")
     torch.save(model.state_dict(), model_path) # Saves the model weights only not total architecture to reuse later
 
     # Plot accuracy
-    plot_path = os.path.join(base_dir, "assets", f"model_training_curve_fuzzy{mode}.png")
+    plot_path = os.path.join(base_dir, "assets", f"model_training_curve_softlabels{mode}.png")
     plt.plot(train_acc, label="Train Acc")
     plt.plot(val_acc, label="Val Acc")
     plt.legend()