data cleanres

.gitignore

@@ -14,6 +14,7 @@ temp.py
 *.csv
 *.json*
 /*data*/
+notes/
 
 # Front end
 node_modules

scripts/clean.py

@@ -0,0 +1,22 @@
+"""
+Clean up the raw data files so as to curate specifically-required 
+"""
+
+import subprocess
+import threading
+import os
+
+def run_script(script_path):
+    subprocess.run(["python", script_path], cwd=os.path.dirname(__file__))
+
+# Run both cleaning scripts in parallel for speed
+t1 = threading.Thread(target=run_script, args=("cleaners/clean_ms.py",))
+t2 = threading.Thread(target=run_script, args=("cleaners/clean_ds.py",))
+
+t1.start()
+t2.start()
+
+t1.join()
+t2.join()
+
+print("All cleaning scripts completed.")

scripts/cleaners/clean_ds.py

@@ -0,0 +1,56 @@
+"""
+RANDOMLY Takes 10,000 lines from ../raw_data/raw_dialogsum_train.csv, 1,000 lines from ../raw_data/raw_dialogsum_test.csv, and 700 lines from ../raw_data/raw_dialogsum_val.csv. Then converts each one to JSONL.
+
+"""
+import random
+import json
+import csv
+import os
+
+def reservoir_sample_csv(file_path, k):
+    rows = []
+    with open(file_path, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            rows.append(row)
+    if len(rows) <= k:
+        return rows
+    return random.sample(rows, k)
+
+def write_jsonl(rows, output_path):
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        for row in rows:
+            new_data = {
+                "id": row["id"],
+                "original_source": "DialogSum",
+                "dialogue": row["dialogue"],
+                "summary": row["summary"],
+                "topic": row["topic"]
+            }
+            json.dump(new_data, f, indent=2)
+            f.write('\n')
+
+print("Cleaning DialogSum dataset...")
+
+ta = '../raw_data/raw_dialogsum_train.csv'
+tb = '../raw_data/raw_dialogsum_test.csv'
+vc = '../raw_data/raw_dialogsum_val.csv'
+
+train_loc = '../clean1/ds/dialogsum_train_10k.jsonl'
+test_loc = '../clean1/ds/dialogsum_test_1k.jsonl'
+val_loc = '../clean1/ds/dialogsum_val_700.jsonl'
+
+print("Sampling rows from raw data CSV files...")
+
+train_rows = reservoir_sample_csv(ta, 10000)
+test_rows = reservoir_sample_csv(tb, 1000)
+val_rows = reservoir_sample_csv(vc, 700)
+
+print("Collected Samples. Writing to JSONL files...")
+
+write_jsonl(train_rows, train_loc)
+write_jsonl(test_rows, test_loc)
+write_jsonl(val_rows, val_loc)
+
+print("Done")

scripts/cleaners/clean_ms.py

@@ -0,0 +1,59 @@
+"""
+RANDOMLY Takes 10,000 lines from ../raw_data/raw_mediasum_train_data.txt, 1,000 lines from ../raw_data/raw_mediasum_test_data.txt, and 1,000 lines from ../raw_data/raw_mediasum_val_data.txt. Then converts each one to JSONL.
+
+"""
+import random
+import json
+import os
+
+def reservoir_sample(file_path, k):
+    reservoir = []
+    with open(file_path, 'r', encoding='utf-8') as f:
+        for i, line in enumerate(f):
+            if i < k:
+                reservoir.append(line.strip())
+            else:
+                j = random.randint(0, i)
+                if j < k:
+                    reservoir[j] = line.strip()
+    return reservoir
+
+def write_jsonl(lines, output_path):
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        for line in lines:
+            data = json.loads(line)
+            new_data = {
+                "id": data["id"],
+                "original_source": "MediaSum",
+                "url": data["url"],
+                "summary": data["summary"],
+                "transcript": data["utt"],
+                "speaker": data["speaker"]
+            }
+            json.dump(new_data, f, indent=2)
+            f.write('\n')
+
+print("Cleaning Mediasum dataset...")
+
+ta = '../raw_data/raw_mediasum_train_data.txt'
+tb = '../raw_data/raw_mediasum_test_data.txt'
+vc = '../raw_data/raw_mediasum_val_data.txt'
+
+train_loc = '../clean1/ms/mediasum_train_10k.jsonl'
+test_loc = '../clean1/ms/mediasum_test_1k.jsonl'
+val_loc = '../clean1/ms/mediasum_val_1k.jsonl'
+
+print("Sampling lines from raw data files...")
+
+train_lines = reservoir_sample(ta, 10000)
+test_lines = reservoir_sample(tb, 1000)
+val_lines = reservoir_sample(vc, 1000)
+
+print("Collected Samples. Writing to JSONL files...")
+
+write_jsonl(train_lines, train_loc)
+write_jsonl(test_lines, test_loc)
+write_jsonl(val_lines, val_loc)
+
+print("Done")

scripts/pull.py

@@ -3,17 +3,17 @@ Pulls raw samples of 10k each from the [cited in README] datasets used in this p
 In the final version of the training data, a lot of the example outputs are tuned, and they are all merged into a single 
 
 HuggingFace seems to have disabled this functionality.
-Currently trying to see how to work around it.
+Currently trying to see how to work around it
 """
 
 import json
 from datasets import load_dataset
 
 targets = {
-    "mediasum": ("nbroad/mediasum", None, "train"),  # Parquet‑exported version, no loader script needed :contentReference[oaicite:0]{index=0}
-    "dialogsum": ("knkarthick/dialogsum", None, "train"),  # CSV on HF :contentReference[oaicite:1]{index=1}
-    "squality": ("mattercalm/squality", None, "train"),  # assumed generic supported format
-    "msmarco_corpus": ("Hyukkyu/beir-msmarco", "corpus", "train"),  # Parquet migrated version :contentReference[oaicite:2]{index=2}
+    "mediasum": ("nbroad/mediasum", None, "train"),
+    "dialogsum": ("knkarthick/dialogsum", None, "train"),
+    "squality": ("mattercalm/squality", None, "train"),
+    "msmarco_corpus": ("Hyukkyu/beir-msmarco", "corpus", "train"),
 }
 
 for name, (repo, config, split) in targets.items():