replace links and usernames for transformers

main-script.py

@@ -145,8 +145,9 @@ def process_and_save(file_path: str, raw_text: str, url: str, title: str, scrape
     Processes the raw text into two versions (Transformers & TF-IDF) and saves a single JSON.
     """
     # 1. Prepare Transformers Text
-    # transformers_text = scraper_helper.smart_respace(text=raw_text)
-
+    transformers_text = scraper_helper.replace_urls(text=raw_text)
+    transformers_text = scraper_helper.replace_usernames(text=transformers_text)
+    
     # 2. Prepare TF-IDF Text (Heavy cleaning)
     tf_text = raw_text
     tf_text = scraper_helper.lowercase_text(text=tf_text)
@@ -165,7 +166,7 @@ def process_and_save(file_path: str, raw_text: str, url: str, title: str, scrape
         "title": title,
         "url": url,
         "date_scraped": datetime.now().isoformat(), # Added date
-        "transformers_text": raw_text,
+        "transformers_text": transformers_text,
         "tf_idf_text": tf_text
     }