joins all the json lyric files into a corpus

tmp.py

@@ -0,0 +1,46 @@
+import os
+import json
+
+input_folder = "rap_dataset"
+output_file = "rap_corpus.txt"
+
+total_songs = 0
+total_chars = 0
+
+print(f"Starting merge: reading from '{input_folder}' -> writing to '{output_file}'\n")
+
+# Open the master output file once
+with open(output_file, "w", encoding="utf-8") as f_out:
+    
+    # Iterate through every JSON file in your dataset folder
+    for filename in os.listdir(input_folder):
+        if filename.endswith(".json"):
+            filepath = os.path.join(input_folder, filename)
+            
+            try:
+                with open(filepath, "r", encoding="utf-8") as f_in:
+                    artist_data = json.load(f_in)
+                    
+                    # artist_data is a list of dictionaries based on your scraper
+                    for song in artist_data:
+                        lyrics = song.get("lyrics", "").strip()
+                        
+                        if lyrics:
+                            # Write lyrics followed by a double newline to separate songs
+                            f_out.write(lyrics + "\n\n")
+                            
+                            total_songs += 1
+                            total_chars += len(lyrics)
+                            
+                print(f"Merged: {filename}")
+                
+            except Exception as e:
+                print(f"Error processing {filename}: {e}")
+
+print("\n" + "-"*30)
+print("Corpus Merge Complete!")
+print("-"*30)
+print(f"Total songs merged: {total_songs}")
+print(f"Total characters:   {total_chars:,}")
+print(f"Estimated words:    {total_chars // 5:,} (assuming ~5 chars per word)")
+print(f"Output saved to:    {output_file}")