Spaces:
Sleeping
Sleeping
File size: 2,747 Bytes
210d88d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
import json
def load_data(filename):
data = []
try:
with open(filename, 'r') as f:
content = f.read().strip()
if not content:
print(f"Warning: {filename} is empty.")
return []
# Try parsing as standard JSON list
try:
data = json.loads(content)
if isinstance(data, list):
print(f"Successfully loaded {filename} as JSON list.")
return data
else:
print(f"Warning: {filename} is valid JSON but not a list (got {type(data)}). Wrapping in list.")
return [data]
except json.JSONDecodeError:
# Try parsing as JSON Lines (one JSON object per line)
try:
lines = content.split('\n')
data = [json.loads(line) for line in lines if line.strip()]
print(f"Successfully loaded {filename} as JSON Lines ({len(data)} entries).")
return data
except json.JSONDecodeError as e:
print(f"Error: Failed to parse {filename} as JSON or JSON Lines. Error: {e}")
return []
except FileNotFoundError:
print(f"Error: {filename} not found.")
return []
def merge_datasets():
master_data = load_data('master_dataset.json')
new_data = load_data('data.json')
print(f"Entries from master_dataset.json: {len(master_data)}")
print(f"Entries from data.json: {len(new_data)}")
if not master_data and not new_data:
print("No data loaded. Exiting.")
return
# Combine lists
combined_data = master_data + new_data
# Remove duplicates
unique_data = []
seen = set()
for entry in combined_data:
# Sort keys to ensure consistent string representation
# Use simple string checks to avoid complex object issues
try:
entry_str = json.dumps(entry, sort_keys=True)
if entry_str not in seen:
seen.add(entry_str)
unique_data.append(entry)
except TypeError:
print(f"Warning: Skipping non-serializable entry: {entry}")
print(f"Total entries after merge (removing duplicates): {len(unique_data)}")
# Write to new file as JSON List (standard format for most tools)
output_filename = 'merged_dataset.json'
with open(output_filename, 'w') as f:
json.dump(unique_data, f, indent=2)
print(f"Merged data saved to {output_filename}")
if __name__ == "__main__":
merge_datasets()
|