dataset.py

import numpy as np
import os

def generate_npy_files(num_files=5, samples_per_file=1_000_000):
    """Generates .npy files containing 1D arrays of random floats."""
    
    print(f"Starting generation of {num_files} files...")
    
    for i in range(1, num_files + 1):
        filename = f"{i}.npy"
        
        # Generate 1M samples of random data (float64 by default)
        data = np.random.randint(low=-128, high=128, size=samples_per_file, dtype=np.int8)
        if not os.path.exists("data"):
            os.mkdir("data")
            # Save to disk with specified folder
        np.save(f"data/{filename}", data)
    
        file_size_mb = os.path.getsize(f"data/{filename}") / (1024 * 1024)
        print(f"Created {filename} | Shape: {data.shape} | Size: {file_size_mb:.2f} MB")

    print("\nSuccess: All files generated in the current directory.")

if __name__ == "__main__":
    generate_npy_files()