#!/usr/bin/env python # coding: utf-8 import polars as pl from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from sklearn.model_selection import train_test_split dict_data={} df = pl.read_csv("merged_data.csv",encoding="latin-1") print(df.shape) print(df.dtypes) print(df.columns) label_count = df[' Label'].value_counts() print(label_count) import pandas as pd from sklearn.preprocessing import StandardScaler import csv import numpy as np #data = pd.read_csv('merged_data.csv', low_memory=False) #3119345 chunk_size = 100000 # Numerical features to scale # numerical_features = ['Flow ID', 'Source IP', 'Destination IP', 'Label'] scaler = StandardScaler() print(type(df)) y = df.select(' Label') X = df.drop(' Label') print(f"Processed chunk with {len(X)} rows") # Assuming y_train contains your target labels num_classes = len(np.unique(y)) # Calculate the number of unique classes # Split data into training and testing sets X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) print(f"X_train shape: {X_train.shape}") # Should be (num_samples, num_features) print(f"y_train shape: {y_train.shape}") print(X_train.shape) # Create MLP model model = Sequential() model.add(Dense(32, activation='relu', input_dim=X_train.shape[1])) # Input layer with 84 neurons model.add(Dense(32, activation='relu')) # Hidden layer with 64 neurons model.add(Dense(num_classes, activation='softmax')) # Output layer with num_classes neurons (adjust num_classes) # Compile the model model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) # Train the model model.fit(X_train, y_train, epochs=1000, batch_size=32, validation_data=(X_test, y_test)) # Evaluate the model loss, accuracy = model.evaluate(X_test, y_test) print('Test accuracy:', accuracy) filename = 'finalized_model.sav' pickle.dump(model, open(filename, 'wb'))