IDS / app.py
MulukenS's picture
Rename Untitled2.py to app.py
523a89f verified
Raw History Blame Contribute Delete
1.94 kB
#!/usr/bin/env python
# coding: utf-8
import polars as pl
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn.model_selection import train_test_split
dict_data={}
df = pl.read_csv("merged_data.csv",encoding="latin-1")
print(df.shape)
print(df.dtypes)
print(df.columns)
label_count = df[' Label'].value_counts()
print(label_count)
import pandas as pd
from sklearn.preprocessing import StandardScaler
import csv
import numpy as np
#data = pd.read_csv('merged_data.csv', low_memory=False)
#3119345
chunk_size = 100000
# Numerical features to scale
# numerical_features = ['Flow ID', 'Source IP', 'Destination IP', 'Label']
scaler = StandardScaler()
print(type(df))
y = df.select(' Label')
X = df.drop(' Label')
print(f"Processed chunk with {len(X)} rows")
# Assuming y_train contains your target labels
num_classes = len(np.unique(y)) # Calculate the number of unique classes
# Split data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
print(f"X_train shape: {X_train.shape}") # Should be (num_samples, num_features)
print(f"y_train shape: {y_train.shape}")
print(X_train.shape)
# Create MLP model
model = Sequential()
model.add(Dense(32, activation='relu', input_dim=X_train.shape[1])) # Input layer with 84 neurons
model.add(Dense(32, activation='relu')) # Hidden layer with 64 neurons
model.add(Dense(num_classes, activation='softmax')) # Output layer with num_classes neurons (adjust num_classes)
# Compile the model
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# Train the model
model.fit(X_train, y_train, epochs=1000, batch_size=32, validation_data=(X_test, y_test))
# Evaluate the model
loss, accuracy = model.evaluate(X_test, y_test)
print('Test accuracy:', accuracy)
filename = 'finalized_model.sav'
pickle.dump(model, open(filename, 'wb'))