Upload 10 files

app.py

@@ -0,0 +1,11 @@
+import streamlit as st
+import eda
+import prediction
+
+page = st.sidebar.selectbox('Select Page: ', ('EDA', 'Prediction'))
+
+if page == 'EDA':
+    eda.app()
+else:
+    prediction.app()
+    

column_names.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff2e2f25fd401dc4c2cdfd6a49eb8fe51a4d515fba9fe064e6b194318921fb29
+size 1406

eda.py

@@ -0,0 +1,78 @@
+import streamlit as st
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import plotly.express as px
+from PIL import Image
+
+def app():
+    # title
+    st.title('Malicious or Benign Website Prediction')
+
+    # subheader
+    st.subheader('EDA for Malicious or Benign Website Prediction')
+
+    # add image
+    image = Image.open('web.jpg')
+    st.image(image, caption = 'Malicious or Benign Website')
+
+    # Markdown
+    st.markdown('----')
+
+    # Masukkan pandas dataframe
+
+    # show dataframe
+    df = pd.read_csv('dataset.csv')
+    st.dataframe(df)
+    
+    object_columns = df.select_dtypes(include=['object']).columns
+    numerical_columns = df.select_dtypes(exclude=['object']).columns
+
+    st.write('#### Plot Categorical Columns using Pie Chart')
+    option_cat = st.selectbox('Select Column:', ('CHARSET', 'SERVER', 'WHOIS_COUNTRY', 'WHOIS_STATEPRO'))
+    fig = plt.figure(figsize=(15,5))
+    plt.pie(df[option_cat].value_counts(), labels=df[option_cat].value_counts().index, autopct='%1.1f%%', startangle=180)
+    st.pyplot(fig)
+    
+    # # plot historical date data with lineplot for WHOIS_REGDATE and WHOIS_UPDATED_DATE separated by type column
+    # date_columns = ['WHOIS_REGDATE', 'WHOIS_UPDATED_DATE']
+    # st.write('#### Plot Historical Date Data with Lineplot')
+    # option_date = st.selectbox('Select Column:', ('WHOIS_REGDATE', 'WHOIS_UPDATED_DATE'))
+    # fig = plt.figure(figsize=(15,5))
+    # sns.lineplot(x=option_date, y='Type', data=df)
+    # st.pyplot(fig)
+        
+    st.write('#### Plot Numerical Columns')
+    option = st.selectbox('Select Column:', ('URL_LENGTH', 'NUMBER_SPECIAL_CHARACTERS', 'CONTENT_LENGTH', 'APP_PACKETS', 'DNS_QUERY_TIMES'))
+    fig = plt.figure(figsize=(15,5))
+    sns.histplot(df[option], bins=30, kde=True)
+    st.pyplot(fig)
+    
+    option_pay = st.selectbox('Select Column:', ('TCP_CONVERSATION_EXCHANGE', 'DIST_REMOTE_TCP_PORT', 'REMOTE_IPS', 'APP_BYTES'))
+    fig = plt.figure(figsize=(15,5))
+    sns.histplot(df[option_pay], bins=30, kde=True)
+    st.pyplot(fig)
+    
+    option_bill_amt = st.selectbox('Select Column:', ('SOURCE_APP_PACKETS', 'REMOTE_APP_PACKETS', 'SOURCE_APP_BYTES', 'REMOTE_APP_BYTES'))
+    fig = plt.figure(figsize=(15,5))
+    sns.histplot(df[option_bill_amt], bins=30, kde=True)
+    st.pyplot(fig)
+
+    # plot type column count with boxplot color with type column
+    st.write('#### Plot Type Column Count with Boxplot')
+    fig = plt.figure(figsize=(15,5))
+    sns.boxplot(x='Type', y='URL_LENGTH', data=df, hue='Type')
+    st.pyplot(fig)
+
+    # Sort DataFrame by 'Type'
+    df = df.sort_values('Type')
+
+    # Membuat plotly plot
+    st.write('#### Plotly Plot - URL_LENGTH vs CONTENT_LENGTH')
+    fig  = plt.figure(figsize=(15,5))
+    sns.scatterplot(x='URL_LENGTH', y='CONTENT_LENGTH', data=df, hue='Type')
+    st.pyplot(fig)
+    
+
+if __name__ == '__main__':
+    app()

encoder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4781c04a1bac238f9ae766bd4588248a6594d41754ce98e101281915c96de98
+size 659

model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b2497b428c40346d3449e91bd4521b187a16122c5c9d8835e5dae01f785f6ca
+size 139783

prediction.py

@@ -0,0 +1,203 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import pickle
+
+# load all files
+
+with open("model.pkl", "rb") as f: # load the model
+    model = pickle.load(f)
+    
+with open("scaler.pkl", "rb") as f:
+    scaler = pickle.load(f)
+
+with open("encoder.pkl", "rb") as f: # load the scaler
+    encoder = pickle.load(f)
+
+with open('column_names.pkl', 'rb') as f:
+    column_names = pickle.load(f)
+
+    # 'URL_LENGTH': URL_LENGTH,
+    # 'NUMBER_SPECIAL_CHARACTERS': NUMBER_SPECIAL_CHARACTERS,
+    # 'CONTENT_LENGTH': CONTENT_LENGTH,
+    # 'WHOIS_REGDATE': WHOIS_REGDATE,
+    # 'WHOIS_UPDATED_DATE': WHOIS_UPDATED_DATE,
+    # 'TCP_CONVERSATION_EXCHANGE': TCP_CONVERSATION_EXCHANGE,
+    # 'DIST_REMOTE_TCP_PORT': DIST_REMOTE_TCP_PORT,
+    # 'REMOTE_IPS': REMOTE_IPS,
+    # 'APP_BYTES': APP_BYTES,
+    # 'SOURCE_APP_PACKETS': SOURCE_APP_PACKETS,
+    # 'REMOTE_APP_PACKETS': REMOTE_APP_PACKETS,
+    # 'SOURCE_APP_BYTES': SOURCE_APP_BYTES,
+    # 'REMOTE_APP_BYTES': REMOTE_APP_BYTES,
+    # 'APP_PACKETS': APP_PACKETS,
+    # 'DNS_QUERY_TIMES': DNS_QUERY_TIMES
+    
+    #INT BLOCK
+    # URL_LENGTH = np.random.randint(16.000000, 159.000000, size=10)
+    # NUMBER_SPECIAL_CHARACTERS = np.random.randint(5.000000, 28.000000, size=10)
+    # CONTENT_LENGTH = np.random.randint(0, 9806.000000, size=10)
+
+    # d1 = datetime.strptime('1990-07-26', '%Y-%m-%d')
+    # d2 = datetime.strptime('2017-04-14', '%Y-%m-%d')
+    # WHOIS_UPDATED_DATE = random_date(d1, d2)
+    # WHOIS_REGDATE = random_date(d1, d2)
+
+    # TCP_CONVERSATION_EXCHANGE = np.random.randint(0, 84.000000, size=10)
+    # DIST_REMOTE_TCP_PORT = np.random.randint(0, 20.000000, size=10)
+    # REMOTE_IPS = np.random.randint(0, 16, size=10)
+    # APP_BYTES = np.random.randint(0, 9302, size=10)
+
+    # SOURCE_APP_PACKETS = np.random.randint(0, 103, size=10)
+    # REMOTE_APP_PACKETS = np.random.randint(0, 99, size=10)
+    # SOURCE_APP_BYTES = np.random.randint(0, 38681, size=10)
+    # REMOTE_APP_BYTES = np.random.randint(0, 10693, size=10)
+    # APP_PACKETS = np.random.randint(0, 103, size=10)
+    # DNS_QUERY_TIMES = np.random.randint(0, 14, size=10)
+
+
+def app():
+    
+    with st.form('from_website_data'):
+        
+        charset_choice = {1: "ISO-8859-1", 2: "UTF-8", 3: "utf-8", 4: "us-ascii", 5: "iso-8859-1", 6: "unknown", 7: "windows-1252", 8: "windows-1251"}
+        
+        CHARSET = st.selectbox("Select Charset", options=list(charset_choice.values()))
+        
+        server_choice = {1: "other", 2: "Apache", 3: "nginx", 4: "cloudflare-nginx", 5: "nginx/1.12.0", 6: "Apache/2.2.22 (Debian)", 7: "nginx/1.8.0", 8: "nginx/1.10.1", 9: "Microsoft-HTTPAPI/2.0", 10: "Microsoft-IIS/6.0", 11: "Apache/2.4.23 (Unix) OpenSSL/1.0.1e-fips mod_bwlimited/1.4"}
+
+        SERVER = st.selectbox("Select Server", options=list(server_choice.values()))
+        
+        whois_country_choice = {1: "US", 2: "other", 3: "unknown", 4: "PA", 5: "GB", 6: "CN", 7: "KR", 8: "CA", 9: "UK", 10: "CZ", 11: "FR"}
+        
+        WHOIS_COUNTRY = st.selectbox("Select Country", options=list(whois_country_choice.values()))
+        
+        WHOIS_STATEPRO_choice = {1: "other", 2: "CA", 3: "unknown", 4: "California", 5: "PA", 6: "Washington", 7: "Arizona", 8: "ON", 9: "WA", 10: "FL"}
+
+        WHOIS_STATEPRO = st.selectbox("Select States", options=list(WHOIS_STATEPRO_choice.values()))
+        
+        URL_LENGTH = st.number_input('URL_LENGTH', min_value=16, max_value=159, value=50)
+        NUMBER_SPECIAL_CHARACTERS = st.number_input('NUMBER_SPECIAL_CHARACTERS', min_value=5, max_value=28, value=5)
+        CONTENT_LENGTH = st.number_input('CONTENT_LENGTH', min_value=0, max_value=9806, value=50)
+        TCP_CONVERSATION_EXCHANGE = st.number_input('TCP_CONVERSATION_EXCHANGE', min_value=0, max_value=84, value=50)
+        DIST_REMOTE_TCP_PORT = st.number_input('DIST_REMOTE_TCP_PORT', min_value=0, max_value=20, value=0)
+        REMOTE_IPS = st.number_input('REMOTE_IPS', min_value=0, max_value=16, value=0)
+        APP_BYTES = st.number_input('APP_BYTES', min_value=0, max_value=9302, value=50)
+        SOURCE_APP_PACKETS = st.number_input('SOURCE_APP_PACKETS', min_value=0, max_value=103, value=50)
+        REMOTE_APP_PACKETS = st.number_input('REMOTE_APP_PACKETS', min_value=0, max_value=99, value=50)
+        SOURCE_APP_BYTES = st.number_input('SOURCE_APP_BYTES', min_value=0, max_value=38681, value=50)
+        REMOTE_APP_BYTES = st.number_input('REMOTE_APP_BYTES', min_value=0, max_value=10693, value=50)
+        APP_PACKETS = st.number_input('APP_PACKETS', min_value=0, max_value=103, value=50)
+        DNS_QUERY_TIMES = st.number_input('DNS_QUERY_TIMES', min_value=0, max_value=14, value=5)
+        
+        # input for date
+        # 2006-03-22
+        WHOIS_REGDATE = st.date_input('WHOIS_REGDATE', format="YYYY-MM-DD")
+        WHOIS_UPDATED_DATE = st.date_input('WHOIS_UPDATED_DATE', format="YYYY-MM-DD")
+        
+        #submit buttion
+        submitted = st.form_submit_button('Predict')
+    
+    data_inf = {
+        'URL_LENGTH': URL_LENGTH,
+        'NUMBER_SPECIAL_CHARACTERS': NUMBER_SPECIAL_CHARACTERS,
+        'CONTENT_LENGTH': CONTENT_LENGTH,
+        'WHOIS_REGDATE': WHOIS_REGDATE,
+        'WHOIS_UPDATED_DATE': WHOIS_UPDATED_DATE,
+        'TCP_CONVERSATION_EXCHANGE': TCP_CONVERSATION_EXCHANGE,
+        'DIST_REMOTE_TCP_PORT': DIST_REMOTE_TCP_PORT,
+        'REMOTE_IPS': REMOTE_IPS,
+        'APP_BYTES': APP_BYTES,
+        'SOURCE_APP_PACKETS': SOURCE_APP_PACKETS,
+        'REMOTE_APP_PACKETS': REMOTE_APP_PACKETS,
+        'SOURCE_APP_BYTES': SOURCE_APP_BYTES,
+        'REMOTE_APP_BYTES': REMOTE_APP_BYTES,
+        'APP_PACKETS': APP_PACKETS,
+        'DNS_QUERY_TIMES': DNS_QUERY_TIMES,
+        'CHARSET': CHARSET,
+        'SERVER': SERVER,
+        'WHOIS_COUNTRY': WHOIS_COUNTRY,
+        'WHOIS_STATEPRO': WHOIS_STATEPRO
+    }
+
+    
+    
+    data_inf = pd.DataFrame([data_inf])
+    # st.dataframe(data_inf)
+
+    def encode_and_create_dataframe_train(df, column):
+        # Fit a separate OneHotEncoder for the column
+        transformed_data = encoder.fit_transform(df[[column]])
+
+        # Get feature names for the column
+        feature_names = encoder.get_feature_names_out(input_features=[column])
+
+        # Create a DataFrame for the column
+        transformed_df = pd.DataFrame(transformed_data.toarray(), 
+                                    index=df.index, 
+                                    columns=feature_names)
+    
+        return transformed_df, encoder
+
+    # logic ketika user submit
+    if submitted:
+        #split between numerical and categorical columns
+        data_inf_num = data_inf[['URL_LENGTH', 'NUMBER_SPECIAL_CHARACTERS', 'CONTENT_LENGTH', 
+                                 'WHOIS_REGDATE', 'WHOIS_UPDATED_DATE', 'TCP_CONVERSATION_EXCHANGE', 
+                                 'DIST_REMOTE_TCP_PORT', 'REMOTE_IPS', 'APP_BYTES', 'SOURCE_APP_PACKETS', 
+                                 'REMOTE_APP_PACKETS', 'SOURCE_APP_BYTES', 'REMOTE_APP_BYTES', 'APP_PACKETS', 
+                                 'DNS_QUERY_TIMES']]
+        data_inf_cat = data_inf[['CHARSET', 'SERVER', 'WHOIS_COUNTRY', 'WHOIS_STATEPRO']]
+                
+        # Convert to datetime format
+        data_inf_num['WHOIS_REGDATE'] = pd.to_datetime(data_inf_num['WHOIS_REGDATE'])
+        data_inf_num['WHOIS_UPDATED_DATE'] = pd.to_datetime(data_inf_num['WHOIS_UPDATED_DATE'])
+
+        # Extract year as integer
+        data_inf_num['WHOIS_REGDATE'] = data_inf_num['WHOIS_REGDATE'].dt.year
+        data_inf_num['WHOIS_UPDATED_DATE'] = data_inf_num['WHOIS_UPDATED_DATE'].dt.year
+        
+        # scaling and encoding
+        data_inf_num_scaled = scaler.transform(data_inf_num)
+        
+        # transform to dataframe
+        data_inf_num_scaled = pd.DataFrame(data_inf_num_scaled, columns=data_inf_num.columns)
+
+        capped_CHARSET, ohe_CHARSET = encode_and_create_dataframe_train(data_inf_cat, 'CHARSET')
+        capped_SERVER, ohe_SERVER = encode_and_create_dataframe_train(data_inf_cat, 'SERVER')
+        capped_WHOIS_COUNTRY, ohe_WHOIS_COUNTRY = encode_and_create_dataframe_train(data_inf_cat, 'WHOIS_COUNTRY')
+        capped_WHOIS_STATEPRO, ohe_WHOIS_STATEPRO = encode_and_create_dataframe_train(data_inf_cat, 'WHOIS_STATEPRO')
+        
+        # concat all data 
+        data_inf_final = pd.concat([data_inf_num_scaled, capped_CHARSET, capped_SERVER, capped_WHOIS_COUNTRY, capped_WHOIS_STATEPRO], axis=1)
+        
+        if len(column_names) != len(set(column_names)):
+            st.write("column_names contains duplicates")
+            
+        if len(data_inf_final.columns) != len(set(data_inf_final.columns)):
+            st.write("data_inf_final has duplicate column names")
+                
+        # reindex to match the training columns
+        data_inf_final = data_inf_final.reindex(columns=column_names)
+
+        # Check Missing Values
+        data_inf_final.isnull().sum()
+
+        # fill null value with zeros
+        data_inf_final = data_inf_final.fillna(0)
+        
+        #predict using linear reg model
+
+        y_pred_inf = model.predict(data_inf_final)
+        
+        st.dataframe(data_inf)
+        
+        if y_pred_inf == 0:
+            # write with green color
+            st.markdown("<h1 style='text-align: center; color: green;'>Predicted Class: Benign</h1>", unsafe_allow_html=True)        
+        else:
+            st.markdown("<h1 style='text-align: center; color: red;'>Predicted Class: Malicious</h1>", unsafe_allow_html=True)                
+        
+        
+if __name__ == '__main__':
+    app()

requirements.txt

@@ -0,0 +1,9 @@
+streamlit
+pandas
+seaborn
+matplotlib
+numpy
+plotly
+Pillow
+xgboost
+scikit-learn==1.2.2

scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861784cd69b250f457bfbf7b7ef84c51d1afa037fc952e546886c2e703464b0f
+size 1003