streamlit_viz.py

import joblib
import time

import plotly.graph_objects as go
import streamlit as st
import pandas as pd
import numpy as np

FEATS = [
  'srcip',
  'sport',
  'dstip',
  'dsport',
  'proto',
  #'state',  I dropped this one when I trained the model
  'dur',
  'sbytes',
  'dbytes',
  'sttl',
  'dttl',
  'sloss',
  'dloss',
  'service',
  'Sload',
  'Dload',
  'Spkts',
  'Dpkts',
  'swin',
  'dwin',
  'stcpb',
  'dtcpb',
  'smeansz',
  'dmeansz',
  'trans_depth',
  'res_bdy_len',
  'Sjit',
  'Djit',
  'Stime',
  'Ltime',
  'Sintpkt',
  'Dintpkt',
  'tcprtt',
  'synack',
  'ackdat',
  'is_sm_ips_ports',
  'ct_state_ttl',
  'ct_flw_http_mthd',
  'is_ftp_login',
  'ct_ftp_cmd',
  'ct_srv_src',
  'ct_srv_dst',
  'ct_dst_ltm',
  'ct_src_ltm',
  'ct_src_dport_ltm',
  'ct_dst_sport_ltm',
  'ct_dst_src_ltm',
]

# Generated from
# mokole.com/palette.html
COLORS = [
  '#000000',
  '#808080',
  '#2f4f4f',
  '#556b2f',
  '#8b4513',
  '#228b22',
  '#800000',
  '#808000',
  '#3cb371',
  '#663399',
  '#b8860b',
  '#008b8b',
  '#4682b4',
  '#d2691e',
  '#9acd32',
  '#cd5c5c',
  '#00008b',
  '#32cd32',
  '#8fbc8f',
  '#b03060',
  '#d2b48c',
  '#ff0000',
  '#ffa500',
  '#ffd700',
  '#ffff00',
  '#0000cd',
  '#00ff00',
  '#8a2be2',
  '#00ff7f',
  '#4169e1',
  '#dc143c',
  '#00ffff',
  '#00bfff',
  '#f4a460',
  '#adff2f',
  '#ff6347',
  '#da70d6',
  '#d8bfd8',
  '#ff00ff',
  '#f0e68c',
  '#6495ed',
  '#dda0dd',
  '#b0e0e6',
  '#98fb98',
  '#7fffd4',
  '#ff69b4',

]

def build_parents(tree, visit_order, node_id2plot_id):
  parents = [None]
  parent_plot_ids = [None]
  directions = [None]
  for i in visit_order[1:]:
    parent = tree[tree['right']==i].index
    if parent.empty:
      p = tree[tree['left']==i].index[0]
      parent_plot_ids.append(str(node_id2plot_id[p]))
      parents.append(p)
      directions.append('l')
    else:
      parent_plot_ids.append(str(node_id2plot_id[parent[0]]))
      parents.append(parent[0])
      directions.append('r')
  return parents, parent_plot_ids, directions


def build_labels_colors(tree, visit_order, parents, parent_plot_ids, directions):
  labels = ['Histogram Gradient-Boosted Decision Tree']
  colors = ['white']
  for i, parent, parent_plot_id, direction in zip(
    visit_order,
    parents,
    parent_plot_ids,
    directions
  ):
    # skip the first one (the root)
    if i == 0:
      continue
    node = tree.loc[i]
    feat = FEATS[int(tree.loc[int(parent), 'feature_idx'])]

    thresh = tree.loc[int(parent), 'num_threshold']
    if direction == 'l':
      labels.append(f"[{parent_plot_id}.L] {feat} <= {thresh}")
    else:
      labels.append(f"[{parent_plot_id}.R] {feat} > {thresh}")

    # colors
    offset = FEATS.index(feat)
    colors.append(COLORS[offset])
  return labels, colors


def build_plot(tree):
  #https://stackoverflow.com/questions/64393535/python-plotly-treemap-ids-format-and-how-to-display-multiple-duplicated-labels-i
  # if you use `ids`, then `parents` has to be in terms of `ids`
  visit_order = breadth_first_traverse(tree)
  node_id2plot_id = {node:i for i, node in enumerate(visit_order)}
  parents, parent_plot_ids, directions = build_parents(tree, visit_order, node_id2plot_id)
  labels, colors = build_labels_colors(tree, visit_order, parents, parent_plot_ids, directions)
  # this should just be ['0', '1', '2', . . .]
  plot_ids = [str(node_id2plot_id[x]) for x in visit_order]

  return go.Treemap(
    values=tree['count'].to_numpy(),
    labels=labels,
    ids=plot_ids,
    parents=parent_plot_ids,
    marker_colors=colors,
  )


def breadth_first_traverse(tree):
  """
  https://www.101computing.net/breadth-first-traversal-of-a-binary-tree/
  Iterative version makes more sense since I have the whole tree in a table
  instead of just nodes and pointers
  """
  q = [0]
  visited_nodes = []
  while len(q) != 0:
    cur = q.pop(0)
    visited_nodes.append(cur)

    if tree.loc[cur, 'left'] != 0:
      q.append(tree.loc[cur, 'left'])

    if tree.loc[cur, 'right'] != 0:
      q.append(tree.loc[cur, 'right'])

  return visited_nodes


def main():
  # load the data
  hgb = joblib.load('hgb_classifier.joblib')
  trees = [pd.DataFrame(x[0].nodes) for x in hgb._predictors]
  # make the plots
  graph_objs = [build_plot(tree) for tree in trees]
  figures = [go.Figure(graph_obj) for graph_obj in graph_objs]
  # each frame has to have a name
  # https://community.plotly.com/t/animation-with-slider-not-moving-when-pressing-play/34763/2
  frames = [go.Frame(data=graph_obj, name=str(i)) for i, graph_obj in enumerate(graph_objs)]
  # show them with streamlit

  #st.markdown('# Thankfully, Visualizing Decision Trees is Hard')
  st.markdown('# Thankfully, visualizing decision trees is hard')
  st.markdown('## Setting the scene')
  st.markdown("""
I make a lot of dashboards, which means I make a lot of the same plots over and over.
Desperate for some creative outlet, I wanted to make a new visualization—
something I'd never seen before.
Inspired by interactive visualizations like 
[Tensorflow Playground](https://playground.tensorflow.org)
and
[GAN Lab](https://poloclub.github.io/ganlab),
I decided to wanted to watch some kind of gradient-boosted tree as it learned.
""")

  st.markdown('## Some kind of gradient-boosted tree')
  st.markdown("""
I trained an ensemble of
[Histogram-based Gradient Boosting Decision Trees](https://scikit-learn.org/stable/modules/ensemble.html#histogram-based-gradient-boosting)
 on some
[data](https://research.unsw.edu.au/projects/unsw-nb15-dataset).
That algorithm looks at its mistakes and tries to avoid those mistakes the next time around.

To do that, it starts off with a decision tree.
From there, it looks at the points that tree got wrong and makes another decision tree that tries
to get those points right.
Then it looks at that second tree's mistakes and makes a third tree that tries to fix those mistakes.
And so on.

My model ends up with 10 trees.
""")

  st.markdown('## Behold')

  st.markdown("""
I've plotted the progression of those 10 trees as an animated series of interactive Plotly tree maps.
The nodes are color-coded by which feature the decision tree used to make that split.

I've also labeled each node with the feature name and the decision boundary.
If you click on a node, Plotly will show the path to that node in a banner at the top of the plot so you can see how a point ends up in the node you clicked.

The numbers and letters in brackets like `[3.L]` refer to the parent node's position in a breadth-first traversal of the tree and whether the current node is a left or right child of that parent.
Plotly unfortunately plots everything flipped for some reason, so all the `R` nodes are on the left and vice versa.

I recommend expanding the plot by clicking the arrows in the top right corner since Streamlit makes the plot really small.
It takes a second to get going after you hit `Play`.

""")



  # Build the slider steps
  slider_steps = []
  for i in range(len(trees)):
    slider_steps.append({
      'args': [
        [i],
        {
          'frame': {'duration': 300, 'redraw': True},
          'mode': 'immediate',
          'transition': {'duration': 300}
        }
      ],
      'label': i,
      'method': 'animate',
    })

  sliders_dict = {
    'active': 0,
    'currentvalue': {
      'font': {'size': 20},
      'prefix': 'Tree ',
      'visible': True
    },
    'transition': {'duration': 300},
    'steps': slider_steps
  }

  # Maybe just show a Plotly animated chart
  # https://plotly.com/python/animations/#using-a-slider-and-buttons
  # They don't really document the animation stuff on their website
  # but it's in here
  # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plot-schema.json
  # I guess it's only in the JS docs and hasn't made it to the Python docs yet
  # https://plotly.com/javascript/animations/
  # trying to find stuff here instead
  # https://plotly.com/python-api-reference/generated/plotly.graph_objects.layout.updatemenu.html?highlight=updatemenu

  # this one finally set the speed
  # no mention of how they figured this out but thank goodness I found it
  # https://towardsdatascience.com/basic-animation-with-matplotlib-and-plotly-5eef4ad6c5aa

  # this also has custom animation speeds in it
  # https://plotly.com/python/custom-buttons/#reference
  ani_fig = go.Figure(
    data=graph_objs[0],
    frames=frames,
    layout=go.Layout(
      updatemenus=[{
        'type':'buttons',
        # https://plotly.com/python/reference/layout/updatemenus/

        # Always show the background color on buttons
        # streamlit breaks the background color of the active button in darkmode
        'showactive': False,
        # background color of the buttons
        'bgcolor': '#fff',
        # font in the buttons
        'font': {'color': '#000'},
        # border color of the buttons
        'bordercolor': '#000',

        # Play and Pause buttons
        # trying to copy this exactly
        # https://plotly.com/python/animations/#adding-control-buttons-to-animations
        'buttons':[{
          'label':'Play',
          'method': 'animate',
          'args':[None, {
            'fromcurrent': True,
            'frame': {'duration':5000},
            'transition': {'duration': 2500},
            }],
          },
          {
          'label': 'Pause',
          'method': 'animate',
          'args':[[None], {
            'frame': {'duration': 0},
            'transition': {'duration': 0},
            'mode': 'immediate'
            }]
          }
       ]
      }],
     # add the slider to the layout
     sliders=[sliders_dict]
    )
  )
  st.plotly_chart(ani_fig)

  st.markdown("""
This actually turned out to be a lot harder than I thought it would be.
Plotly doesn't have many examples of how to create animations like this in Python.
[The only example I could find](https://plotly.com/python/animations/#using-a-slider-and-buttons)
was derided as an
["old example [. . .] that is not the best one to learn how to define an animation with slider."](https://community.plotly.com/t/slider-not-updating-during-animation/37261)

That helpful poster didn't point out any other examples, so that one is still pretty much all I have to go on.

Later on,
[a different answer by the same poster](https://community.plotly.com/t/animation-with-slider-not-moving-when-pressing-play/34763)
got me out of a jam.
As far as I can tell, this poster `empet` is the only person in the world who understands Plotly's animations in Python.
""")

  st.markdown('## Check out the data!')
  st.markdown("""
This plot is similar to the plot above, but the slider here coordinates with a table of the data I extracted to plot each tree.
""")

  # This works the way I want
  # but the plot is tiny
  # also it recalcualtes all of the plots
  # every time the slider value changes
  #
  # I tried to cache the plots but build_plot() takes
  # a DataFrame which is mutable and therefore unhashable I guess
  # so it won't let me cache that function
  # I could pack the dataframe bytes to smuggle them past that check
  # but whatever
  idx = st.slider(
    label='Which tree do you want to see?',
    min_value=0,
    max_value=len(figures)-1,
    value=0,
    step=1
  )
  st.markdown(f'### Tree {idx}')
  st.plotly_chart(figures[idx])
  st.dataframe(trees[idx])
  st.markdown("""
This section is mostly just to warn you against making the same foolhardy decision to marry the innermost guts of SciKit-Learn to the sparsely documented world of Python Plotly animations.

I'm glad it was challenging, though.
I did go into this hoping for something more interesting than a donut plot.
Maybe I'll think on the `value` and `gain` fields a bit and come up with a version 2.
""")

if __name__=='__main__':
  main()