import torch import scipy.io as sio from argparse import Namespace import numpy as np import streamlit as st import matplotlib.pyplot as plt from PIL import Image, ImageEnhance import multispectral as mspec def GetRGB( snp, scale=2 ): try: snp = np.transpose( snp, [ 1, 2, 0 ] ).astype( float ) except: pass # not a multi-channel image snp = np.round( 255*( snp - snp.min() ) / ( snp.max() - snp.min() ) ).astype( np.uint8 ) im = Image.fromarray( snp ) im = im.resize( ( im.size[0]//scale, im.size[1]//scale ), resample=Image.Resampling.NEAREST ) if np.ndim( snp ) == 3: enhancer = ImageEnhance.Brightness( im ) im = enhancer.enhance( 5 ) return im def plot_zoomed_region( image, x1, y1, x2, y2, title='Zoomed in region' ): # Crop the image to the zoom box coordinates cropped_image = image.crop( ( x1, y1, x2, y2 ) ) # Convert cropped image to numpy array for plotting img_array = np.array( cropped_image ) # Plot the cropped image fig, ax = plt.subplots() ax.imshow( img_array ) ax.set_title( title ) ax.axis( 'off' ) return fig def dashboard_header(): cultural_reference = 'https://www.youtube.com/watch?v=ACmydtFDTGs' st.markdown( '# `hotdog-nothotdog v2.0`' ) st.markdown( '## - Jian Yang (possibly)' ) st.markdown( 'Cultural reference: [this clip](%s) from _Silicon Valley_. '%cultural_reference ) return def dashboard_intro(): st.markdown( ''' # TL;DR An interactive, trained-from-scratch semantic segmenter for 6-channel multi-spectral imagery data, trained solely on the RIT-18 dataset. This is a work in progress. Go to the app. # Dataset description RIT-18 consists of 19 classes within the scan range of a multi-spectral drone camera over a New York state park. It is highly unbalanced. # Training methodology This model was trained repeatedly on randomized $256 \\times 256$ pixel slices of the same swathe region of the drone, but with the pixel labels at each iteration increasingly diverse. For example, for the very first iteration, the dataset was relabeled from 19 classes to only "Vegetation" and "Other" (_i.e._, a binary segmenter). The resulting model became the starting point for the training of the next iteration, in which the pixels with the "Other" label were further re-labeled "Water" and "Other", for a total of 3 classes. Similarly, in later iterations, "Vegetation" will be broken down to "Trees" and "Grass" and so on. In this manner, the goal is to obtain a final model trained on all 19 classes of the RIT-18 dataset. # Goal To see if it's possible to incrementally train the model in this manner. If so, how influential are the near infrared (NIR) bands in the segmentation outcome? - **Dataset**: [RIT-18](https://arxiv.org/abs/1703.06452) - **Model architecture**: U-net # The app ''', unsafe_allow_html=True ) return def get_model( num_classes, model_state ): mynn = mspec.MultiSpectNet( img_size=[ 256, 256 ], num_channels=7, num_classes=num_classes, learning_rate=1.e-3 ) mynn.model.load_state_dict( torch.load( model_state ) ) return mynn