app.py

import json
from typing import List, Dict, Tuple, Union, Any

import streamlit as st
from annotated_text import annotated_text
sidebar = st.sidebar
def bs_unigram_match_annotated_text(belief_state_example) -> List[Union[str, Tuple]]:
    gold_set = set(belief_state_example['gold'].split(' '))
    input_set = set(" ".join(belief_state_example['input']).split(' '))
    generated = belief_state_example['generated']
    result = []
    for word in generated.split(' '):
        if word in gold_set:
            result.append((word, 'gold', '#dfd'))  # gold overlap => label green
        elif word in input_set:
            result.append((word, 'in', '#eea'))  # input overlap => label yellow
        else:
            result.append(word + ' ')  # no overlap => no label (replace space)
    return result

# load in data
pptod_examples: List[Dict] = []

models: Dict[str, Dict[str, Any]] = {
    'pptod-small': {
        'name': 'pptod-small',
        'description': 'a T5 model that has been pre-trained on the ToD-BERT dataset **in this data format.** As such, '
                       'it is familiar with the meaning of these special separator tokens. However, it does not have'
                       'MultiWoZ training experience, so while it has adapted to the belief state grammar generally, it '
                       'is unaware of the particular slot name conventions of MultiWoZ.',
        'output_file': './output/pptod-small-10-percent.jsonl'
    },
    't5-small': {
        'name': 't5-small',
        'description': 'a T5 model with no dialogue experience. Data input has been transformed to exclude special tokens'
                       'that the model could not be familiar with.',
        'output_file': './output/t5-small-10-percent.jsonl'
    },
    'bart': {
        'name': 'bart',
        'description': 'a BART model with no dialogue experience. Data input has been transformed to exclude special tokens'
                       'that the model could not be familiar with.',
        'output_file': './output/bart-100ish-examples.jsonl'
    },
    'dialogpt': {
        'name': 'dialogpt',
        'description': 'DialoGPT is a (fine-tuned GPT-2) dialogue response generation model for multiturn conversations from 147M Reddit Conversation chains',
        'output_file': './output/dialogpt-100ish-examples.jsonl'
    },
    'my-t5-pptod-checkpoint': {
        'name': 'my-t5-pptod-checkpoint',
        'description': 'My re-implementation of the pptod approach, in re-structured code',
        'output_file': './output/my-pre-train-t5-100ish-examples.jsonl'
    },
    'my-t5-fine-tune-checkpoint': {
        'name': 'my-t5-fine-tune-checkpoint',
        'description': 'My re-implementation of the pptod approach, in re-structured code, fine-tuned on MultiWoZ',
        'output_file': './output/my-fine-tune-t5-100ish-examples.jsonl'
    }
}
for model_def in models.values():
    model_def['examples'] = []
    with open(model_def['output_file'], 'r') as f:
        for line in f.readlines():
            model_def['examples'].append(json.loads(line.strip()))


model_names = list(models.keys())


model_name = sidebar.selectbox('Model', model_names)
active_model = models[model_name]

st.write(f"""
#### Inputs

**Selected Model:** `{active_model['name']}`

{active_model['description']}

""")
"""
### Belief State Prediction

Below is the predicted belief state as a sequence. 

- `input` denotes the input, which has been transformed into a list for
human readability but is presented to the model as a sequence.
- `gold` is the target belief state in sequence form (slot-name slot-value pairs)
- `generated` is the model generated belief state sequence
"""
titles = [f"{i}: {e[0]['turn_domain'][0]} (Turn {e[0]['turn_num']})" for i, e in enumerate(active_model['examples'])]
title = sidebar.selectbox("Development Example", titles)
active_example = active_model['examples'][int(title[0])][0]

active_belief_spans = active_example['bspn_input'].split("> <")
active_example_bs = {'input':
              [ ('<' if i > 0 else '') +
                string +
                ('>' if string[-1] is not '>' and len(active_belief_spans) > 1 else '')
                for i, string in enumerate(active_belief_spans)],
          'generated': active_example['bspn_gen'],
          'gold': active_example['bspn']}

st.write(active_example_bs)
"""
##### Generated Overlap
"""
annotated_text(*bs_unigram_match_annotated_text(active_example_bs))

"""
---

### Response Generation

Below is the predicted response as a sequence. 

- `input` denotes the input, which has been transformed into a list for
human readability but is presented to the model as a sequence.
- `gold` is the target response sequence
- `generated` is the model generated response
"""
#title = st.selectbox("Development Example", titles)

active_example_resp = {'input':
              [ ('<' if i > 0 else '') +
                string +
                ('>' if string[-1] is not '>' else '')
                for i, string in enumerate(active_example['resp_input'].split("> <"))],
          'generated': active_example['resp_gen'],
          'gold': active_example['resp']}

st.write(active_example_resp)
"""
##### Generated Overlap
"""
annotated_text(*bs_unigram_match_annotated_text(active_example_resp))