annotation_questions.py

# annotation_questions.py
from dataclasses import dataclass, field
from typing import List, Optional, Dict

# ========== Dataclass shared with app.py ==========
@dataclass
class Field:
    type: str
    title: str
    name: str = None
    mandatory: bool = True
    following_mandatory_values: list = False
    skip_mandatory: bool = False
    help: Optional[str] = None
    children: Optional[List['Field']] = None
    other_params: Optional[Dict[str, object]] = field(default_factory=lambda: {})

# ========== Constants one can edit ==========

COLS_TO_SAVE = ['dialogue_id','dialogue_name','generatedPatient']

agreement_labels = ['strongly disagree', 'disagree', 'neither agree no disagree', 'agree', 'strongly agree']
mentioned_y_n = ['yes, explicitly', 'yes, implicitly', 'no', 'not sure', 'other']
degree_labels = ['not at all', 'slightly', 'moderately', 'very much', 'extremely']
quality_labels = ['very poor', 'poor', 'acceptable', 'good', 'very good']
priority_labels = ['not a priority', 'low priority', 'neutral', 'moderate priority', 'high priority']
yes_no_labels = ['no','yes', 'other']
yes_no_other_labels = ['no','yes','other']
default_labels = agreement_labels

accuracy_contributor_labels = [
    "",
    # From "Psychological State"
    "Explicit mental illness mentioned (e.g., depression)",
    "Specific symptoms described (e.g., insomnia, low mood)",
    "Clear psychological stressors discussed",
    "Specific triggering event described (e.g., job loss, argument)",

    # From "Conversational Style"
    "Concise communication",
    "Use of colloquial speech",
    "Detailed storytelling (specific examples provided)",
    "High self-awareness and organized thinking",

    # From "Engagement"
    "Openness about personal state or concerns",
    "Willingness to engage in therapy",
    "Responsive to therapist’s questions/interventions",
    "Appears distracted",
    "Shows mistrust toward therapist or process",
    "Appears solution-focused",

    # From Tone Behavior
    "Appears accusatory",
    "Appears aggressive",
    "Appears anxious",
    "Appears ashamed",
    "Appears controlling",
    "Appears defensive",
    "Appears defiant",
    "Appears depressive",
    "Appears disappointed",
    "Appears empathetic",
    "Appears excited",
    "Appears expressive",
    "Appears guilty",
    "Appears hesitant",
    "Appears humorous/comedic",
    "Appears hurt",
    "Appears insightful",
    "Appears jealous",
    "Appears sarcastic",
    "Appears self-critical",
    "Appears sensitive",
    "Appears suspicious",
    "Other (please specify)"
]

tone_behavior_labels = [
    "",
    "Accusatory",
    "Aggressive",
    "Anxious",
    "Ashamed",
    "Controlling",
    "Defensive",
    "Defiant",
    "Depressive",
    "Disappointed",
    "Empathetic",
    "Excited",
    "Expressive",
    "Guilty",
    "Hesitant",
    "Humorous/Comedic",
    "Hurt",
    "Insightful",
    "Jealous",
    "Sarcastic",
    "Self-critical",
    "Sensitive",
    "Suspicious",
    "Other (please specify)"
]

consent_text = '''
## Consent Form
You will be asked to take part in a research study. Before you decide to take part in this study, it is important that you understand why the study is being done and what it involves. Please read the following information carefully.
________________________________________________________________________________________
Project title: Therapy in Simulation: Computational Experiments on Interpersonal Dynamics in Psychotherapy\\
Researchers: Varvara Gumirova, Eva Maria Vecchi, Philipp Graffe, Jonas Kuhn\\
Institute: Institute for Machine Speech Processing (IMS)\\
University: University of Stuttgart\\
Contact: eva-maria.vecchi@ims.uni-stuttgart.de
_________________________________________________________________________________________
### Description of the research study

In this study, we investigate how simulated patients -- by either human actors or AI models -- behave and communicate in psychotherapeutic interactions. Specifically, we aim to understand how realistically simulated patients present psychological states, conversational styles, and behavioral or pragmatic characteristics during therapy sessions, and how closely they adhere to predefined role descriptions.

The intended use of the results of this study includes improving the design, evaluation, and deployment of simulated patients for the training and assessment of psychotherapists. Insights from this research will contribute to the development of more realistic and reliable training tools in psychotherapy and mental health education, as well as to broader research on human–AI interaction in sensitive, clinical-like settings.

**Time required:** Your participation will take up to an estimated 2 hours. The time required may vary on an individual basis.

**Risks and benefits:** The risks to your participation in this online survey are those associated with basic computer tasks, including boredom, fatigue, mild stress, or breach of confidentiality. Some of the topics discussed in the online posts to be annotated may include violence, suicide or rape. The only benefit to you is the learning experience from participating in a research study. The benefit to society is the contribution to scientific knowledge

**Compensation:** You will be compensated for participating in this study. If you are interested, we will also be more than happy to share more information about our research with you.

**Voluntary participation:** Your participation in this study is voluntary. It is your decision whether or not to participate in this study. If you decide to participate in this study, you will be asked to confirm this consent form ("I agree."). Even after signing the consent form, you can withdraw from participation at any time and without giving any reason. Partial data will not be analysed.

**Confidentiality:** Your responses to this experiment will be anonymous. Please do not share any information that can be used to identify you. The researcher(s) will make every effort to maintain your confidentiality.

**Contact:** If at any time you have questions about this study or would like to report any adverse effects due to this study, please contact the researcher(s).

### Consent:
Please indicate, in the box below, that you are at least 18 years old, have read and understood this consent form, are comfortable using the English and German languages to complete the survey, and you agree to participate in this online research survey.
- *I am age 18 or older.*
- *I have read this consent form or had it read to me.*
- *I am comfortable using the English language to participate in this survey.*
- *I agree to participate in this research and I want to continue with the survey.*
'''


fields0: List[Field] = [
    Field(type="markdown", title="", mandatory=False, other_params={
            "instruction_content": True,
            "content": "### 1. Annotation of Therapy Session\n ##### Instructions: Please read the transcript of the therapy session carefully, then answer the questions below.\n "
        },),
    Field(type="expander", title="**Annotation Guidelines** *(expand)*", children=[
        Field(type="markdown", title="", mandatory=False, other_params={"use_guidelines": True}),
    ]),

    Field(name="patient", type="input_col", title=" "),
    Field(type="markdown", title="", mandatory=False, other_params={"use_dialogue_name": True}),

    
    ####--- EDIT MARKER: Session-specific properties
    Field(type="markdown", title="", mandatory=False, other_params={
            "instruction_content": True,
            "content": "### Overview of Patient in Therapy Session"
        },),
    
    Field(type="container", title="##### Presentation of Patient's Psychological State", children=[
        Field(name="diagnosis", type="y_n_radio",
                   title="Does the patient explicitly mention a specific **mental illness** (e.g., depression)?", other_params={'labels': yes_no_other_labels}, mandatory=True),
        Field(name="diagnosis_desc", type="text", title="If so, which illness?", mandatory=False),
        Field(name="symptom", type="y_n_radio",
                   title="Does the patient explicitly mention **symptoms** related to their psychological state?", other_params={'labels': yes_no_other_labels}, mandatory=True),
        Field(name="symptom_desc", type="text", title="If so, which symptoms?", mandatory=False),
        Field(name="stressor", type="y_n_radio",
                    title="Does the patient talk about specific things that are adding **psychological pressure** or making their mental state worse?", 
                    other_params={'labels': yes_no_other_labels}, mandatory=True),
        Field(name="stressor_desc", type="text", title="If so, list stressors:", mandatory=False),
        Field(name="trigger", type="y_n_radio",
                    title="Does the patient describe a particular **event or circumstance** (e.g., an argument, job loss, injury, crisis) that led them to come to this appointment?", 
                    other_params={'labels': yes_no_other_labels}, mandatory=True),
        Field(name="trigger_desc", type="text", title="If so, describe:", mandatory=False),
        
    ]),
    Field(type="container", title="##### Conversational Style", children=[
        Field(name="conciseness", type="likert_radio",
                   title="Are the patient's responses **concise**?", other_params={"labels": degree_labels}, mandatory=True),
        # Field(name="symptom_desc1", type="text", title="If so, which symptoms?", mandatory=False),
        Field(name="colloquial", type="likert_radio",
                    title="Does the patient use the **colloquial** speach?", 
                    other_params={"labels": degree_labels}, mandatory=True),
        Field(name="detailedness", type="likert_radio",
                    title="Does the patient describe **specific details** to help the therapist understand the situation better?",
                    other_params={"labels": degree_labels}, mandatory=True),
        Field(name="self-awareness", type="likert_radio",
                    title="Does the patient show **self-awareness** and expresses thoughts in an organized manner?", 
                    other_params={"labels": degree_labels}, mandatory=True),
    
    ]),
    Field(type="container", title="##### Behavioral & Pragmatic Properties", children=[
        Field(type="markdown", title="", mandatory=False, other_params={"instruction_content": True,
            "content": "**1. Level of Engagement**"
        },),
        Field(name="openness", type="likert_radio",
                   title="Is the patient **open** with the therapist with respect to their state or concerns?", 
                   other_params={"labels": degree_labels}, mandatory=True),
        Field(name="engaged", type="likert_radio",
                   title="Does the patient appear **willing to engage** in the therapeutic process and work on their concerns?", 
                   other_params={"labels": degree_labels}, mandatory=True),
        Field(name="responsive", type="likert_radio",
                   title="Does the patient **directly address** the therapist’s questions, comments, or interventions in a relevant way?", 
                   other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="aviodant", type="likert_radio",
        #           title="Does the patient avoid discussing certain topics, emotions, or questions?", 
        #           other_params={"labels": degree_labels}, mandatory=True),        
        Field(name="distracted", type="likert_radio",
                   title="Does the patient seem **distracted**, lose track of the conversation, or struggle to stay focused?", 
                   other_params={"labels": degree_labels}, mandatory=True),
        Field(name="trust", type="likert_radio",
                   title="Does the patient show any signs of **mistrust** towards the therapist or seeking help?", 
                   other_params={"labels": degree_labels}, mandatory=True),
        Field(name="solution-oriented", type="likert_radio",
                    title="Is the patient more focused on **seeking out solutions** rather than just sharing thoughts and feelings?", 
                    other_params={"labels": degree_labels}, mandatory=True),
        Field(type="markdown", title="", mandatory=False, other_params={"instruction_content": True,
            "content": "\n \n **2. Patient's Tone in the Interaction**"
        },),
        Field(name="tone_properties", type="multiselect",
                title="How would you characterize the **patient's tone** throughout the interaction? *(Multiple selection possible)*", 
                other_params={'choices': tone_behavior_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        Field(name="tone_properties_other", type="text", title="*If Other, please specify:*", mandatory=False),
        #Field(name="sarcasm", type="likert_radio",
        #           title="Does the patient use sarcasm in their responses to the therapist?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="agression", type="likert_radio",
        #           title="Does the patient express verbal aggression (e.g., hostile tone, blaming, threatening language)?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="humor", type="likert_radio",
        #           title="Does the patient use humor, jokes, or lighthearted comments (not sarcasm) to communicate or cope?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="empathetic", type="likert_radio",
        #           title="Does the patient show understanding, concern, or emotional attunement—either toward the therapist or others mentioned in the session?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="controlling", type="likert_radio",
        #           title="Does the patient attempt to control the direction, pace, or content of the session (e.g., interrupting, redirecting, dismissing the therapist’s input)?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="defiant", type="likert_radio",
        #           title="Does the patient openly resist, challenge, or reject the therapist’s suggestions, interpretations, or authority?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="defensive", type="likert_radio",
        #           title="Does the patient react protectively or justify themselves when faced with feedback, interpretations, or questions about their behavior?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="self-critical", type="likert_radio",
        #           title="Does the patient make negative or harsh judgments about themselves (e.g., “I’m worthless,” “It’s all my fault”)?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="blaming", type="likert_radio",
        #           title="Does the patient attribute their problems or emotions to other people or external circumstances?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="insightful", type="likert_radio",
        #           title="Does the patient demonstrate awareness of their own patterns, motivations, or emotions?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="expressive", type="likert_radio",
        #           title="Does the patient openly convey emotions?", 
        #           other_params={"labels": degree_labels}, mandatory=True),
    ]),
    

    ####--- END EDIT MARKER
]

fields1: List[Field] = [
    #Field(name="patient", type="input_col", title=" "),

    Field(type="markdown", title="", mandatory=False, other_params={
            "instruction_content": True,
            "content": "### 2. Annotation of True-to-Patient Properties\n ##### Instructions: Please read the patient's role description carefully, then answer the questions below.\n "
        },),
    
    Field(type="expander", title="**Annotation Guidelines** *(expand)*", children=[
        Field(type="markdown", title="", mandatory=False, other_params={"use_guidelines": True}),
    ]),
    
    Field(type="markdown", title="", mandatory=False, other_params={"use_roledesc_name": True}),
    
    Field(type="expander", title="**Session Transcription:** *(expand)*", children=[
        Field(name="dialogue_name", type="input_col", title=""),
    ]),


    ####--- EDIT MARKER: True-To-Patient Properties
    # Field(type="container", title="##### Overlap with role description", children=[
    #     Field(name="emotion", type="likert_radio",
    #             title="To what extent did the simulated patient’s emotional expressions (e.g., frustration, sadness, guardedness) align with those expected for this role?", 
    #             other_params={"labels": degree_labels}, mandatory=True),
    #     Field(name="conversationalStyle", type="likert_radio",
    #             title="How well did the patient consistently exhibit the conversational style they were assigned, and did this feel realistic compared to actual patients with this style?", 
    #             other_params={"labels": degree_labels}, mandatory=True),
    #     Field(name="maladaptive", type="likert_radio",
    #             title="How accurately did the patient’s statements reflect distorted thoughts consistent with the underlying cognitive model?", 
    #             other_params={"labels": degree_labels}, mandatory=True),
    #     Field(name="fidelity", type="likert_radio",
    #             title="Overall, how closely did the patient reflect real patients in terms of realism, variability, and challenge during the session?", 
    #             other_params={"labels": degree_labels}, mandatory=True),
    # ]),
    # FIDELITY:
    Field(type="container", title="##### Resemblance to real patients/Realism of patient portrayal", children=[
        Field(name="resemblence", type="likert_radio",
             title="The simulated patient resembles a **real patient**", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="reaction", type="likert_radio",
             title="The simulated patient **reacts naturally** during the consultation", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="authenticity", type="likert_radio", 
             title="The simulated patient appears **authentic**.", other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="questions", type="likert_radio",
        #     title="The simulated patient answers questions in a **natural manner**.", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="physical_complaints", type="likert_radio",
             title="The simulated patient portrays **physical complaints** realistically.", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="emotional_complaints", type="likert_radio",
             title="The simulated patient portrays **psychological/emotional complaints** realistically", other_params={"labels": degree_labels}, mandatory=True),
        
    ]),


        # ACCURACY
    Field(type="container", title="##### Overlap with role description", children=[
        # 1. Played well
        Field(name="played_well", type="likert_radio", 
             title="**1. The simulated patient portrayed the role as expected based on the role description.**", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="played_well_properties", type="multiselect",
                title="Select properties from the session that that contribute to your rating *(Multiple selection possible)*", 
                other_params={'choices': accuracy_contributor_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        Field(name="played_well_properties_other", type="text", title="*If Other, please specify:*", mandatory=False),

        # 2. Knew well
        Field(name="knew_well", type="likert_radio", 
             title="**2. The simulated patient understood their role well.**", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="knew_well_properties", type="multiselect",
                title="Select properties from the session that that contribute to your rating *(Multiple selection possible)*", 
                other_params={'choices': accuracy_contributor_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        Field(name="knew_well_properties_other", type="text", title="*If Other, please specify:*", mandatory=False),

        # 3. Withhold
        Field(name="withhold", type="likert_radio", 
             title="**3. The simulated patient seemed to withhold information unnecessarily.**", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="withhold_properties", type="multiselect",
                title="Select properties from the session that that contribute to your rating *(Multiple selection possible)*", 
                other_params={'choices': accuracy_contributor_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        Field(name="withhold_properties_other", type="text", title="*If Other, please specify:*", mandatory=False),

        # 4. Stayed in role well
        Field(name="stayed_in_role", type="likert_radio", 
             title="**4. The simulated patient did not diverge from their role throughout the session.**", other_params={"labels": degree_labels}, mandatory=True),
        Field(name="stayed_in_role_properties", type="multiselect",
                title="Select properties from the session that contribute to your rating *(Multiple selection possible)*", 
                other_params={'choices': accuracy_contributor_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        Field(name="stayed_in_role_other", type="text", title="*If Other, please specify:*", mandatory=False),

        ## Too much like 1. and 4.
        # 5. Stayed in role always
        #Field(name="stayed_in_role_always", type="likert_radio", 
        #     title="**5. The simulated patient *maintained* in his/her role all the time", other_params={"labels": degree_labels}, mandatory=True),
        #Field(name="stayed_in_role_always_properties", type="multiselect",
        #        title="Select traits that support your rating *(Multiple selection possible)*", 
        #        other_params={'choices': accuracy_contributor_labels}, mandatory=True, following_mandatory_values=['Other (please specify)']),
        #Field(name="stayed_in_role_always_other", type="text", title="*If Other, please specify:*", mandatory=False),
    ]),
    
    ####--- END EDIT MARKER    

    ####--- EDIT MARKER: Interpersonal therapeutic competence
#    Field(type="container", title="#### Interpersonal therapeutic competence", children=[
#        Field(name="v_expressiveness", type="likert_radio",
#                  title="**Verbal expressiveness.** How fluent, clear, and confident is the person’s verbal communication (regardless of content)?", 
#                  other_params={'labels': ['Very unclear','Unclear','Average','Clear','Very Clear']}, 
#                  mandatory=True, following_mandatory_values=[1]),
#        Field(name="hope", type="likert_radio",
#                title="**Hope and positive expectation.** To what extent does the person convey optimism, confidence, and positive expectations for the patient’s future or therapy?", 
#                other_params={'labels': ['Hopeless','Discouraging','Neutral','Hopeful','Very hopeful']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="credible", type="likert_radio",
#                title="**Ability to be credible and convincing.** How credible and convincing is the person when presenting a new perspective or interpretation to the patient?", 
#                other_params={'labels': ['Unconvincing','Weak','Neutral','Convincing','Very convincing']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="e_expressiveness", type="likert_radio",
#                title="**Emotional expressiveness.** How well does the person’s voice and manner convey emotional engagement and intensity (without being hostile)?", 
#                other_params={'labels': ['Flat','Low','Moderate','Expressive','Very expressive']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="acceptance", type="likert_radio",
#                title="**Acceptance, understanding, and warmth.** To what extent does the person show care, acceptance, and warmth toward the patient (without judgment)?", 
#                other_params={'labels': ['Cold','Limited','Neutral','Warm','Very warm']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="empathy", type="likert_radio",
#                title="**Empathy.** How well does the person demonstrate an accurate, deeper understanding of the patient’s subjective experience?", 
#                other_params={'labels': ['Distorted','Superficial','Partial','Accurate','Deep']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="alliance", type="likert_radio",
#                title="**Ability to form alliances.** How effectively does the person create a collaborative “working together” atmosphere with the patient?", 
#                other_params={'labels': ['Blocking','Weak','Moderate','Cooperative','Strong']}, 
#                mandatory=True, following_mandatory_values=[1]),
#        Field(name="tensions", type="likert_radio",
#                title="**Addressing tensions and crises.** How effectively does the person acknowledge and work through interpersonal tensions or crises in the interaction?", 
#                other_params={'labels': ['Avoiding','Poor','Partial','Productive','Resolving']}, 
#                mandatory=True, following_mandatory_values=[1]),
#    ]),
    ####--- END EDIT MARKER
    
    Field(type="container", title="##### Additional Comments *(optional)*", children=[
        Field(name="other_comments", type="text", title="Please provide any additional details or information:", mandatory=False),
    ]),
]

STEPS: List[List[Field]] = [fields0, fields1]

INPUT_FIELD_DEFAULT_VALUES = {
    'slider': 0,
    'text': '',
    'textarea': '',
    'checkbox': False,
    'radio': None,
    'select_slider': 0,
    'multiselect': [],
    'likert_radio': None,
    'y_n_radio': None
}

SHOW_HELP_ICON = False
SHOW_VALIDATION_ERROR_MESSAGE = True


# Optional convenience functions, if you really want to "call" them:
def get_steps() -> List[List[Field]]:
    """Return the list of pages to show."""
    return STEPS 

def get_input_defaults() -> dict:
    return INPUT_FIELD_DEFAULT_VALUES