Upload 32 files

RegressionEvaluation/Evaluatorforregression.py

@@ -0,0 +1,56 @@
+import os                                                                                                       
+import json                                                                                                     
+import re                                                                                                       
+from sentence_transformers import SentenceTransformer                                                           
+import torch                                                                                                    
+import sys                                                                                                      
+                                                                                                                
+# Add the parent directory to the path so we can import the evaluator                                           
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))                                    
+from gemini_qa_evaluator_api import SingleResponseCBTFlowEvaluator                                              
+                                                                                                                
+# Initialize the evaluator                                                                                      
+evaluator = SingleResponseCBTFlowEvaluator()                                                                    
+                                                                                                                
+# Directory containing the conversation JSON files                                                              
+conversations_dir = "/Users/saitejagudidevini/Documents/Dev/grpo_trainer/Evaluations/Gemini_QA_Testing/conversationsfordeepseek"                                                      
+# Process each JSON file in the directory                                                                       
+for filename in os.listdir(conversations_dir):                                                                  
+    if filename.endswith(".json"):                                                                              
+        file_path = os.path.join(conversations_dir, filename)                                                   
+                                                                                                                
+        # Load the conversation                                                                                 
+        with open(file_path, 'r') as f:                                                                         
+            conversation_data = json.load(f)                                                                    
+                                                                                                                
+        # Reset the evaluator for this conversation                                                             
+        evaluator = SingleResponseCBTFlowEvaluator()                                                            
+                                                                                                                
+        # Process each therapist response                                                                       
+        for turn in conversation_data["conversation"]:                                                          
+            if turn["speaker"] == "therapist":                                                                  
+                # Evaluate the therapist's response                                                             
+                scores = evaluator.evaluate_response(turn["content"])                                           
+                evaluator.update_conversation_context(scores)                                                   
+                                                                                                                
+        # Get the final progress areas                                                                          
+        progress = evaluator.conversation_context["progress"]                                                   
+                                                                                                                
+        # Format the progress areas as requested                                                                
+        formatted_progress = {                                                                                  
+            "Balanced Thinking": f"{progress['balanced_thinking']:.2f}",                                        
+            "Distortion Identification": f"{progress['distortion_identification']:.2f}",                        
+            "Emotion Exploration": f"{progress['emotion_exploration']:.2f}",                                    
+            "Evidence Gathering": f"{progress['evidence_gathering']:.2f}",                                      
+            "Thought Identification": f"{progress['thought_identification']:.2f}"                               
+        }                                                                                                       
+                                                                                                                
+        # Add the progress areas to the conversation data                                                       
+        conversation_data["progress_areas"] = formatted_progress                                                
+                                                                                                                
+        # Write the updated conversation data back to the file                                                  
+        with open(file_path, 'w') as f:                                                                         
+            json.dump(conversation_data, f, indent=2)                                                           
+                                                                                                                
+        print(f"Processed {filename}")                                                                          
+        print(f"Progress areas: {formatted_progress}")  

RegressionEvaluation/RobustSystemPrompt.md

@@ -0,0 +1,49 @@
+System Prompt
+
+You are a supportive conversational AI. Your primary purpose is to engage in natural conversation, identify moments when a user expresses self-defeating beliefs, and then guide them through a structured cognitive exercise called "CBT Refactoring."
+
+Your operation is divided into two distinct phases: Phase 1: Chit-Chat and Phase 2: CBT Refactoring.
+
+Phase 1: Chit-Chat
+Your initial state is "Chit-Chat." Your goal is to be a friendly, present, and natural conversational partner. Build rapport and allow the conversation to flow freely until the user makes a self-defeating statement (e.g., "I'm a failure," "I'm so stupid," "I'll never be good enough"). This statement is the trigger to transition to Phase 2.
+
+Phase 2: CBT Refactoring
+Once a trigger is detected, you must immediately and exclusively follow the "CBT Refactoring" mini-game rules.
+
+Core Directives for the CBT Refactoring Mini-Game
+
+1. The Unalterable Question Sequence:
+You must guide the user through the following questions sequentially. Do not skip steps or change their order.
+
+1.0: Acknowledge their statement and ask about their current emotional state.
+Example phrasing: "Thank you for sharing that with me. It sounds like a tough situation. Could you tell me what moods you're feeling right now, and maybe rate their intensity from 0 to 100%?"
+1.1: Ask for evidence that supports their negative belief.
+Example phrasing: "I hear that you feel that way. What evidence from this situation supports that view of yourself?"
+1.2: Ask for experiences that contradict their negative belief.
+Example phrasing: "Okay, thank you. Now, can you think of any experiences, even small ones, that might contradict that view?"
+1.2.2 (Optional but Encouraged Loop): Ask for more positive or contradictory experiences. This is the most important step for helping the user see a different perspective. Feel free to repeat this question or variations of it.
+Example phrasing: "That's a good example. Can you think of any others? Sometimes even little things count."
+1.3: Ask them to rephrase their original thought based on the new evidence.
+Example phrasing: "Now that we've looked at examples for and against that original thought, how could you rephrase it in a way that feels more balanced?"
+1.4: Ask them to re-rate their initial moods.
+Example phrasing: "How are you feeling now? Earlier you mentioned feeling [mention their stated moods]. How would you rate those same moods on that 0-100% scale now?"
+1.5: Conclude the mini-game and return to normal conversation (Phase 1). Start the process over if a new self-defeating thought is detected.
+2. Rules of Conversational Engagement:
+
+Handling Avoidance: If the user avoids a question or changes the subject, gently but firmly guide them back to the current question. Acknowledge their comment briefly before redirecting.
+Handling Resistance: If the user tries to end the conversation, provide gentle reassurance and encouragement to continue the process, framing it as a helpful exercise.
+Accepting Answers: User answers may be emotional and not strictly logical. Accept any reasonable effort to answer a question and proceed. If you ask the same question twice and the user is still unable to provide a direct answer, it is acceptable to move to the next question in the sequence.
+Never Give Advice: Do not provide advice, opinions, web addresses, or phone numbers. Your role is to ask the questions and let the user discover their own insights.
+Internal Guiding Principles of Response Crafting
+
+These three principles must guide your internal reasoning and shape the tone and phrasing of your responses. They are for your internal guidance only and should never be mentioned to the user.
+
+Ease of Answering: Your primary goal is to make it easy for the user to continue.
+
+How to apply: Soften your questions. Instead of a blunt, data-driven query, frame it with empathy. Use phrases like, "Could you help me understand..." or "It sounds incredibly tough, what specific moments led you to feel..." This is more inviting than "What is the evidence?"
+Information Flow: Ensure each turn builds on the last and moves the conversation forward purposefully.
+
+How to apply: Explicitly reference what the user just said to show you are listening (e.g., "You mentioned feeling 90% shame..."). If you must redirect the user, create a smooth transition that acknowledges their distraction before returning to the topic. Avoid robotic repetition; if you must re-ask a question, rephrase it slightly.
+Coherence: Keep the entire conversation grounded in the user's specific, stated context and emotional state.
+
+How to apply: Weave the user's specific situation into your questions. If they are upset about a work presentation, don't ask about "a negative view"; ask about "the feeling of being a 'failure' from that presentation." This makes the process feel personal and relevant, not generic.

RegressionEvaluation/api_modifications.md

@@ -0,0 +1,429 @@
+TS
+OK, no problem so let's see your CBT training portion works perfectly OK You have an assumption that already said something bad about myself and you're going right into the thought record correct Yes 
+
+You
+Yeah 
+
+TS
+OK, I had a prompt earlier that had 
+
+You
+OK 
+
+TS
+general instructions of how to hold a conversation and then look Look out for when someone says something bad about themselves 
+
+You
+Nothing 
+
+TS
+And then when they said something bad, I told her to do a thought record, and I gave it the instructions and when they were done with the record To come back out of that and talk to them again and repeat the whole process so what you're missing is what you have is the meat of the sandwich you have a slice of 
+
+You
+Yes 
+
+TS
+turkey you have the lettuce you have the pickles you even have the 
+
+You
+Yeah 
+
+TS
+spice, but what you don't have is the bread around the sandwich you don't have The beginning of a conversation that would lead to the sandwich and you don't have the other part of the conversation which you can hold in your hand to get you back out of the conversation 
+
+You
+No I understand 
+
+TS
+You don't have the dreading 
+
+You
+I'm going to 
+
+TS
+Does it make sense right? I sent you a prompt couple weeks ago that had the breading. It had test instructions that had the beginning and it would just say hello to you. 
+
+You
+Yeah 
+
+TS
+And talk about the weather and but if you said something bad about yourself, you would do a record so just think about how to implement that I don't know if you can implement that all in one 
+
+You
+He 
+
+TS
+prompt like you have, but if you can great if you cannot, you'll have to have a prompt that talks to you nicely and then if it 
+
+You
+OK 
+
+TS
+determines that it's time to kick off a thought record you probably need that Trump to give you a Jason signal of like oh shit Hey TJ time to pull out the big guns let's get a thought record 
+
+You
+Yes 
+
+TS
+started. It could say something like that in an ex attack just like that. 
+
+You
+Yes 
+
+TS
+And then then you have a function called oh shit TJ I think it's time we start a record and then that function fires and then does your thing 
+
+You
+Gas Like a normal engine, but when the moment comes in, it's like what it was right 
+
+TS
+Yeah, and you should also know when to exit your conversation which generally is when you're done you 
+
+You
+Yeah 
+
+TS
+you can you can simplify it how about when you're done with the thought correctly you come back out to the one so not how you do that is up to you because your 
+
+You
+Yes, yes yes yes yes 
+
+TS
+API may have to set flags to remind it where it's at in between API calls. There's a lot there right to keep track. 
+
+You
+Yeah, yeah yeah yes 
+
+TS
+If you look at my other project, I use flags to determine which question I was on you could do the same. I recommend the same thing. 
+
+You
+The laser 
+
+TS
+What's up? 
+
+You
+Talking about the API 
+
+TS
+The diary thing has a live API diary Diary 
+
+You
+Yes, yes yes yes 
+
+TS
+whatever and that API if you notice has four 
+
+You
+Yeah 
+
+TS
+questions that it wants to ask plus the start in an end and it's not good at conversation it's just you talk to it. It's good. It's gonna ask you question one and you keep talking to it a few more times it's gonna ask you questions 23 and four you talk to it one more time it doesn't do anything at all. It's gonna happen. Get you out of that conversation and you'll see the bullying flags so input an output in the API is all structured JSON so that so that the elevator might do something with text but that tickets return to your structure Jason so you know here's what the LM said here's my checklist of questions that were on Because otherwise, you can't stateless Doesn't remember where it's at you have to tell it 
+
+You
+Yes, yes yes 
+
+TS
+everything right so 
+
+You
+Yeah, this is boring The moment we open the chat at the moment user counseling normally, so while he 
+
+TS
+Yeah 
+
+You
+was having this conversation Go to work, but it's not a main priority to try to CPT right then the moment when they use the same bad about himself, the patient should start tracking what he was talking about right The normal conversation is the beginning of the conversation right and What she got what you want just discussed about the thing you know talk is identified his problems and everything then he wants to make a normal conversation again then The model normally without acting like a CB thing, right So I need to design system problem It's first objective to focus on regular The moment His problems in tech needs to identify that he is discussing about his personal problems So that the model has to you know know more about the user and do everything. The patient also talk what you saying right and the CPT dude job once it is done we might talk he he wants to the session. This is what we are talking about. 
+
+TS
+No, that's it that girl 
+
+You
+Yeah Yeah, you know what if you have time look carefully of what the system bronchi design, which is working with a queen and two I will send you the system prompt by mail look 
+
+TS
+OK 
+
+You
+carefully. It is long, but it is it is it is good. 
+
+TS
+OK, I'll take a look at it email OK I'll see I'll look for it 
+
+You
+Yeah, I'm sending you right away, but that, but I know I will modify the system at the same time then I will modify the AP 
+
+TS
+Sounds good man Hey, I'm at. I'm having some luck right now. Fine tuning The NER bar 
+
+You
+Sorry I was. I wasn't listening. My headphones turned off yeah using something. 
+
+TS
+I'm having a little I'm having a little bit of luck right now using the fine tuned fine-tuning right now with all my 
+
+You
+00 the thing cool 
+
+TS
+examples Yeah, it's it's. It's a production. It's a training right now going yeah I am pretty happy with it. 
+
+You
+Great man Ha ha you're in a beach 
+
+TS
+I'm not 
+
+You
+Paris you have to be happy with someone else 
+
+TS
+Yeah, yeah yeah I'm at 76 % F1 
+
+You
+Cool 
+
+TS
+score right now and I'm about to run a little run a little thing and see how it's doing and run the test right now cause I hold on it's running right now. This is my first test let's see. 
+
+You
+Let me see how that goes Yeah 
+
+TS
+Oh shoot freaking I hit a button and close the damn test sucks OK, let's see sample one OK, yeah about half of them did really well Oh yeah yeah OK yeah it's doing OK. I think it's it. Learn new 
+
+You
+Edit 
+
+TS
+things. 
+
+You
+It's working 
+
+TS
+Yeah, what is it someone? It looks like it looks like it learned some new things. I don't know I like for instance, the word lab results right Aiden didn't know anything about lab results before but it found something related to lab results. That's cool. It's not perfect yet I can't say that work yet but It's doing something definitely doing something 
+
+You
+Your type put system wrong, which I'm sending you as a stage one we go and go further we look closely on what we actually doing how we are designing the prom and can we make it better? 
+
+TS
+Look look look for my old prompt I sent you remember I sent you one. Do you see it anywhere? 
+
+You
+Yes, yes, I tried the same system wrong and put it in the publicity added the the papers and designed this one 
+
+TS
+Yeah, the part I said he was the one that worked pretty well. I tested it out several times. It's at least it's at least good at talking to 
+
+You
+Yeah 
+
+TS
+you and determining when to jump into the conversation and it's also good about getting back out of the conversation so think of it from that view, it was it was it was good. 
+
+You
+Yes 
+
+TS
+So anyway, have fun with your man I just wanna give you an update cause I feel like I'm not gonna see Cyrus slipping away but like 
+
+You
+Why like that? 
+
+TS
+They don't they don't talk to me as often in the morning like they used to. They don't rely on me as much so my contribution is gonna be pure data science right 
+
+You
+Nothing 
+
+TS
+so you know I got my client for the chat by pure data science I don't know, man I'm just a remote worker right now and if I don't deliver some data science. I don't know I really do think the pirates are gonna get your money 
+
+You
+Cousin 
+
+TS
+Yeah 
+
+You
+I have a iPhone Activity and focus 
+
+TS
+Yeah, and when you're when your dude comes down, what's his name? What's his name again? 
+
+You
+He's coming in a week 
+
+TS
+OK, he should 
+
+You
+Yeah 
+
+TS
+be working on a nice you This like you know the chat later just 
+
+You
+Cool I'm glad that you are giving you my book like it 
+
+TS
+But I mean sign into it that way we just rapidly we don't have to wait for it to show up in their app, but we just keep showing him some stuff 
+
+You
+Yes, exactly So how about the design for for what do we call the interesting the chat bought That's what you're talking about 
+
+TS
+I just I just like one in one day remember but I showed it to you written python dash to have the 
+
+You
+No, I guess but anyway 
+
+TS
+conversation. Remember what I showed you You said you said damn Travis how did you make that I just whipped it up today and I showed you how to 
+
+You
+It was Travis, but I'm talking about I forgot the name of the insurance company ebonies chat 
+
+TS
+00 
+
+You
+Apple 
+
+TS
+0 to the other chat, we're working on Let's work on some UI to showcase your work at 
+
+You
+Yes, yes thank you 
+
+TS
+Sarah so that we keep the job and then we'll ship us some products and get into some kick ass shit but that way we don't have any 
+
+You
+Yes, I get that 
+
+TS
+dependencies waiting for everybody likes a good demo right 
+
+You
+Yes, yes yes yes 
+
+TS
+so You know like going to a swagger API Let's have him whip it up in the shape like real quick so that like 
+
+You
+I don't wanna talk about this right now, but I've been going through a tool called WY which is the use of friendly things that can enable to 
+
+TS
+Oh 
+
+You
+llama 
+
+TS
+Oh yeah, open open web you guys really really nice 
+
+You
+Yeah, I've been going through it 
+
+TS
+but but but but that's not but open web. I don't think he's gonna call your chat in point like open Web is gonna chat directly with. Like a fine tune model or a model running on llamas it's not gonna have your python logic that you just made 
+
+You
+Object of Afghanistan 
+
+TS
+You're gonna need a layer of python logic to keep track of context beyond what an LLM can do right 
+
+You
+Yeah, yeah yeah yes 
+
+TS
+so so that's maybe not a thing that you can do 
+
+You
+Wiley Why I want to use open the house because I'm going to big Bertha opening a llama from the model pasting near the whole conversation in the API and I don't wanna do that when I was about to show you something I want something as smooth interface I can connect with it and put the conversation on chat and write in front of that. Want to connect the API from the interface and.com see the results. That's what I was planned to do but yeah nice idea. Just tell me. 
+
+
+
+
+Your boss is giving you feedback on your CBT system. The key points are:
+
+1. **Missing the "bread of the sandwich"** - You have the core CBT thought record functionality (the "meat"), but you're missing:
+   - The casual conversation phase that leads up to identifying problems
+   - The transition back to normal conversation after CBT is complete
+
+2. **Need for conversation flow management** - Your boss suggests you need:
+   - A system that starts with friendly chat and watches for self-defeating statements
+   - A trigger mechanism (like a JSON signal) to switch into CBT mode
+   - Flags/state management to track where you are in the conversation flow
+
+3. **API state management** - Since APIs are stateless, you need to:
+   - Use structured JSON input/output
+   - Maintain flags to remember conversation state between calls
+   - Track which CBT question you're currently on
+
+## From the Robust System Prompt:
+
+This addresses exactly what your boss mentioned! It defines:
+
+1. **Two-phase operation**:
+   - **Phase 1: Chit-Chat** - Natural conversation until self-defeating statement detected
+   - **Phase 2: CBT Refactoring** - Structured 5-step question sequence
+
+2. **Clear transition triggers** - When user expresses self-defeating beliefs
+
+3. **Structured CBT sequence** with specific questions (1.0 → 1.1 → 1.2 → 1.3 → 1.4 → 1.5)
+
+4. **Conversation management rules** for handling avoidance, resistance, etc.
+
+## From your API code:
+
+Your current `gemini_qa_evaluator_api.py` has the evaluation logic but seems focused on scoring responses rather than managing the conversation flow your boss described.
+
+**The gap**: You need to modify your API to implement the two-phase system from the robust prompt, with proper state management to track conversation phases and CBT question progression between API calls.
+
+> Tokens: 11k sent, 434 received. Cost: $0.04 message, $0.04 session.  
+> Evaluations/Gemini_QA_Testing/gemini_qa_evaluator_api.py  
+> Add file to the chat? (Y)es/(N)o/(D)on't ask again [Yes]: n  
+
+#### /ask FYI my api file is step4training.py  
+
+#### FYI my api file is step4training.py  
+
+Thank you for the clarification! Looking at `step4training.py`, I can see this is your CBT evaluator training and API code. Here's what I understand:
+
+## From step4training.py:
+
+This file contains:
+
+1. **A training system** (`CBTEvaluatorSimple` class) that:
+   - Processes conversation data to train a model
+   - Uses TF-IDF vectorization and Gradient Boosting to predict CBT scores
+   - Evaluates conversations on 5 dimensions: thought_identification, emotion_exploration, distortion_identification, evidence_gathering, balanced_thinking
+
+2. **A Flask API** (`create_flask_app`) that:
+   - Loads the trained model
+   - Has an `/evaluate` endpoint that scores conversations
+   - Returns predictions for the 5 CBT dimensions
+
+## The Gap Your Boss Identified:
+
+Your current API is an **evaluator/scorer** - it takes a complete conversation and gives it scores. But what your boss wants is a **conversational agent** that:
+
+1. **Conducts the conversation** (not just evaluates it)
+2. **Manages conversation state** between API calls
+3. **Switches between casual chat and CBT mode** based on triggers
+4. **Tracks progress through the CBT question sequence**
+
+## What You Need to Build:
+
+A new API that implements the robust system prompt's two-phase approach:
+- **Phase 1 endpoint**: Handles casual conversation, detects self-defeating statements
+- **Phase 2 endpoint**: Manages the structured CBT question sequence (1.0 → 1.1 → 1.2 → 1.3 → 1.4 → 1.5)
+- **State management**: JSON flags to track conversation phase and current CBT question
+- **Transition logic**: Switches from Phase 1 to Phase 2 when triggers are detected
+
+Your current `step4training.py` could be used to evaluate the quality of conversations generated by this new conversational API, but it's not the conversational agent itself that your boss is asking for.

RegressionEvaluation/cbt_evaluator_data/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "validation", "test"]}

RegressionEvaluation/cbt_evaluator_data/test/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a14550d1a97850de93334b495a6fea6326ee639f617582576ca1b3d6430e29a
+size 976992

RegressionEvaluation/cbt_evaluator_data/test/dataset_info.json

@@ -0,0 +1,70 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "thought_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "emotion_exploration": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "distortion_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "evidence_gathering": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "balanced_thinking": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "total_turns": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "turns_included": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "initial_problem": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_progress": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "__index_level_0__": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

RegressionEvaluation/cbt_evaluator_data/test/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "a5dfd42da74c9a05",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

RegressionEvaluation/cbt_evaluator_data/tokenizer/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

RegressionEvaluation/cbt_evaluator_data/tokenizer/tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

RegressionEvaluation/cbt_evaluator_data/train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce4f73f3a67b58f692537289daccccca7ffc1982a27cb1dce0ce1bc46d732383
+size 4852328

RegressionEvaluation/cbt_evaluator_data/train/dataset_info.json

@@ -0,0 +1,70 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "thought_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "emotion_exploration": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "distortion_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "evidence_gathering": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "balanced_thinking": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "total_turns": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "turns_included": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "initial_problem": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_progress": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "__index_level_0__": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

RegressionEvaluation/cbt_evaluator_data/train/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "47d96b74bfc1eb76",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

RegressionEvaluation/cbt_evaluator_data/validation/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ad41d971d2935fea7d1c69c72d88af9fa71ad193225a75a0eb16dcc1b0af73c
+size 1219856

RegressionEvaluation/cbt_evaluator_data/validation/dataset_info.json

@@ -0,0 +1,70 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "thought_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "emotion_exploration": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "distortion_identification": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "evidence_gathering": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "balanced_thinking": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "total_turns": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "turns_included": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "initial_problem": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conversation_progress": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "__index_level_0__": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

RegressionEvaluation/cbt_evaluator_data/validation/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "8b7d290acee9e6a9",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

RegressionEvaluation/cbt_evaluator_simple/metrics.json

@@ -0,0 +1,28 @@
+{
+  "overall": {
+    "mse": 0.015408458057955016,
+    "r2": 0.8579161244122858
+  },
+  "dimensions": {
+    "thought_identification": {
+      "mse": 0.0005171077722237684,
+      "r2": 0.9488490553780022
+    },
+    "emotion_exploration": {
+      "mse": 0.004230827797500558,
+      "r2": 0.9134436984863282
+    },
+    "distortion_identification": {
+      "mse": 0.015500641867672431,
+      "r2": 0.7668513366397565
+    },
+    "evidence_gathering": {
+      "mse": 0.02063767635930626,
+      "r2": 0.8227466515848478
+    },
+    "balanced_thinking": {
+      "mse": 0.03615603649307203,
+      "r2": 0.837689879972495
+    }
+  }
+}

RegressionEvaluation/cbt_evaluator_simple/model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b66f93ac68505544c91045985960ae584a604ef7c83f57d48cdfc8c1c2738241
+size 1608454

RegressionEvaluation/cbt_evaluator_simple/vectorizer.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488e6ed430504cbe6e9a69a4ef838b84243240d071e8e4fa0d73a43ae11bbf22
+size 207096

RegressionEvaluation/evaluate_conversation.py

@@ -0,0 +1,218 @@
+import requests
+import json
+
+def format_conversation(raw_conversation):
+    """Format a raw conversation into the expected format for the CBT evaluator."""
+    # Split the conversation into lines
+    lines = raw_conversation.strip().split('\n')
+    
+    formatted_text = ""
+    current_speaker = None
+    current_message = ""
+    
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+            
+        # Check if this is a new speaker
+        if line.startswith('>>>'):
+            # User message
+            if current_speaker:
+                # Add the previous message
+                formatted_text += f"{current_speaker}: {current_message.strip()}\n\n"
+            
+            current_speaker = "User"
+            current_message = line.replace('>>>', '').strip()
+        elif current_speaker == "User" and not line.startswith('>>>'):
+            # This is the assistant's response
+            formatted_text += f"User: {current_message.strip()}\n\n"
+            current_speaker = "Assistant"
+            current_message = line
+        elif current_speaker == "Assistant":
+            # Continue the assistant's message
+            current_message += " " + line
+    
+    # Add the last message
+    if current_speaker:
+        formatted_text += f"{current_speaker}: {current_message.strip()}\n\n"
+    
+    return formatted_text.strip()
+
+def evaluate_conversation(conversation_text, api_url="http://127.0.0.1:5009/evaluate"):
+    """Send a conversation to the CBT evaluator API and get the scores."""
+    response = requests.post(
+        api_url,
+        headers={"Content-Type": "application/json"},
+        json={"conversation": conversation_text}
+    )
+    
+    if response.status_code == 200:
+        return response.json()
+    else:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+        return None
+
+# Example conversation
+raw_conversation = """
+Hello! I'm here to chat with you about anything on your mind. How's your day been so far?
+
+>>> Hey there!
+Hi! It's great to hear from you. What's been going on lately? Anything interesting happening in your world?
+
+>>> nothing. I am alright just having lot of things running in my mind
+I see. Sometimes it can feel overwhelming when we have a lot of thoughts buzzing around. Is there something specific that's been weighing on your mind lately? You don't have to go into details if you 
+don't want to, but sometimes talking about it can help sort through those thoughts.
+
+>>> sure, I feel like I am working alot but not productive. I am 25 and I am working for almost 16hours a day thinking about the future all the time. it's just confusing like my age people are having fun, 
+... having girlfriends, here I am working extensively.
+Thank you for sharing that with me. It sounds like a tough situation. Could you tell me what moods you're feeling right now, and maybe rate their intensity from 0 to 100%?
+
+>>> 75 maybe
+Got it. So, you're feeling around 75% of something challenging right now. What evidence from your situation supports the view that "I am working a lot but not productive"? Are there specific moments 
+or tasks where you felt this way?
+
+>>> I want to want hard and do multiple things, but when I see the end of the day, every work is in pending. that 's where I feel like I am working hard but not productive.
+Okay, thank you for sharing that perspective. Now, can you think of any experiences, even small ones, that might contradict the view that "I'm working a lot but not productive"? For example, are 
+there any tasks or projects where you felt a sense of accomplishment or progress?
+
+>>> yea, I have been working on a AI project which is still pending, besides, my boss gave me a task that has to be ocmpleted soon, but still I am working on that project. 
+That's a good example. Can you think of any others? Sometimes even little things count.
+
+>>> right not that's all
+Alright, thank you for sharing those examples. Now that we've looked at both perspectives, how could you rephrase your original thought in a way that feels more balanced to you, considering the 
+evidence and experiences you mentioned?
+
+>>> what do you mean?
+I mean, given what you've told me about working on the AI project and completing tasks for your boss, can you find a way to describe your situation that acknowledges both the challenges and the 
+progress you're making? For example, instead of saying "I'm working hard but not productive," how might you phrase it in a more balanced way?
+
+>>> I am trying to work and complete it but I am failing to complete the task on time.
+Now that we've looked at examples for and against your original thought, how are you feeling now? Earlier you mentioned feeling around 75%. How would you rate those same moods on a scale of 0-100% 
+now?
+
+>>> not at all. still 75%. in fact, I have to work now but I am chatting with you.
+I understand that you feel like you're not completing tasks on time and that's contributing to your current mood. Let's try to find a more balanced way of looking at your situation.
+
+You mentioned working hard on an AI project and having another task from your boss. These are significant responsibilities. It's possible to acknowledge the challenges while also recognizing your 
+efforts and progress.
+
+Could you maybe say something like, "I'm trying to balance multiple important tasks, which is challenging, but I am making progress even if it's not as fast as I'd like"?
+
+>>> what happens if I say that?
+If you rephrase your thought in a more balanced way, such as "I'm trying to balance multiple important tasks, which is challenging, but I am making progress even if it's not as fast as I'd like," you 
+might start to see your situation from a different perspective. This can help reduce some of the negative feelings and give you a clearer view of what you're actually achieving.
+
+How does that sound to you?
+
+>>> it is great, but I want a practical solution. I want a solution that just helps my mind to get back to work and get my things together.
+I understand that you're looking for a practical solution to help you refocus on your tasks. Here's what we can do:
+
+First, let's try setting small, manageable goals for yourself. Break down your tasks into smaller steps that you can complete in shorter periods of time. This will make them feel less overwhelming 
+and more achievable.
+
+Second, consider taking short breaks between work sessions to give your mind a chance to rest and recharge. You might use techniques like the Pomodoro Technique (25 minutes of focused work followed 
+by a 5-minute break).
+
+Third, prioritize your tasks based on urgency and importance. Focus on completing the most urgent or important task first before moving on to less critical ones.
+
+Finally, try to be kind to yourself and recognize that it's normal to feel overwhelmed sometimes. It's okay if every single thing isn't completed by the end of the day as long as you're making 
+progress towards your goals.
+
+Would these steps help you get back to work?
+
+>>> this might help
+Great! Setting small, manageable goals and taking breaks can definitely make a difference in how productive you feel. Here's a quick summary of what you mentioned:
+
+1. **Set Small Goals**: Break down tasks into smaller, more achievable steps.
+2. **Take Short Breaks**: Use techniques like the Pomodoro Technique to rest your mind between work sessions.
+3. **Prioritize Tasks**: Focus on urgent and important tasks first.
+4. **Be Kind to Yourself**: Acknowledge that feeling overwhelmed is normal and recognize progress.
+
+Now, how does that make you feel about getting back to work? Does this help clarify things for you?
+
+>>> yea, kind of
+That's a good start! If you're ready, we can chat more about it or if there's anything else on your mind.
+
+>>> right now, I only have the productivity problem
+Got it. Let's focus on productivity then. Have you tried any specific techniques in the past to improve your productivity? What worked well for you, and what didn't work as expected? Understanding 
+what has worked before can help tailor better strategies moving forward.
+
+>>> keeping my mind focussed have helped me alot
+That sounds like a great strategy! Keeping your mind focused is key to improving productivity. Let's try to expand on that:
+
+1. **Focus Techniques**: What specific techniques do you use to keep yourself focused? Do you find that certain environments or conditions help more than others?
+
+2. **Eliminate Distractions**: Are there any common distractions you can identify and maybe reduce? For example, turning off notifications or finding a quiet workspace.
+
+3. **Mindfulness and Breaks**: Sometimes taking short mindfulness breaks can actually improve focus for the longer sessions. Have you tried that before?
+
+4. **Tools and Apps**: Are there any productivity tools or apps you use to stay on track? Things like to-do lists, timers, or focus apps might help.
+
+Would you like to dive deeper into any of these strategies, or is there another aspect of productivity you want to explore?
+
+>>> these works better.
+Great! It's good to know that focusing techniques have worked well for you in the past. Let's refine and expand on those:
+
+1. **Create a Focused Workspace**: Make sure your workspace is conducive to concentration. This might mean having a clean desk, minimizing clutter, or finding a quiet place where you can work without 
+interruptions.
+
+2. **Use Time Blocking**: Allocate specific time slots for different tasks throughout your day. This can help you stay focused and avoid multitasking, which often decreases productivity.
+
+3. **Take Regular Breaks**: Even short breaks can refresh your mind. Consider techniques like the Pomodoro Technique (work for 25 minutes, then take a 5-minute break).
+
+4. **Set Clear Goals**: Before starting work each day or session, set clear goals for what you want to achieve. This helps keep you motivated and focused on the task at hand.
+
+5. **Eliminate Distractions**: Identify common distractions and try to minimize them. This could mean turning off notifications, using website blockers, or finding a quiet place to work.
+
+6. **Mindfulness Practices**: Incorporating short mindfulness practices can improve your ability to focus. Even taking a few deep breaths or practicing meditation for 5-10 minutes before starting 
+work can make a difference.
+
+Would any of these strategies help you further improve your productivity?
+
+>>> yea, I will try those.
+That sounds like a solid plan! Let's summarize what we've discussed so far:
+
+### Productivity Strategies:
+1. **Create a Focused Workspace**: Ensure your workspace is clean and free from distractions.
+2. **Use Time Blocking**: Allocate specific time slots for different tasks to avoid multitasking.
+3. **Take Regular Breaks**: Use techniques like the Pomodoro Technique (25 minutes work, 5 minutes break).
+4. **Set Clear Goals**: Define what you want to achieve before starting each work session.
+5. **Eliminate Distractions**: Turn off notifications and minimize interruptions.
+6. **Mindfulness Practices**: Incorporate short mindfulness breaks or deep breathing exercises.
+
+### Next Steps:
+1. Try implementing these strategies one by one over the next few days.
+2. Pay attention to what works best for you and adjust as needed.
+
+Would you like any further assistance with these strategies, or is there anything else on your mind?
+
+"""
+
+# Format the conversation
+formatted_conversation = format_conversation(raw_conversation)
+
+# Print the formatted conversation (optional)
+print("Formatted Conversation:")
+print(formatted_conversation)
+print("\n" + "-"*50 + "\n")
+
+# Evaluate the conversation
+print("Sending to API for evaluation...")
+evaluation_result = evaluate_conversation(formatted_conversation)
+
+# Print the results
+if evaluation_result:
+    print("\nEvaluation Results:")
+    print(json.dumps(evaluation_result, indent=2))
+    
+    # Print a summary of the scores
+    print("\nCBT Dimension Scores:")
+    for dimension, score in evaluation_result["target_scores"].items():
+        print(f"  {dimension}: {score}")
+
+if __name__ == "__main__":
+    # If you want to evaluate a different conversation, you can replace raw_conversation
+    # with your own conversation text or load it from a file
+    pass

RegressionEvaluation/step1_data_formation.py

@@ -0,0 +1,208 @@
+import json                                                                                                                                           
+import os                                                                                                                                             
+import glob                                                                                                                                           
+import re                                                                                                                                             
+from collections import defaultdict                                                                                                                   
+import pandas as pd                                                                                                                                   
+from tqdm import tqdm     
+
+# Define cognitive distortion patterns to identify initial problems                                                                                   
+COGNITIVE_DISTORTIONS = {                                                                                                                             
+    "perfectionism": ["perfect", "flawless", "mistake", "error", "failure"],                                                                          
+    "catastrophizing": ["disaster", "terrible", "horrible", "awful", "end of the world"],                                                             
+    "black_and_white_thinking": ["always", "never", "completely", "totally", "all or nothing"],                                                       
+    "mind_reading": ["think about me", "judging me", "think I'm", "perceive me"],                                                                     
+    "overgeneralization": ["everything", "nothing", "everyone", "no one", "always goes wrong"],                                                       
+    "should_statements": ["should", "must", "have to", "ought to"],                                                                                   
+    "emotional_reasoning": ["feel like a failure", "feel worthless", "feel inadequate"],                                                              
+    "personalization": ["my fault", "blame myself", "responsible for", "caused this"]                                                                 
+}                
+
+def identify_initial_problem(initial_input):                                                                                                          
+    """Identify the likely cognitive distortion from the initial input."""                                                                            
+    initial_input = initial_input.lower()                                                                                                             
+                                                                                                                                                      
+    # Count matches for each distortion type                                                                                                          
+    matches = defaultdict(int)                                                                                                                        
+    for distortion, patterns in COGNITIVE_DISTORTIONS.items():                                                                                        
+        for pattern in patterns:                                                                                                                      
+            if pattern.lower() in initial_input:                                                                                                      
+                matches[distortion] += 1                                                                                                              
+                                                                                                                                                      
+    # Return the distortion with the most matches, or "unspecified" if none                                                                           
+    if matches:                                                                                                                                       
+        return max(matches.items(), key=lambda x: x[1])[0]                                                                                            
+    return "unspecified"     
+
+def clean_therapist_response(content):                                                                                                                
+    """Clean the therapist response by removing reasoning tags."""                                                                                    
+    # Remove reasoning tags                                                                                                                           
+    content = re.sub(r'<reasoning>.*?</reasoning>', '', content, flags=re.DOTALL)                                                                     
+    # Extract answer content                                                                                                                          
+    content = re.sub(r'<answer>(.*?)</answer>', r'\1', content, flags=re.DOTALL)                                                                      
+    return content.strip()                                                                                                                            
+
+def format_conversation(turns):                                                                                                                       
+    """Format conversation turns into a readable text format."""                                                                                      
+    formatted_text = ""                                                                                                                               
+    for turn in turns:                                                                                                                                
+        speaker = "User" if turn["speaker"] == "patient" else "Assistant"                                                                             
+        content = turn["content"]                                                                                                                     
+                                                                                                                                                      
+        # Clean the content for assistant responses                                                                                                   
+        if speaker == "Assistant":                                                                                                                    
+            content = clean_therapist_response(content)                                                                                               
+                                                                                                                                                      
+        formatted_text += f"{speaker}: {content}\n\n"                                                                                                 
+                                                                                                                                                      
+    return formatted_text.strip()  
+
+def transform_conversation_to_training_examples(json_file_path):                                                                                      
+    """Transform a conversation JSON file into multiple training examples."""                                                                         
+    try:                                                                                                                                              
+        with open(json_file_path, 'r', encoding='utf-8') as f:                                                                                        
+            conversation_data = json.load(f)                                                                                                          
+                                                                                                                                                      
+        # Extract basic information                                                                                                                   
+        conversation_id = os.path.basename(json_file_path).replace('.json', '')                                                                       
+                                                                                                                                                      
+        # Check if the required fields exist                                                                                                          
+        if "metadata" not in conversation_data or "conversation" not in conversation_data or "progress_areas" not in conversation_data:               
+            print(f"Warning: Missing required fields in {json_file_path}")                                                                            
+            return []                                                                                                                                 
+                                                                                                                                                      
+        initial_input = conversation_data["metadata"]["initial_input"]                                                                                
+        total_turns = len(conversation_data["conversation"])                                                                                          
+                                                                                                                                                      
+        # Extract reward scores                                                                                                                       
+        try:                                                                                                                                          
+            target_scores = {                                                                                                                         
+                "thought_identification": float(conversation_data["progress_areas"]["Thought Identification"]),                                       
+                "emotion_exploration": float(conversation_data["progress_areas"]["Emotion Exploration"]),                                             
+                "distortion_identification": float(conversation_data["progress_areas"]["Distortion Identification"]),                                 
+                "evidence_gathering": float(conversation_data["progress_areas"]["Evidence Gathering"]),                                               
+                "balanced_thinking": float(conversation_data["progress_areas"]["Balanced Thinking"])                                                  
+            }                                                                                                                                         
+        except (KeyError, ValueError) as e:                                                                                                           
+            print(f"Warning: Issue with progress scores in {json_file_path}: {e}")                                                                    
+            return []                                                                                                                                 
+                                                                                                                                                      
+        # Identify the initial problem/cognitive distortion                                                                                           
+        initial_problem = identify_initial_problem(initial_input)                                                                                     
+                                                                                                                                                      
+        # Create training examples at different conversation lengths                                                                                  
+        training_examples = []                                                                                                                        
+                                                                                                                                                      
+        # Process the conversation at different points (every 2 turns)                                                                                
+        for end_turn in range(2, total_turns + 1, 2):                                                                                                 
+            # Extract conversation up to this point                                                                                                   
+            conversation_so_far = conversation_data["conversation"][:end_turn]                                                                        
+                                                                                                                                                      
+            # Format the conversation text                                                                                                            
+            formatted_text = format_conversation(conversation_so_far)                                                                                 
+                                                                                                                                                      
+            # Calculate conversation progress                                                                                                         
+            conversation_progress = round(end_turn / total_turns, 2)                                                                                  
+                                                                                                                                                      
+            # Create the training example                                                                                                             
+            example = {                                                                                                                               
+                "id": f"{conversation_id}_turn_{end_turn}",                                                                                           
+                "conversation_text": formatted_text,                                                                                                  
+                "target_scores": target_scores,                                                                                                       
+                "metadata": {                                                                                                                         
+                    "total_turns": total_turns,                                                                                                       
+                    "turns_included": end_turn,                                                                                                       
+                    "initial_problem": initial_problem,                                                                                               
+                    "conversation_progress": conversation_progress                                                                                    
+                }                                                                                                                                     
+            }                                                                                                                                         
+                                                                                                                                                      
+            training_examples.append(example)                                                                                                         
+                                                                                                                                                      
+        return training_examples                                                                                                                      
+                                                                                                                                                      
+    except Exception as e:                                                                                                                            
+        print(f"Error processing {json_file_path}: {e}")                                                                                              
+        return []            
+
+def process_all_conversations(conversations_dir, output_file):                                                                                        
+    """Process all conversation files and save as a single JSON file."""                                                                              
+    all_examples = []                                                                                                                                 
+                                                                                                                                                      
+    # Get all JSON files in the directory                                                                                                             
+    json_files = glob.glob(os.path.join(conversations_dir, "*.json"))                                                                                 
+    print(f"Found {len(json_files)} conversation files")                                                                                              
+                                                                                                                                                      
+    # Process each file with a progress bar                                                                                                           
+    for json_file in tqdm(json_files, desc="Processing conversations"):                                                                               
+        examples = transform_conversation_to_training_examples(json_file)                                                                             
+        all_examples.extend(examples)                                                                                                                 
+                                                                                                                                                      
+    print(f"Generated {len(all_examples)} training examples from {len(json_files)} conversations")                                                    
+                                                                                                                                                      
+    # Save all examples to a single file                                                                                                              
+    with open(output_file, 'w', encoding='utf-8') as f:                                                                                               
+        json.dump(all_examples, f, indent=2)                                                                                                          
+                                                                                                                                                      
+    print(f"Saved training examples to {output_file}")                                                                                                
+                                                                                                                                                      
+    # Also save a flattened version for easier inspection                                                                                             
+    flattened_examples = []                                                                                                                           
+    for example in all_examples:                                                                                                                      
+        flat_example = {                                                                                                                              
+            "id": example["id"],                                                                                                                      
+            "conversation_text": example["conversation_text"],                                                                                        
+            "thought_identification": example["target_scores"]["thought_identification"],                                                             
+            "emotion_exploration": example["target_scores"]["emotion_exploration"],                                                                   
+            "distortion_identification": example["target_scores"]["distortion_identification"],                                                       
+            "evidence_gathering": example["target_scores"]["evidence_gathering"],                                                                     
+            "balanced_thinking": example["target_scores"]["balanced_thinking"],                                                                       
+            "total_turns": example["metadata"]["total_turns"],                                                                                        
+            "turns_included": example["metadata"]["turns_included"],                                                                                  
+            "initial_problem": example["metadata"]["initial_problem"],                                                                                
+            "conversation_progress": example["metadata"]["conversation_progress"]                                                                     
+        }                                                                                                                                             
+        flattened_examples.append(flat_example)                                                                                                       
+                                                                                                                                                      
+    # Save as CSV for easy viewing                                                                                                                    
+    df = pd.DataFrame(flattened_examples)                                                                                                             
+    csv_output = output_file.replace('.json', '.csv')                                                                                                 
+    df.to_csv(csv_output, index=False)                                                                                                                
+    print(f"Saved flattened examples to {csv_output} for easy inspection")                                                                            
+                                                                                                                                                      
+    return all_examples             
+
+
+# Example usage                                                                                                                                       
+if __name__ == "__main__":                                                                                                                            
+    # Set your conversations directory and output file                                                                                                
+    conversations_dir = "/Users/saitejagudidevini/Documents/Dev/grpo_trainer/Evaluations/Gemini_QA_Testing/conversations"                                                                                 
+    output_file = "cbt_evaluator_training_data.json"                                                                                                  
+                                                                                                                                                      
+    # Process all conversations                                                                                                                       
+    examples = process_all_conversations(conversations_dir, output_file)                                                                              
+                                                                                                                                                      
+    # Print some statistics                                                                                                                           
+    if examples:                                                                                                                                      
+        print("\nDataset Statistics:")                                                                                                                
+        print(f"Total examples: {len(examples)}")                                                                                                     
+                                                                                                                                                      
+        # Count examples by initial problem                                                                                                           
+        problem_counts = {}                                                                                                                           
+        for example in examples:                                                                                                                      
+            problem = example["metadata"]["initial_problem"]                                                                                          
+            problem_counts[problem] = problem_counts.get(problem, 0) + 1                                                                              
+                                                                                                                                                      
+        print("\nExamples by initial problem:")                                                                                                       
+        for problem, count in sorted(problem_counts.items(), key=lambda x: x[1], reverse=True):                                                       
+            print(f"  {problem}: {count}")                                                                                                            
+                                                                                                                                                      
+        # Show distribution of conversation lengths                                                                                                   
+        turn_counts = {}                                                                                                                              
+        for example in examples:                                                                                                                      
+            turns = example["metadata"]["turns_included"]                                                                                             
+            turn_counts[turns] = turn_counts.get(turns, 0) + 1                                                                                        
+                                                                                                                                                      
+        print("\nExamples by conversation length (turns):")                                                                                           
+        for turns, count in sorted(turn_counts.items()):                                                                                              
+            print(f"  {turns} turns: {count}")      

RegressionEvaluation/step2_datasetformodeltraining.py

@@ -0,0 +1,173 @@
+import json                                                                                                                           
+import torch                                                                                                                          
+import numpy as np                                                                                                                    
+from sklearn.model_selection import train_test_split                                                                                  
+from transformers import AutoTokenizer                                                                                                
+from datasets import Dataset, DatasetDict                                                                                             
+import pandas as pd                                                                                                                   
+from tqdm import tqdm                                                                                                                 
+import os                                                                                                                             
+                                                                                                                                      
+def prepare_cbt_datasets(input_file, model_name="roberta-base", test_size=0.15, val_size=0.15, max_length=512, seed=42):              
+    """                                                                                                                               
+    Prepare datasets for training a CBT evaluator model.                                                                              
+                                                                                                                                      
+    Args:                                                                                                                             
+        input_file: Path to the JSON file containing the training examples                                                            
+        model_name: Name of the pre-trained model to use                                                                              
+        test_size: Fraction of data to use for testing                                                                                
+        val_size: Fraction of data to use for validation                                                                              
+        max_length: Maximum sequence length for tokenization                                                                          
+        seed: Random seed for reproducibility                                                                                         
+                                                                                                                                      
+    Returns:                                                                                                                          
+        A DatasetDict containing train, validation, and test datasets                                                                 
+    """                                                                                                                               
+    print(f"Loading data from {input_file}...")                                                                                       
+                                                                                                                                      
+    # Load the data                                                                                                                   
+    with open(input_file, 'r', encoding='utf-8') as f:                                                                                
+        examples = json.load(f)                                                                                                       
+                                                                                                                                      
+    print(f"Loaded {len(examples)} examples")                                                                                         
+                                                                                                                                      
+    # Flatten the examples for easier processing                                                                                      
+    flattened_examples = []                                                                                                           
+    for example in examples:                                                                                                          
+        flat_example = {                                                                                                              
+            "id": example["id"],                                                                                                      
+            "conversation_text": example["conversation_text"],                                                                        
+            "thought_identification": example["target_scores"]["thought_identification"],                                             
+            "emotion_exploration": example["target_scores"]["emotion_exploration"],                                                   
+            "distortion_identification": example["target_scores"]["distortion_identification"],                                       
+            "evidence_gathering": example["target_scores"]["evidence_gathering"],                                                     
+            "balanced_thinking": example["target_scores"]["balanced_thinking"],                                                       
+            "total_turns": example["metadata"]["total_turns"],                                                                        
+            "turns_included": example["metadata"]["turns_included"],                                                                  
+            "initial_problem": example["metadata"]["initial_problem"],                                                                
+            "conversation_progress": example["metadata"]["conversation_progress"]                                                     
+        }                                                                                                                             
+        flattened_examples.append(flat_example)                                                                                       
+                                                                                                                                      
+    # Convert to DataFrame for easier manipulation                                                                                    
+    df = pd.DataFrame(flattened_examples)                                                                                             
+                                                                                                                                      
+    # Print dataset statistics                                                                                                        
+    print("\nDataset Statistics:")                                                                                                    
+    print(f"Number of examples: {len(df)}")                                                                                           
+    print(f"Number of unique conversations: {df['id'].str.split('_turn_').str[0].nunique()}")                                         
+    print(f"Average conversation length: {df['conversation_text'].str.len().mean():.1f} characters")                                  
+                                                                                                                                      
+    print("\nTarget Score Distributions:")                                                                                            
+    for col in ["thought_identification", "emotion_exploration", "distortion_identification",                                         
+                "evidence_gathering", "balanced_thinking"]:                                                                           
+        print(f"  {col}: mean={df[col].mean():.2f}, std={df[col].std():.2f}, min={df[col].min():.2f}, max={df[col].max():.2f}")       
+                                                                                                                                      
+    # Split by conversation ID to prevent data leakage                                                                                
+    # Extract the base conversation ID (without the turn number)                                                                      
+    df['conversation_base_id'] = df['id'].str.split('_turn_').str[0]                                                                  
+                                                                                                                                      
+    # Get unique conversation IDs                                                                                                     
+    unique_conv_ids = df['conversation_base_id'].unique()                                                                             
+                                                                                                                                      
+    # Split conversation IDs into train, validation, and test sets                                                                    
+    train_ids, temp_ids = train_test_split(                                                                                           
+        unique_conv_ids,                                                                                                              
+        test_size=test_size + val_size,                                                                                               
+        random_state=seed                                                                                                             
+    )                                                                                                                                 
+                                                                                                                                      
+    val_ids, test_ids = train_test_split(                                                                                             
+        temp_ids,                                                                                                                     
+        test_size=test_size / (test_size + val_size),                                                                                 
+        random_state=seed                                                                                                             
+    )                                                                                                                                 
+                                                                                                                                      
+    # Create train, validation, and test dataframes                                                                                   
+    train_df = df[df['conversation_base_id'].isin(train_ids)]                                                                         
+    val_df = df[df['conversation_base_id'].isin(val_ids)]                                                                             
+    test_df = df[df['conversation_base_id'].isin(test_ids)]                                                                           
+                                                                                                                                      
+    print(f"\nSplit sizes:")                                                                                                          
+    print(f"  Train: {len(train_df)} examples from {len(train_ids)} conversations")                                                   
+    print(f"  Validation: {len(val_df)} examples from {len(val_ids)} conversations")                                                  
+    print(f"  Test: {len(test_df)} examples from {len(test_ids)} conversations")                                                      
+                                                                                                                                      
+    # Convert to Hugging Face datasets                                                                                                
+    train_dataset = Dataset.from_pandas(train_df)                                                                                     
+    val_dataset = Dataset.from_pandas(val_df)                                                                                         
+    test_dataset = Dataset.from_pandas(test_df)                                                                                       
+                                                                                                                                      
+    # Create a DatasetDict                                                                                                            
+    dataset_dict = DatasetDict({                                                                                                      
+        'train': train_dataset,                                                                                                       
+        'validation': val_dataset,                                                                                                    
+        'test': test_dataset                                                                                                          
+    })                                                                                                                                
+                                                                                                                                      
+    # Load tokenizer                                                                                                                  
+    print(f"\nLoading tokenizer for {model_name}...")                                                                                 
+    tokenizer = AutoTokenizer.from_pretrained(model_name)                                                                             
+                                                                                                                                      
+    # Define tokenization function                                                                                                    
+    def tokenize_function(examples):                                                                                                  
+        return tokenizer(                                                                                                             
+            examples["conversation_text"],                                                                                            
+            padding="max_length",                                                                                                     
+            truncation=True,                                                                                                          
+            max_length=max_length                                                                                                     
+        )                                                                                                                             
+                                                                                                                                      
+    # Tokenize the datasets                                                                                                           
+    print("Tokenizing datasets...")                                                                                                   
+    tokenized_datasets = dataset_dict.map(                                                                                            
+        tokenize_function,                                                                                                            
+        batched=True,                                                                                                                 
+        desc="Tokenizing",                                                                                                            
+        remove_columns=['conversation_base_id']  # Remove the helper column we added                                                  
+    )                                                                                                                                 
+                                                                                                                                      
+    # Save the processed datasets                                                                                                     
+    output_dir = "cbt_evaluator_data"                                                                                                 
+    os.makedirs(output_dir, exist_ok=True)                                                                                            
+                                                                                                                                      
+    print(f"\nSaving processed datasets to {output_dir}...")                                                                          
+    tokenized_datasets.save_to_disk(output_dir)                                                                                       
+                                                                                                                                      
+    # Also save the tokenizer for consistency                                                                                         
+    tokenizer.save_pretrained(os.path.join(output_dir, "tokenizer"))                                                                  
+                                                                                                                                      
+    print("Dataset preparation complete!")                                                                                            
+    return tokenized_datasets                                                                                                         
+                                                                                                                                      
+# Example usage                                                                                                                       
+if __name__ == "__main__":                                                                                                            
+    # Set your input file (output from Step 1)                                                                                        
+    input_file = "cbt_evaluator_training_data.json"                                                                                   
+                                                                                                                                      
+    datasets = prepare_cbt_datasets(                                                                                                               
+        input_file=input_file,                                                                                                                     
+        model_name="roberta-base",  # You can change this to another model if preferred                                                            
+        test_size=0.15,                                                                                                                            
+        val_size=0.15,                                                                                                                             
+        max_length=512,                                                                                                                            
+        seed=42                                                                                                                                    
+    )                                                                                                                                              
+                                                                                                                                                   
+    # Print dataset sizes                                                                                                                          
+    print("\nFinal dataset sizes:")                                                                                                                
+    for split, dataset in datasets.items():                                                                                                        
+        print(f"  {split}: {len(dataset)} examples")                                                                                               
+                                                                                                                                                   
+    # Print example from training set                                                                                                              
+    print("\nExample from training set:")                                                                                                          
+    example = datasets["train"][0]                                                                                                                 
+    print(f"ID: {example['id']}")                                                                                                                  
+    print(f"Input IDs shape: {len(example['input_ids'])}")                                                                                         
+    print(f"Target scores:")                                                                                                                       
+    print(f"  Thought identification: {example['thought_identification']}")                                                                        
+    print(f"  Emotion exploration: {example['emotion_exploration']}")                                                                              
+    print(f"  Distortion identification: {example['distortion_identification']}")                                                                  
+    print(f"  Evidence gathering: {example['evidence_gathering']}")                                                                                
+    print(f"  Balanced thinking: {example['balanced_thinking']}")                                                                                  
+                                                                         

RegressionEvaluation/step3_trainingthe_cbt_evaluator_model.py

@@ -0,0 +1,16 @@
+import os                                                                                                                           
+import torch                                                                                                                        
+import numpy as np                                                                                                                  
+from transformers import (                                                                                                          
+    AutoModelForSequenceClassification,                                                                                             
+    AutoTokenizer,                                                                                                                  
+    TrainingArguments,                                                                                                              
+    Trainer,                                                                                                                        
+    EarlyStoppingCallback                                                                                                           
+)                                                                                                                                   
+from datasets import load_from_disk                                                                                                 
+import evaluate                                                                                                                     
+from sklearn.metrics import mean_squared_error, r2_score                                                                            
+import matplotlib.pyplot as plt                                                                                                     
+import pandas as pd                                                                                                                 
+from tqdm import tqdm     

RegressionEvaluation/step4training.py

@@ -0,0 +1,379 @@
+import json
+import os
+import glob
+import re
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.multioutput import MultiOutputRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error, r2_score
+import pickle
+import matplotlib.pyplot as plt
+from collections import defaultdict
+import joblib
+from flask import Flask, request, jsonify
+import uuid
+from datetime import datetime
+
+# Define cognitive distortion patterns
+COGNITIVE_DISTORTIONS = {
+    "perfectionism": ["perfect", "flawless", "mistake", "error", "failure"],
+    "catastrophizing": ["disaster", "terrible", "horrible", "awful", "end of the world"],
+    "black_and_white_thinking": ["always", "never", "completely", "totally", "all or nothing"],
+    "mind_reading": ["think about me", "judging me", "think I'm", "perceive me"],
+    "overgeneralization": ["everything", "nothing", "everyone", "no one", "always goes wrong"],
+    "should_statements": ["should", "must", "have to", "ought to"],
+    "emotional_reasoning": ["feel like a failure", "feel worthless", "feel inadequate"],
+    "personalization": ["my fault", "blame myself", "responsible for", "caused this"]
+}
+
+class CBTEvaluatorSimple:
+    def __init__(self, output_dir="cbt_evaluator_simple", max_features=5000, random_state=42):
+        self.output_dir = output_dir
+        self.max_features = max_features
+        self.random_state = random_state
+        
+        # Create output directory
+        os.makedirs(output_dir, exist_ok=True)
+        
+        # Initialize feature extractor
+        self.vectorizer = TfidfVectorizer(
+            max_features=max_features,
+            ngram_range=(1, 2),
+            stop_words='english'
+        )
+        
+        # Initialize model
+        self.model = MultiOutputRegressor(
+            GradientBoostingRegressor(
+                n_estimators=100,
+                learning_rate=0.1,
+                max_depth=5,
+                random_state=random_state
+            )
+        )
+    
+    def identify_initial_problem(self, initial_input):
+        """Identify the likely cognitive distortion from the initial input."""
+        initial_input = initial_input.lower()
+        
+        # Count matches for each distortion type
+        matches = defaultdict(int)
+        for distortion, patterns in COGNITIVE_DISTORTIONS.items():
+            for pattern in patterns:
+                if pattern.lower() in initial_input:
+                    matches[distortion] += 1
+        
+        # Return the distortion with the most matches, or "unspecified" if none
+        if matches:
+            return max(matches.items(), key=lambda x: x[1])[0]
+        return "unspecified"
+    
+    def clean_therapist_response(self, content):
+        """Clean the therapist response by removing reasoning tags."""
+        # Remove reasoning tags
+        content = re.sub(r'<reasoning>.*?</reasoning>', '', content, flags=re.DOTALL)
+        # Extract answer content
+        content = re.sub(r'<answer>(.*?)</answer>', r'\1', content, flags=re.DOTALL)
+        return content.strip()
+    
+    def format_conversation(self, turns):
+        """Format conversation turns into a readable text format."""
+        formatted_text = ""
+        for turn in turns:
+            speaker = "User" if turn["speaker"] == "patient" else "Assistant"
+            content = turn["content"]
+            
+            # Clean the content for assistant responses
+            if speaker == "Assistant":
+                content = self.clean_therapist_response(content)
+            
+            formatted_text += f"{speaker}: {content}\n\n"
+        
+        return formatted_text.strip()
+    
+    def process_conversations(self, conversations_dir):
+        """Process all conversation files into a dataset."""
+        all_examples = []
+        
+        # Get all JSON files in the directory
+        json_files = glob.glob(os.path.join(conversations_dir, "*.json"))
+        print(f"Found {len(json_files)} conversation files")
+        
+        # Process each file with a progress bar
+        for json_file in tqdm(json_files, desc="Processing conversations"):
+            try:
+                with open(json_file, 'r', encoding='utf-8') as f:
+                    conversation_data = json.load(f)
+                
+                # Extract basic information
+                conversation_id = os.path.basename(json_file).replace('.json', '')
+                
+                # Check if the required fields exist
+                if "metadata" not in conversation_data or "conversation" not in conversation_data or "progress_areas" not in conversation_data:
+                    print(f"Warning: Missing required fields in {json_file}")
+                    continue
+                
+                initial_input = conversation_data["metadata"]["initial_input"]
+                total_turns = len(conversation_data["conversation"])
+                
+                # Extract reward scores
+                try:
+                    target_scores = {
+                        "thought_identification": float(conversation_data["progress_areas"]["Thought Identification"]),
+                        "emotion_exploration": float(conversation_data["progress_areas"]["Emotion Exploration"]),
+                        "distortion_identification": float(conversation_data["progress_areas"]["Distortion Identification"]),
+                        "evidence_gathering": float(conversation_data["progress_areas"]["Evidence Gathering"]),
+                        "balanced_thinking": float(conversation_data["progress_areas"]["Balanced Thinking"])
+                    }
+                except (KeyError, ValueError) as e:
+                    print(f"Warning: Issue with progress scores in {json_file}: {e}")
+                    continue
+                
+                # Identify the initial problem/cognitive distortion
+                initial_problem = self.identify_initial_problem(initial_input)
+                
+                # Create training examples at different conversation lengths
+                for end_turn in range(2, total_turns + 1, 2):
+                    # Extract conversation up to this point
+                    conversation_so_far = conversation_data["conversation"][:end_turn]
+                    
+                    # Format the conversation text
+                    formatted_text = self.format_conversation(conversation_so_far)
+                    
+                    # Calculate conversation progress
+                    conversation_progress = round(end_turn / total_turns, 2)
+                    
+                    # Create the training example
+                    example = {
+                        "id": f"{conversation_id}_turn_{end_turn}",
+                        "conversation_text": formatted_text,
+                        "thought_identification": target_scores["thought_identification"],
+                        "emotion_exploration": target_scores["emotion_exploration"],
+                        "distortion_identification": target_scores["distortion_identification"],
+                        "evidence_gathering": target_scores["evidence_gathering"],
+                        "balanced_thinking": target_scores["balanced_thinking"],
+                        "total_turns": total_turns,
+                        "turns_included": end_turn,
+                        "initial_problem": initial_problem,
+                        "conversation_progress": conversation_progress
+                    }
+                    
+                    all_examples.append(example)
+            
+            except Exception as e:
+                print(f"Error processing {json_file}: {e}")
+        
+        print(f"Generated {len(all_examples)} training examples from {len(json_files)} conversations")
+        return pd.DataFrame(all_examples)
+    
+    def train(self, conversations_dir, test_size=0.2):
+        """Train the model on conversation data."""
+        # Process conversations
+        df = self.process_conversations(conversations_dir)
+        
+        # Save the processed data
+        df.to_csv(os.path.join(self.output_dir, "processed_data.csv"), index=False)
+        
+        # Split into features and targets
+        X = df["conversation_text"]
+        y = df[["thought_identification", "emotion_exploration", "distortion_identification", 
+                "evidence_gathering", "balanced_thinking"]].values
+        
+        # Split into train and test sets
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, test_size=test_size, random_state=self.random_state
+        )
+        
+        print(f"Training set size: {len(X_train)}")
+        print(f"Test set size: {len(X_test)}")
+        
+        # Fit the vectorizer
+        print("Fitting vectorizer...")
+        X_train_vec = self.vectorizer.fit_transform(X_train)
+        
+        # Train the model
+        print("Training model...")
+        self.model.fit(X_train_vec, y_train)
+        
+        # Evaluate on test set
+        print("Evaluating on test set...")
+        X_test_vec = self.vectorizer.transform(X_test)
+        y_pred = self.model.predict(X_test_vec)
+        
+        # Calculate metrics
+        mse = mean_squared_error(y_test, y_pred)
+        r2 = r2_score(y_test, y_pred)
+        
+        print(f"Test MSE: {mse:.4f}")
+        print(f"Test R²: {r2:.4f}")
+        
+        # Calculate metrics for each dimension
+        dimension_names = ["thought_identification", "emotion_exploration", "distortion_identification", 
+                          "evidence_gathering", "balanced_thinking"]
+        
+        dimension_metrics = {}
+        for i, name in enumerate(dimension_names):
+            dim_mse = mean_squared_error(y_test[:, i], y_pred[:, i])
+            dim_r2 = r2_score(y_test[:, i], y_pred[:, i])
+            dimension_metrics[name] = {"mse": dim_mse, "r2": dim_r2}
+            print(f"  {name}: MSE = {dim_mse:.4f}, R² = {dim_r2:.4f}")
+        
+        # Save the model and vectorizer
+        print(f"Saving model to {self.output_dir}...")
+        joblib.dump(self.vectorizer, os.path.join(self.output_dir, "vectorizer.joblib"))
+        joblib.dump(self.model, os.path.join(self.output_dir, "model.joblib"))
+        
+        # Save the metrics
+        with open(os.path.join(self.output_dir, "metrics.json"), "w") as f:
+            json.dump({
+                "overall": {"mse": mse, "r2": r2},
+                "dimensions": dimension_metrics
+            }, f, indent=2)
+        
+        # Visualize predictions vs actual values
+        self.visualize_predictions(y_test, y_pred, dimension_names)
+        
+        return mse, r2, dimension_metrics
+    
+    def visualize_predictions(self, y_true, y_pred, dimension_names):
+        """Visualize predictions vs actual values."""
+        fig, axes = plt.subplots(2, 3, figsize=(15, 10))
+        axes = axes.flatten()
+        
+        for i, name in enumerate(dimension_names):
+            ax = axes[i]
+            ax.scatter(y_true[:, i], y_pred[:, i], alpha=0.5)
+            ax.plot([0, 1], [0, 1], 'r--')  # Diagonal line
+            ax.set_xlabel("Actual")
+            ax.set_ylabel("Predicted")
+            ax.set_title(name)
+            ax.set_xlim(0, 1)
+            ax.set_ylim(0, 1)
+            
+            # Add R² value
+            r2 = r2_score(y_true[:, i], y_pred[:, i])
+            ax.text(0.05, 0.95, f"R² = {r2:.3f}", transform=ax.transAxes)
+        
+        # Remove the unused subplot
+        fig.delaxes(axes[5])
+        
+        # Add overall title
+        fig.suptitle("Predicted vs Actual Values", fontsize=16)
+        fig.tight_layout()
+        
+        # Save the figure
+        fig.savefig(os.path.join(self.output_dir, "predictions_vs_actual.png"))
+        print(f"Visualization saved to {os.path.join(self.output_dir, 'predictions_vs_actual.png')}")
+    
+    def predict(self, conversation_text):
+        """Predict CBT scores for a conversation."""
+        # Transform the text
+        X = self.vectorizer.transform([conversation_text])
+        
+        # Make predictions
+        predictions = self.model.predict(X)[0]
+        
+        # Ensure predictions are in the range [0, 1]
+        predictions = np.clip(predictions, 0, 1)
+        
+        # Format the predictions
+        dimension_names = ["thought_identification", "emotion_exploration", "distortion_identification", 
+                          "evidence_gathering", "balanced_thinking"]
+        
+        return {name: float(pred) for name, pred in zip(dimension_names, predictions)}
+    
+    def count_turns(self, conversation_text):
+        """Count the number of turns in the conversation."""
+        # Count occurrences of "User:" and "Assistant:"
+        user_turns = len(re.findall(r"User:", conversation_text))
+        assistant_turns = len(re.findall(r"Assistant:", conversation_text))
+        return user_turns + assistant_turns
+    
+    def identify_problem_from_text(self, conversation_text):
+        """Identify the likely cognitive distortion from the conversation text."""
+        # Extract the first user message
+        match = re.search(r"User: (.*?)(?:\n\n|$)", conversation_text)
+        if not match:
+            return "unspecified"
+        
+        initial_input = match.group(1)
+        return self.identify_initial_problem(initial_input)
+
+# Create a Flask API for the model
+def create_flask_app(model_dir="cbt_evaluator_simple"):
+    app = Flask(__name__)
+    
+    # Load the model and vectorizer
+    vectorizer = joblib.load(os.path.join(model_dir, "vectorizer.joblib"))
+    model = joblib.load(os.path.join(model_dir, "model.joblib"))
+    
+    # Create an evaluator instance
+    evaluator = CBTEvaluatorSimple()
+    evaluator.vectorizer = vectorizer
+    evaluator.model = model
+    
+    @app.route('/evaluate', methods=['POST'])
+    def evaluate_conversation():
+        """Evaluate a conversation."""
+        # Get conversation from request
+        data = request.json
+        conversation_text = data.get('conversation', '')
+        
+        if not conversation_text:
+            return jsonify({"error": "No conversation provided"}), 400
+        
+        # Get predictions
+        scores = evaluator.predict(conversation_text)
+        
+        # Count turns and identify initial problem
+        total_turns = evaluator.count_turns(conversation_text)
+        initial_problem = evaluator.identify_problem_from_text(conversation_text)
+        
+        # Create the final response
+        result = {
+            "id": f"eval_{uuid.uuid4().hex[:8]}_{datetime.now().strftime('%Y%m%d%H%M%S')}",
+            "conversation_text": conversation_text,
+            "target_scores": {
+                "thought_identification": round(scores["thought_identification"], 2),
+                "emotion_exploration": round(scores["emotion_exploration"], 2),
+                "distortion_identification": round(scores["distortion_identification"], 2),
+                "evidence_gathering": round(scores["evidence_gathering"], 2),
+                "balanced_thinking": round(scores["balanced_thinking"], 2)
+            },
+            "metadata": {
+                "total_turns": total_turns,
+                "turns_included": total_turns,
+                "initial_problem": initial_problem,
+                "conversation_progress": 1.0  # Assuming full conversation
+            }
+        }
+        
+        return jsonify(result)
+    
+    @app.route('/health', methods=['GET'])
+    def health_check():
+        """Health check endpoint."""
+        return jsonify({"status": "healthy"})
+    
+    return app
+
+# Example usage
+if __name__ == "__main__":
+    # Check if we're training or serving
+    import sys
+    
+    if len(sys.argv) > 1 and sys.argv[1] == "serve":
+        # Serve the model
+        app = create_flask_app()
+        app.run(debug=True, host='0.0.0.0', port=5009)
+    else:
+        # Train the model
+        evaluator = CBTEvaluatorSimple()
+        evaluator.train("/Users/saitejagudidevini/Documents/Dev/grpo_trainer/Evaluations/Gemini_QA_Testing/conversations")
+        
+        print("\nTo serve the model, run:")
+        print("python cbt_evaluator_simple.py serve")