Update app.py

app.py

@@ -185,14 +185,12 @@ def push_file_to_huggingface_dataset(file_path, path_in_repo, commit_message_pre
         print(log_message)
         return log_message
 
-# --- Main Generation and Push Function (Now a generator for streaming) ---
+# --- Main Generation and Push Function ---
 def generate_and_display_conversations(num_conversations_input, custom_prompts_input, custom_system_prompt_input,
                                      commit_subject, commit_body, selected_model_name_input):
     num_conversations = int(num_conversations_input)
     if num_conversations <= 0:
-        # Use gr.NoOp() for the output that shouldn't be updated
-        yield gr.NoOp(), "Please enter a number of conversations greater than zero.\n"
-        return
+        return "Please enter a number of conversations greater than zero.", ""
 
     os.makedirs(OUTPUT_DIR, exist_ok=True)
 
@@ -204,47 +202,52 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
                 try:
                     existing_conversations.append(json.loads(line.strip()))
                 except json.JSONDecodeError as e:
-                    # Use gr.NoOp() for the output that shouldn't be updated
-                    yield gr.NoOp(), f"Skipping malformed JSON line in {DATA_FILE}: {line.strip()} - {e}\n"
-                    
+                    print(f"Skipping malformed JSON line in {DATA_FILE}: {line.strip()} - {e}")
+
     # Deduplicate existing conversations
     seen_conversations = set()
     cleaned_existing_conversations = []
     for conv_entry in existing_conversations:
+        # Use a string representation of the whole entry for deduplication
         conv_str = json.dumps(conv_entry, sort_keys=True)
         if conv_str not in seen_conversations:
             cleaned_existing_conversations.append(conv_entry)
             seen_conversations.add(conv_str)
     
     # Validate and filter existing conversations for completeness (expected length)
-    expected_msg_len = lambda n_exchanges: 1 + 1 + n_exchanges * 2 # System + initial human + (AI_turn + human_followup) * n_exchanges
+    expected_msg_len = lambda n_exchanges: 1 + 1 + n_exchanges + (n_exchanges - 1) # System + initial human + AI turns + human follow-ups
     
     validated_existing_conversations = []
     initial_cleaned_count = len(cleaned_existing_conversations)
     for conv_entry in cleaned_existing_conversations:
         conv_list = conv_entry.get("conversations", [])
-        if len(conv_list) == expected_msg_len(5): # Fixed to 5 exchanges for generation
+        # Assume num_exchanges was 5 for old conversations if not stored
+        # Or more robustly, infer from length.
+        # Given the fixed num_exchanges=5 for generation, we can check for this.
+        if len(conv_list) == expected_msg_len(5):
             validated_existing_conversations.append(conv_entry)
         else:
-            # Use gr.NoOp() for the output that shouldn't be updated
-            yield gr.NoOp(), f"Skipping incomplete/malformed existing conversation (length {len(conv_list)} != {expected_msg_len(5)}): {conv_entry}\n"
+            print(f"Skipping incomplete/malformed existing conversation (length {len(conv_list)} != {expected_msg_len(5)}): {conv_entry}")
 
     all_conversations = list(validated_existing_conversations) # Start with clean existing ones
 
+    generation_log = []
     current_time_loc = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " (An Nhơn, Binh Dinh, Vietnam)"
-    # Use gr.NoOp() for the output that shouldn't be updated
-    yield gr.NoOp(), f"Starting conversation generation at {current_time_loc}\n"
-    yield gr.NoOp(), f"Loaded and cleaned {len(validated_existing_conversations)} existing conversations (initially {initial_cleaned_count} before validation).\n"
-    yield gr.NoOp(), f"Generating {num_conversations} *new* conversations.\n"
+    generation_log.append(f"Starting conversation generation at {current_time_loc}")
+    generation_log.append(f"Loaded and cleaned {len(validated_existing_conversations)} existing conversations (initially {initial_cleaned_count} before validation).")
+    generation_log.append(f"Generating {num_conversations} *new* conversations.")
     
     model_names_to_use = list(AVAILABLE_MODELS.keys())
     if selected_model_name_input and selected_model_name_input in model_names_to_use:
-        model_selection_info = f"Specific model selected for all new conversations: '{selected_model_name_input}'\n"
+        # If a specific model is selected, all conversations in this batch will use that model
+        model_selection_info = f"Specific model selected for all new conversations: '{selected_model_name_input}'"
+        # Determine the model to use for this specific conversation
         selected_model_for_this_conv_batch = selected_model_name_input
     else:
-        model_selection_info = f"No specific model selected or invalid. Models will be chosen randomly per conversation from: {', '.join(model_names_to_use)}\n"
+        # If no specific model or invalid model, random models will be picked per conversation
+        model_selection_info = f"No specific model selected or invalid. Models will be chosen randomly per conversation from: {', '.join(model_names_to_use)}"
         selected_model_for_this_conv_batch = None # Indicate random selection per loop
-    yield gr.NoOp(), model_selection_info
+    generation_log.append(model_selection_info)
 
 
     current_prompts = DEFAULT_INITIAL_PROMPTS
@@ -256,7 +259,7 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
     new_conversations_generated = []
     expected_conversation_length = expected_msg_len(5) # Always 5 exchanges for new generations
 
-    for i in range(num_conversations): # tqdm not directly compatible with yielding, so manually handle progress
+    for i in tqdm(range(num_conversations), desc="Generating conversations"):
         seed = random.randint(0, 1000000)
 
         if custom_system_prompt_input:
@@ -268,44 +271,46 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
         prompt_template = random.choice(current_prompts)
         prompt = prompt_template.replace("[NAME]", random_name)
 
+        # Determine the model to use for this specific conversation
         selected_model_for_this_conv = selected_model_for_this_conv_batch if selected_model_for_this_conv_batch else random.choice(model_names_to_use)
             
-        yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Generating conv {i+1}/{num_conversations} with '{selected_model_for_this_conv}' (System: '{system[:50]}...')\n"
+        generation_log.append(f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Generating conv {i+1}/{num_conversations} with '{selected_model_for_this_conv}' (System: '{system[:50]}...')")
         
         conversation = chat(system, prompt, selected_model_for_this_conv, seed=seed, num_exchanges=5)
 
         if len(conversation) == expected_conversation_length and not any(d.get("from") == "error" for d in conversation):
             new_conv_entry = {"model_used": selected_model_for_this_conv, "conversations": conversation}
+            # Add to all_conversations and new_conversations_generated only if not a duplicate of what's already *in memory*
+            # This handles duplicates from current batch or newly generated identical to existing
             new_conv_str = json.dumps(new_conv_entry, sort_keys=True)
             if new_conv_str not in seen_conversations:
                 all_conversations.append(new_conv_entry)
                 new_conversations_generated.append(new_conv_entry)
-                seen_conversations.add(new_conv_str)
-                yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Successfully generated and added conv {i+1}/{num_conversations}.\n"
+                seen_conversations.add(new_conv_str) # Mark as seen
+                generation_log.append(f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Successfully generated and added conv {i+1}/{num_conversations}.")
             else:
-                yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipped conv {i+1}/{num_conversations} as it's a duplicate.\n"
+                generation_log.append(f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipped conv {i+1}/{num_conversations} as it's a duplicate.")
         else:
-            yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipping conv {i+1}/{num_conversations} due to error or incorrect length ({len(conversation)} messages, expected {expected_conversation_length}).\n"
+            generation_log.append(f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipping conv {i+1}/{num_conversations} due to error or incorrect length ({len(conversation)} messages, expected {expected_conversation_length}).")
             if conversation and conversation[-1].get("from") == "error":
-                yield gr.NoOp(), f"  Error details: {conversation[-1]['value']}\n"
+                generation_log.append(f"  Error details: {conversation[-1]['value']}")
 
     # Save all (cleaned existing + newly generated unique) conversations to JSONL
     with open(DATA_FILE, "w") as f:
         for conv in all_conversations:
             f.write(json.dumps(conv) + "\n")
 
-    yield gr.NoOp(), f"Saved {len(new_conversations_generated)} *new unique* conversations to {DATA_FILE} (total unique and validated: {len(all_conversations)}).\n"
-    yield gr.NoOp(), "Attempting to push main conversations file to Hugging Face Dataset...\n"
+    generation_log.append(f"Saved {len(new_conversations_generated)} *new unique* conversations to {DATA_FILE} (total unique and validated: {len(all_conversations)}).")
+    generation_log.append("Attempting to push main conversations file to Hugging Face Dataset...")
 
     # --- Auto-push main conversations to Hugging Face Dataset ---
     # Use the custom commit message
     commit_message = f"{commit_subject.strip()}\n\n{commit_body.strip()}" if commit_body.strip() else commit_subject.strip()
     push_status = push_file_to_huggingface_dataset(DATA_FILE, "conversations.jsonl", commit_message)
-    yield gr.NoOp(), push_status + "\n"
-    final_log_message = f"Process complete at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} (An Nhơn, Binh Dinh, Vietnam)\n"
-    
-    # Final return for both outputs: The first output (JSON) is now updated, the second (log) gets the final message.
-    yield json.dumps(all_conversations, indent=2), final_log_message
+    generation_log.append(push_status)
+    generation_log.append(f"Process complete at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} (An Nhơn, Binh Dinh, Vietnam)")
+
+    return json.dumps(all_conversations, indent=2), "\n".join(generation_log)
 
 # --- Community Prompts Functions ---
 def load_community_prompts():
@@ -513,8 +518,8 @@ with gr.Blocks() as demo:
             generate_button = gr.Button("Generate & Push Conversations")
 
             output_conversations = gr.JSON(label="Generated Conversations (Content of conversations.jsonl)")
-            # Set streaming=True for the output_log Textbox
             output_log = gr.Textbox(label="Process Log", interactive=False, lines=10, max_lines=20)
+
             # Link commit template dropdown to update fields
             commit_template_dropdown.change(
                 fn=update_commit_fields,
@@ -539,7 +544,7 @@ with gr.Blocks() as demo:
                     model_selector_dropdown # Pass selected model name
                 ],
                 outputs=[output_conversations, output_log],
-                # show_progress=True is handled internally by yielding a generator
+                show_progress=True
             )
 
         with gr.Tab("Community Prompts"):