Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -190,7 +190,8 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 190 |
commit_subject, commit_body, selected_model_name_input):
|
| 191 |
num_conversations = int(num_conversations_input)
|
| 192 |
if num_conversations <= 0:
|
| 193 |
-
|
|
|
|
| 194 |
return
|
| 195 |
|
| 196 |
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
|
@@ -203,7 +204,8 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 203 |
try:
|
| 204 |
existing_conversations.append(json.loads(line.strip()))
|
| 205 |
except json.JSONDecodeError as e:
|
| 206 |
-
|
|
|
|
| 207 |
|
| 208 |
# Deduplicate existing conversations
|
| 209 |
seen_conversations = set()
|
|
@@ -224,14 +226,16 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 224 |
if len(conv_list) == expected_msg_len(5): # Fixed to 5 exchanges for generation
|
| 225 |
validated_existing_conversations.append(conv_entry)
|
| 226 |
else:
|
| 227 |
-
|
|
|
|
| 228 |
|
| 229 |
all_conversations = list(validated_existing_conversations) # Start with clean existing ones
|
| 230 |
|
| 231 |
current_time_loc = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " (An Nhơn, Binh Dinh, Vietnam)"
|
| 232 |
-
|
| 233 |
-
yield gr.
|
| 234 |
-
yield gr.
|
|
|
|
| 235 |
|
| 236 |
model_names_to_use = list(AVAILABLE_MODELS.keys())
|
| 237 |
if selected_model_name_input and selected_model_name_input in model_names_to_use:
|
|
@@ -240,7 +244,7 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 240 |
else:
|
| 241 |
model_selection_info = f"No specific model selected or invalid. Models will be chosen randomly per conversation from: {', '.join(model_names_to_use)}\n"
|
| 242 |
selected_model_for_this_conv_batch = None # Indicate random selection per loop
|
| 243 |
-
yield gr.
|
| 244 |
|
| 245 |
|
| 246 |
current_prompts = DEFAULT_INITIAL_PROMPTS
|
|
@@ -266,7 +270,7 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 266 |
|
| 267 |
selected_model_for_this_conv = selected_model_for_this_conv_batch if selected_model_for_this_conv_batch else random.choice(model_names_to_use)
|
| 268 |
|
| 269 |
-
yield gr.
|
| 270 |
|
| 271 |
conversation = chat(system, prompt, selected_model_for_this_conv, seed=seed, num_exchanges=5)
|
| 272 |
|
|
@@ -277,30 +281,30 @@ def generate_and_display_conversations(num_conversations_input, custom_prompts_i
|
|
| 277 |
all_conversations.append(new_conv_entry)
|
| 278 |
new_conversations_generated.append(new_conv_entry)
|
| 279 |
seen_conversations.add(new_conv_str)
|
| 280 |
-
yield gr.
|
| 281 |
else:
|
| 282 |
-
yield gr.
|
| 283 |
else:
|
| 284 |
-
yield gr.
|
| 285 |
if conversation and conversation[-1].get("from") == "error":
|
| 286 |
-
yield gr.
|
| 287 |
|
| 288 |
# Save all (cleaned existing + newly generated unique) conversations to JSONL
|
| 289 |
with open(DATA_FILE, "w") as f:
|
| 290 |
for conv in all_conversations:
|
| 291 |
f.write(json.dumps(conv) + "\n")
|
| 292 |
|
| 293 |
-
yield gr.
|
| 294 |
-
yield gr.
|
| 295 |
|
| 296 |
# --- Auto-push main conversations to Hugging Face Dataset ---
|
| 297 |
# Use the custom commit message
|
| 298 |
commit_message = f"{commit_subject.strip()}\n\n{commit_body.strip()}" if commit_body.strip() else commit_subject.strip()
|
| 299 |
push_status = push_file_to_huggingface_dataset(DATA_FILE, "conversations.jsonl", commit_message)
|
| 300 |
-
yield gr.
|
| 301 |
final_log_message = f"Process complete at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} (An Nhơn, Binh Dinh, Vietnam)\n"
|
| 302 |
|
| 303 |
-
# Final return for both outputs
|
| 304 |
yield json.dumps(all_conversations, indent=2), final_log_message
|
| 305 |
|
| 306 |
# --- Community Prompts Functions ---
|
|
@@ -509,7 +513,7 @@ with gr.Blocks() as demo:
|
|
| 509 |
generate_button = gr.Button("Generate & Push Conversations")
|
| 510 |
|
| 511 |
output_conversations = gr.JSON(label="Generated Conversations (Content of conversations.jsonl)")
|
| 512 |
-
#
|
| 513 |
output_log = gr.Textbox(label="Process Log", interactive=False, lines=10, max_lines=20, streaming=True)
|
| 514 |
|
| 515 |
# Link commit template dropdown to update fields
|
|
@@ -536,7 +540,7 @@ with gr.Blocks() as demo:
|
|
| 536 |
model_selector_dropdown # Pass selected model name
|
| 537 |
],
|
| 538 |
outputs=[output_conversations, output_log],
|
| 539 |
-
# show_progress=True is handled internally by yielding
|
| 540 |
)
|
| 541 |
|
| 542 |
with gr.Tab("Community Prompts"):
|
|
|
|
| 190 |
commit_subject, commit_body, selected_model_name_input):
|
| 191 |
num_conversations = int(num_conversations_input)
|
| 192 |
if num_conversations <= 0:
|
| 193 |
+
# Use gr.NoOp() for the output that shouldn't be updated
|
| 194 |
+
yield gr.NoOp(), "Please enter a number of conversations greater than zero.\n"
|
| 195 |
return
|
| 196 |
|
| 197 |
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
|
|
|
| 204 |
try:
|
| 205 |
existing_conversations.append(json.loads(line.strip()))
|
| 206 |
except json.JSONDecodeError as e:
|
| 207 |
+
# Use gr.NoOp() for the output that shouldn't be updated
|
| 208 |
+
yield gr.NoOp(), f"Skipping malformed JSON line in {DATA_FILE}: {line.strip()} - {e}\n"
|
| 209 |
|
| 210 |
# Deduplicate existing conversations
|
| 211 |
seen_conversations = set()
|
|
|
|
| 226 |
if len(conv_list) == expected_msg_len(5): # Fixed to 5 exchanges for generation
|
| 227 |
validated_existing_conversations.append(conv_entry)
|
| 228 |
else:
|
| 229 |
+
# Use gr.NoOp() for the output that shouldn't be updated
|
| 230 |
+
yield gr.NoOp(), f"Skipping incomplete/malformed existing conversation (length {len(conv_list)} != {expected_msg_len(5)}): {conv_entry}\n"
|
| 231 |
|
| 232 |
all_conversations = list(validated_existing_conversations) # Start with clean existing ones
|
| 233 |
|
| 234 |
current_time_loc = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " (An Nhơn, Binh Dinh, Vietnam)"
|
| 235 |
+
# Use gr.NoOp() for the output that shouldn't be updated
|
| 236 |
+
yield gr.NoOp(), f"Starting conversation generation at {current_time_loc}\n"
|
| 237 |
+
yield gr.NoOp(), f"Loaded and cleaned {len(validated_existing_conversations)} existing conversations (initially {initial_cleaned_count} before validation).\n"
|
| 238 |
+
yield gr.NoOp(), f"Generating {num_conversations} *new* conversations.\n"
|
| 239 |
|
| 240 |
model_names_to_use = list(AVAILABLE_MODELS.keys())
|
| 241 |
if selected_model_name_input and selected_model_name_input in model_names_to_use:
|
|
|
|
| 244 |
else:
|
| 245 |
model_selection_info = f"No specific model selected or invalid. Models will be chosen randomly per conversation from: {', '.join(model_names_to_use)}\n"
|
| 246 |
selected_model_for_this_conv_batch = None # Indicate random selection per loop
|
| 247 |
+
yield gr.NoOp(), model_selection_info
|
| 248 |
|
| 249 |
|
| 250 |
current_prompts = DEFAULT_INITIAL_PROMPTS
|
|
|
|
| 270 |
|
| 271 |
selected_model_for_this_conv = selected_model_for_this_conv_batch if selected_model_for_this_conv_batch else random.choice(model_names_to_use)
|
| 272 |
|
| 273 |
+
yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Generating conv {i+1}/{num_conversations} with '{selected_model_for_this_conv}' (System: '{system[:50]}...')\n"
|
| 274 |
|
| 275 |
conversation = chat(system, prompt, selected_model_for_this_conv, seed=seed, num_exchanges=5)
|
| 276 |
|
|
|
|
| 281 |
all_conversations.append(new_conv_entry)
|
| 282 |
new_conversations_generated.append(new_conv_entry)
|
| 283 |
seen_conversations.add(new_conv_str)
|
| 284 |
+
yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Successfully generated and added conv {i+1}/{num_conversations}.\n"
|
| 285 |
else:
|
| 286 |
+
yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipped conv {i+1}/{num_conversations} as it's a duplicate.\n"
|
| 287 |
else:
|
| 288 |
+
yield gr.NoOp(), f"[{datetime.datetime.now().strftime('%H:%M:%S')}] Skipping conv {i+1}/{num_conversations} due to error or incorrect length ({len(conversation)} messages, expected {expected_conversation_length}).\n"
|
| 289 |
if conversation and conversation[-1].get("from") == "error":
|
| 290 |
+
yield gr.NoOp(), f" Error details: {conversation[-1]['value']}\n"
|
| 291 |
|
| 292 |
# Save all (cleaned existing + newly generated unique) conversations to JSONL
|
| 293 |
with open(DATA_FILE, "w") as f:
|
| 294 |
for conv in all_conversations:
|
| 295 |
f.write(json.dumps(conv) + "\n")
|
| 296 |
|
| 297 |
+
yield gr.NoOp(), f"Saved {len(new_conversations_generated)} *new unique* conversations to {DATA_FILE} (total unique and validated: {len(all_conversations)}).\n"
|
| 298 |
+
yield gr.NoOp(), "Attempting to push main conversations file to Hugging Face Dataset...\n"
|
| 299 |
|
| 300 |
# --- Auto-push main conversations to Hugging Face Dataset ---
|
| 301 |
# Use the custom commit message
|
| 302 |
commit_message = f"{commit_subject.strip()}\n\n{commit_body.strip()}" if commit_body.strip() else commit_subject.strip()
|
| 303 |
push_status = push_file_to_huggingface_dataset(DATA_FILE, "conversations.jsonl", commit_message)
|
| 304 |
+
yield gr.NoOp(), push_status + "\n"
|
| 305 |
final_log_message = f"Process complete at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} (An Nhơn, Binh Dinh, Vietnam)\n"
|
| 306 |
|
| 307 |
+
# Final return for both outputs: The first output (JSON) is now updated, the second (log) gets the final message.
|
| 308 |
yield json.dumps(all_conversations, indent=2), final_log_message
|
| 309 |
|
| 310 |
# --- Community Prompts Functions ---
|
|
|
|
| 513 |
generate_button = gr.Button("Generate & Push Conversations")
|
| 514 |
|
| 515 |
output_conversations = gr.JSON(label="Generated Conversations (Content of conversations.jsonl)")
|
| 516 |
+
# Set streaming=True for the output_log Textbox
|
| 517 |
output_log = gr.Textbox(label="Process Log", interactive=False, lines=10, max_lines=20, streaming=True)
|
| 518 |
|
| 519 |
# Link commit template dropdown to update fields
|
|
|
|
| 540 |
model_selector_dropdown # Pass selected model name
|
| 541 |
],
|
| 542 |
outputs=[output_conversations, output_log],
|
| 543 |
+
# show_progress=True is handled internally by yielding a generator
|
| 544 |
)
|
| 545 |
|
| 546 |
with gr.Tab("Community Prompts"):
|