Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -57,7 +57,6 @@ except Exception as e:
|
|
| 57 |
question_gen = QuestionGenerator(gemini_client=client)
|
| 58 |
|
| 59 |
# --- Session ID → socket SID mapping ---
|
| 60 |
-
# Maps socket session ID to learner model session ID
|
| 61 |
_socket_to_learner: dict[str, str] = {}
|
| 62 |
|
| 63 |
|
|
@@ -81,44 +80,109 @@ def decode_image(base64_string):
|
|
| 81 |
def sanitize_audio(input_path):
|
| 82 |
"""Force audio into Azure-compliant format: 16kHz, Mono, 16-bit PCM WAV."""
|
| 83 |
output_path = input_path + "_clean.wav"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
command = [
|
| 85 |
-
"ffmpeg", "-y", "-v", "
|
| 86 |
"-i", input_path,
|
| 87 |
"-ac", "1",
|
| 88 |
"-ar", "16000",
|
| 89 |
"-acodec", "pcm_s16le",
|
| 90 |
output_path
|
| 91 |
]
|
|
|
|
|
|
|
|
|
|
| 92 |
try:
|
| 93 |
-
subprocess.run(
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
return output_path
|
|
|
|
| 96 |
except subprocess.CalledProcessError as e:
|
| 97 |
-
logger.error(f"❌ FFmpeg failed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
return None
|
| 99 |
except Exception as e:
|
| 100 |
-
logger.error(f"❌
|
| 101 |
return None
|
| 102 |
|
| 103 |
|
| 104 |
def analyze_audio_volume(file_path):
|
|
|
|
| 105 |
try:
|
| 106 |
with wave.open(file_path, 'rb') as wf:
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
raw_data = wf.readframes(nframes)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
fmt = "%dh" % (len(raw_data) // 2)
|
| 110 |
pcm_data = struct.unpack(fmt, raw_data)
|
|
|
|
| 111 |
if not pcm_data:
|
|
|
|
| 112 |
return False
|
|
|
|
| 113 |
max_val = max(abs(x) for x in pcm_data)
|
| 114 |
-
|
|
|
|
|
|
|
| 115 |
if max_val < 100:
|
| 116 |
-
logger.warning("⚠️ Audio appears SILENT
|
| 117 |
return False
|
|
|
|
|
|
|
|
|
|
| 118 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
except Exception as e:
|
| 120 |
-
logger.warning(f"Could not analyze audio: {e}")
|
| 121 |
-
return True
|
| 122 |
|
| 123 |
|
| 124 |
def get_learner(socket_sid: str):
|
|
@@ -162,7 +226,6 @@ def handle_disconnect():
|
|
| 162 |
learner_id = _socket_to_learner.pop(sid, None)
|
| 163 |
if learner_id:
|
| 164 |
logger.info(f"Client disconnected: socket={sid} learner={learner_id}")
|
| 165 |
-
# Don't delete learner model immediately - allow reconnect grace period
|
| 166 |
else:
|
| 167 |
logger.info(f"Client disconnected: socket={sid}")
|
| 168 |
|
|
@@ -173,26 +236,12 @@ def handle_disconnect():
|
|
| 173 |
|
| 174 |
@socketio.on('load_content_pack')
|
| 175 |
def handle_load_content_pack(data):
|
| 176 |
-
"""
|
| 177 |
-
Load a teacher-uploaded content pack.
|
| 178 |
-
|
| 179 |
-
Expected data:
|
| 180 |
-
{
|
| 181 |
-
"file_bytes": "<base64 encoded DOCX/PDF/JSON>",
|
| 182 |
-
"file_type": "json|docx|pdf",
|
| 183 |
-
"lesson": "KLP7-10",
|
| 184 |
-
"description": "optional description"
|
| 185 |
-
}
|
| 186 |
-
|
| 187 |
-
For JSON packs: must contain {"vocab": [...], "grammar_rules": {...}}
|
| 188 |
-
For DOCX/PDF: Gemini parses it into structured data
|
| 189 |
-
"""
|
| 190 |
logger.info("📦 Content pack upload received")
|
| 191 |
|
| 192 |
try:
|
| 193 |
-
file_type
|
| 194 |
-
file_b64
|
| 195 |
-
lesson
|
| 196 |
description = data.get("description", "Custom content pack")
|
| 197 |
|
| 198 |
if "," in file_b64:
|
|
@@ -215,7 +264,6 @@ def handle_load_content_pack(data):
|
|
| 215 |
})
|
| 216 |
|
| 217 |
elif file_type in ("docx", "pdf"):
|
| 218 |
-
# Use Gemini to parse the document into structured vocab + grammar
|
| 219 |
if not client:
|
| 220 |
emit('content_pack_loaded', {"success": False, "error": "Gemini not available"})
|
| 221 |
return
|
|
@@ -286,16 +334,6 @@ Grammar rule IDs should be snake_case."""
|
|
| 286 |
|
| 287 |
@socketio.on('request_question')
|
| 288 |
def handle_request_question(data):
|
| 289 |
-
"""
|
| 290 |
-
Generate the next question for the learner.
|
| 291 |
-
|
| 292 |
-
Expected data (all optional):
|
| 293 |
-
{
|
| 294 |
-
"grammar_rule": "topic_marker|copula|...", // force a specific type
|
| 295 |
-
"difficulty": 1, // override difficulty
|
| 296 |
-
"interaction_mode": "assemble|choose_select|fill_in|speak" // prefer a mode
|
| 297 |
-
}
|
| 298 |
-
"""
|
| 299 |
from flask import request as req
|
| 300 |
sid = req.sid
|
| 301 |
learner = get_learner(sid)
|
|
@@ -305,13 +343,10 @@ def handle_request_question(data):
|
|
| 305 |
return
|
| 306 |
|
| 307 |
try:
|
| 308 |
-
|
| 309 |
-
forced_rule = data.get("grammar_rule") if data else None
|
| 310 |
override_difficulty = data.get("difficulty") if data else None
|
| 311 |
-
difficulty
|
| 312 |
-
|
| 313 |
-
# Smart rule selection if not forced
|
| 314 |
-
grammar_rule = forced_rule or learner.get_recommended_rule()
|
| 315 |
|
| 316 |
logger.info(f"🎯 Generating question: rule={grammar_rule} difficulty={difficulty} session={learner.session_id}")
|
| 317 |
|
|
@@ -335,48 +370,27 @@ def handle_request_question(data):
|
|
| 335 |
|
| 336 |
@socketio.on('submit_answer')
|
| 337 |
def handle_submit_answer(data):
|
| 338 |
-
"""
|
| 339 |
-
Validate a player's answer.
|
| 340 |
-
|
| 341 |
-
Expected data:
|
| 342 |
-
{
|
| 343 |
-
"question_id": "uuid",
|
| 344 |
-
"question_type": "topic_marker|copula|...",
|
| 345 |
-
"grammar_rule": "topic_marker",
|
| 346 |
-
"interaction_mode": "choose_select|assemble|fill_in",
|
| 347 |
-
"answer": "는", // for choose_select / fill_in
|
| 348 |
-
"token_order": [1, 0, 2], // for assemble mode
|
| 349 |
-
"correct_order": [0, 1, 2], // expected order (from question payload)
|
| 350 |
-
"word_tested": "사과", // for particle questions
|
| 351 |
-
"particle_type": "topic|copula|subject|negative",
|
| 352 |
-
"attempt_number": 1
|
| 353 |
-
}
|
| 354 |
-
"""
|
| 355 |
from flask import request as req
|
| 356 |
-
sid
|
| 357 |
learner = get_learner(sid)
|
| 358 |
|
| 359 |
-
q_type
|
| 360 |
-
grammar_rule
|
| 361 |
interaction_mode = data.get("interaction_mode", "")
|
| 362 |
-
attempt
|
| 363 |
|
| 364 |
try:
|
| 365 |
correct = False
|
| 366 |
|
| 367 |
-
# ── Assemble mode: compare token order ──
|
| 368 |
if interaction_mode == "assemble":
|
| 369 |
submitted = data.get("token_order", [])
|
| 370 |
-
expected
|
| 371 |
-
correct
|
| 372 |
|
| 373 |
-
# ── Choose / fill-in: compare answer to answer_key ──
|
| 374 |
elif interaction_mode in ("choose_select", "fill_in"):
|
| 375 |
-
chosen
|
| 376 |
answer_key = str(data.get("answer_key", "")).strip()
|
| 377 |
-
|
| 378 |
-
# If particle validation, use rule engine
|
| 379 |
-
word_tested = data.get("word_tested")
|
| 380 |
particle_type = data.get("particle_type")
|
| 381 |
|
| 382 |
if word_tested and particle_type:
|
|
@@ -384,44 +398,40 @@ def handle_submit_answer(data):
|
|
| 384 |
else:
|
| 385 |
correct = (chosen == answer_key)
|
| 386 |
|
| 387 |
-
# ── Server-side re-check for indirect quote forms ──
|
| 388 |
if not correct and q_type in ("indirect_quote_dago", "indirect_quote_commands",
|
| 389 |
"indirect_quote_questions", "indirect_quote_suggestions"):
|
| 390 |
-
# For complex grammar, Gemini does a re-check if first attempt fails
|
| 391 |
if client and interaction_mode == "fill_in" and attempt <= 2:
|
| 392 |
correct = _gemini_recheck(data)
|
| 393 |
|
| 394 |
-
# Update mastery
|
| 395 |
if learner:
|
| 396 |
learner.record_outcome(grammar_rule, correct, interaction_mode)
|
| 397 |
|
| 398 |
-
# Build response
|
| 399 |
hint = None
|
| 400 |
if not correct:
|
| 401 |
-
word
|
| 402 |
ptype = data.get("particle_type")
|
| 403 |
if word and ptype:
|
| 404 |
hint = rule_engine.get_hint(word, ptype)
|
| 405 |
else:
|
| 406 |
hint = data.get("hint_text", "Review the grammar rule and try again")
|
| 407 |
|
| 408 |
-
retry_allowed
|
| 409 |
speech_stage_unlocked = correct
|
| 410 |
|
| 411 |
response = {
|
| 412 |
-
"question_id":
|
| 413 |
-
"correct":
|
| 414 |
-
"score_delta":
|
| 415 |
-
"feedback":
|
| 416 |
-
"hint":
|
| 417 |
-
"retry_allowed":
|
| 418 |
-
"attempt_number":
|
| 419 |
"speech_stage_unlocked": speech_stage_unlocked,
|
| 420 |
}
|
| 421 |
|
| 422 |
if learner:
|
| 423 |
response["mastery_update"] = dict(learner.mastery)
|
| 424 |
-
response["streak"]
|
| 425 |
|
| 426 |
emit('answer_result', response)
|
| 427 |
|
|
@@ -436,10 +446,9 @@ def handle_submit_answer(data):
|
|
| 436 |
|
| 437 |
|
| 438 |
def _gemini_recheck(data: dict) -> bool:
|
| 439 |
-
"""Use Gemini to re-check a complex indirect quotation answer."""
|
| 440 |
try:
|
| 441 |
prompt = f"""You are a Korean language grammar validator.
|
| 442 |
-
|
| 443 |
Direct speech: {data.get('direct_speech', '')}
|
| 444 |
Student's indirect speech: {data.get('answer', '')}
|
| 445 |
Expected indirect speech: {data.get('answer_key', '')}
|
|
@@ -461,7 +470,6 @@ Reply with ONLY valid JSON: {{"correct": true}} or {{"correct": false, "reason":
|
|
| 461 |
|
| 462 |
|
| 463 |
def _build_feedback(correct: bool, q_type: str, grammar_rule: str) -> str:
|
| 464 |
-
"""Build encouraging feedback message."""
|
| 465 |
if correct:
|
| 466 |
messages = [
|
| 467 |
"정확해요! Great job! 🎉",
|
|
@@ -473,170 +481,308 @@ def _build_feedback(correct: bool, q_type: str, grammar_rule: str) -> str:
|
|
| 473 |
return random.choice(messages)
|
| 474 |
else:
|
| 475 |
rule_hints = {
|
| 476 |
-
"topic_marker":
|
| 477 |
-
"copula":
|
| 478 |
-
"negative_copula":
|
| 479 |
-
"indirect_quote_dago":
|
| 480 |
-
"indirect_quote_commands":
|
| 481 |
-
"indirect_quote_questions":
|
| 482 |
-
"indirect_quote_suggestions":
|
| 483 |
-
"regret_expression":
|
| 484 |
}
|
| 485 |
base = "다시 해 보세요! Let's try again. "
|
| 486 |
return base + rule_hints.get(grammar_rule, "Review the grammar rule.")
|
| 487 |
|
| 488 |
|
| 489 |
# ===========================================================================
|
| 490 |
-
# 4. PRONUNCIATION ASSESSMENT
|
| 491 |
# ===========================================================================
|
| 492 |
|
| 493 |
@socketio.on('assess_pronunciation')
|
| 494 |
def handle_pronunciation(data):
|
| 495 |
-
"""
|
| 496 |
-
Assess Korean (or any language) pronunciation via Azure.
|
| 497 |
-
|
| 498 |
-
Expected data:
|
| 499 |
-
{
|
| 500 |
-
"audio": "<base64 encoded audio>",
|
| 501 |
-
"text": "저는 학생이에요",
|
| 502 |
-
"lang": "ko-KR", // default ko-KR for Korean
|
| 503 |
-
"grammar_rule": "copula", // optional: for mastery tracking
|
| 504 |
-
"question_id": "uuid" // optional: link to question
|
| 505 |
-
}
|
| 506 |
-
"""
|
| 507 |
from flask import request as req
|
| 508 |
-
sid
|
| 509 |
learner = get_learner(sid)
|
| 510 |
|
| 511 |
-
ref_text
|
| 512 |
-
lang
|
| 513 |
grammar_rule = data.get('grammar_rule', '')
|
| 514 |
|
| 515 |
-
|
|
|
|
|
|
|
|
|
|
| 516 |
|
| 517 |
-
|
| 518 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 519 |
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 525 |
|
| 526 |
-
|
| 527 |
-
temp_raw.write(audio_bytes)
|
| 528 |
-
raw_path = temp_raw.name
|
| 529 |
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
|
|
|
| 533 |
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
reference_text=ref_text,
|
| 543 |
-
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
|
| 544 |
-
granularity=speechsdk.PronunciationAssessmentGranularity.Word,
|
| 545 |
-
enable_miscue=True
|
| 546 |
-
)
|
| 547 |
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
|
|
|
|
|
|
|
|
|
| 553 |
|
| 554 |
-
|
| 555 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
response = {}
|
|
|
|
| 557 |
if result.reason == speechsdk.ResultReason.RecognizedSpeech:
|
| 558 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 559 |
|
| 560 |
detailed_words = []
|
| 561 |
for word in pron_result.words:
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
accuracy = pron_result.accuracy_score
|
| 569 |
-
fluency = pron_result.fluency_score
|
| 570 |
-
completeness = pron_result.completeness_score
|
| 571 |
-
|
| 572 |
-
# Generate teacher-style feedback
|
| 573 |
-
feedback = _build_pronunciation_feedback(
|
| 574 |
-
accuracy, fluency, completeness, detailed_words, ref_text
|
| 575 |
-
)
|
| 576 |
|
| 577 |
response = {
|
| 578 |
-
"success":
|
| 579 |
-
"score":
|
| 580 |
-
"fluency":
|
| 581 |
-
"completeness":
|
| 582 |
"recognized_text": result.text,
|
| 583 |
-
"word_details":
|
| 584 |
-
"feedback":
|
| 585 |
-
"question_id":
|
| 586 |
}
|
| 587 |
|
| 588 |
-
# Update mastery if grammar rule provided and score is high
|
| 589 |
if learner and grammar_rule and accuracy >= 70:
|
| 590 |
learner.record_outcome(grammar_rule, True, "speak")
|
| 591 |
response["mastery_update"] = dict(learner.mastery)
|
| 592 |
|
| 593 |
-
logger.info(f"✅
|
| 594 |
|
| 595 |
elif result.reason == speechsdk.ResultReason.NoMatch:
|
|
|
|
|
|
|
|
|
|
| 596 |
response = {
|
| 597 |
-
"success": False,
|
| 598 |
-
"score": 0,
|
| 599 |
-
"fluency": 0,
|
| 600 |
-
"completeness": 0,
|
| 601 |
"recognized_text": "",
|
| 602 |
"word_details": [],
|
| 603 |
-
"feedback": "I couldn't hear you clearly. Please try speaking again.",
|
| 604 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 605 |
else:
|
|
|
|
| 606 |
response = {
|
| 607 |
-
"success": False,
|
| 608 |
-
"
|
| 609 |
-
"
|
| 610 |
-
"completeness": 0,
|
| 611 |
-
"recognized_text": "",
|
| 612 |
-
"word_details": [],
|
| 613 |
-
"feedback": "Error during recognition. Please try again.",
|
| 614 |
}
|
| 615 |
|
|
|
|
|
|
|
| 616 |
emit('pronunciation_result', response)
|
| 617 |
|
| 618 |
except Exception as e:
|
| 619 |
-
logger.error(f"
|
|
|
|
|
|
|
| 620 |
emit('pronunciation_result', {
|
| 621 |
-
"success": False,
|
| 622 |
-
"
|
| 623 |
-
"fluency": 0,
|
| 624 |
-
"completeness": 0,
|
| 625 |
-
"recognized_text": "",
|
| 626 |
-
"word_details": [],
|
| 627 |
"feedback": "Server error during assessment.",
|
| 628 |
})
|
| 629 |
finally:
|
| 630 |
if raw_path and os.path.exists(raw_path):
|
| 631 |
os.remove(raw_path)
|
|
|
|
| 632 |
if clean_path and os.path.exists(clean_path):
|
| 633 |
os.remove(clean_path)
|
|
|
|
| 634 |
|
| 635 |
|
| 636 |
def _build_pronunciation_feedback(accuracy: float, fluency: float,
|
| 637 |
completeness: float, words: list,
|
| 638 |
ref_text: str) -> str:
|
| 639 |
-
"""Build teacher-style pronunciation feedback."""
|
| 640 |
issues = [w for w in words if w.get("error") not in (None, "None", "") or w.get("score", 100) < 60]
|
| 641 |
|
| 642 |
if accuracy >= 85:
|
|
@@ -664,10 +810,6 @@ def _build_pronunciation_feedback(accuracy: float, fluency: float,
|
|
| 664 |
|
| 665 |
@socketio.on('get_mastery')
|
| 666 |
def handle_get_mastery(data):
|
| 667 |
-
"""
|
| 668 |
-
Unity polls this to display the learner's current mastery state.
|
| 669 |
-
Returns full learner model state for Unity to store if needed.
|
| 670 |
-
"""
|
| 671 |
from flask import request as req
|
| 672 |
learner = get_learner(req.sid)
|
| 673 |
|
|
@@ -680,17 +822,6 @@ def handle_get_mastery(data):
|
|
| 680 |
|
| 681 |
@socketio.on('restore_session')
|
| 682 |
def handle_restore_session(data):
|
| 683 |
-
"""
|
| 684 |
-
Unity can send a previously saved learner state to restore progress.
|
| 685 |
-
|
| 686 |
-
Expected data: the full state object from a previous get_mastery response.
|
| 687 |
-
{
|
| 688 |
-
"session_id": "...",
|
| 689 |
-
"mastery": {...},
|
| 690 |
-
"difficulty": 2,
|
| 691 |
-
...
|
| 692 |
-
}
|
| 693 |
-
"""
|
| 694 |
from flask import request as req
|
| 695 |
sid = req.sid
|
| 696 |
|
|
@@ -705,10 +836,10 @@ def handle_restore_session(data):
|
|
| 705 |
logger.info(f"♻️ Session restored for {learner_id}: difficulty={learner.difficulty}")
|
| 706 |
|
| 707 |
emit('session_restored', {
|
| 708 |
-
"success":
|
| 709 |
-
"session_id":
|
| 710 |
-
"mastery":
|
| 711 |
-
"difficulty":
|
| 712 |
"question_count": learner.question_count,
|
| 713 |
})
|
| 714 |
|
|
@@ -719,17 +850,16 @@ def handle_restore_session(data):
|
|
| 719 |
|
| 720 |
@socketio.on('reset_session')
|
| 721 |
def handle_reset_session(data):
|
| 722 |
-
"""Reset the learner model for a fresh start."""
|
| 723 |
from flask import request as req
|
| 724 |
-
sid
|
| 725 |
learner = get_learner(sid)
|
| 726 |
|
| 727 |
if learner:
|
| 728 |
learner.reset()
|
| 729 |
logger.info(f"🔄 Session reset: {learner.session_id}")
|
| 730 |
emit('session_reset', {
|
| 731 |
-
"success":
|
| 732 |
-
"mastery":
|
| 733 |
"difficulty": learner.difficulty,
|
| 734 |
})
|
| 735 |
else:
|
|
@@ -738,16 +868,6 @@ def handle_reset_session(data):
|
|
| 738 |
|
| 739 |
@socketio.on('update_mastery')
|
| 740 |
def handle_update_mastery(data):
|
| 741 |
-
"""
|
| 742 |
-
Explicit mastery update from Unity (e.g. after a mini-game result).
|
| 743 |
-
|
| 744 |
-
Expected data:
|
| 745 |
-
{
|
| 746 |
-
"grammar_rule": "topic_marker",
|
| 747 |
-
"correct": true,
|
| 748 |
-
"interaction_mode": "assemble"
|
| 749 |
-
}
|
| 750 |
-
"""
|
| 751 |
from flask import request as req
|
| 752 |
learner = get_learner(req.sid)
|
| 753 |
|
|
@@ -756,21 +876,21 @@ def handle_update_mastery(data):
|
|
| 756 |
return
|
| 757 |
|
| 758 |
grammar_rule = data.get("grammar_rule", "")
|
| 759 |
-
correct
|
| 760 |
-
mode
|
| 761 |
|
| 762 |
if grammar_rule:
|
| 763 |
learner.record_outcome(grammar_rule, correct, mode)
|
| 764 |
|
| 765 |
emit('mastery_updated', {
|
| 766 |
-
"mastery":
|
| 767 |
"difficulty": learner.difficulty,
|
| 768 |
-
"streak":
|
| 769 |
})
|
| 770 |
|
| 771 |
|
| 772 |
# ===========================================================================
|
| 773 |
-
# 6. VISUAL RECOGNITION
|
| 774 |
# ===========================================================================
|
| 775 |
|
| 776 |
@socketio.on('verify_object')
|
|
@@ -791,9 +911,9 @@ def handle_object_verification(data):
|
|
| 791 |
schema = {
|
| 792 |
"type": "OBJECT",
|
| 793 |
"properties": {
|
| 794 |
-
"verified":
|
| 795 |
"confidence": {"type": "NUMBER"},
|
| 796 |
-
"feedback":
|
| 797 |
},
|
| 798 |
"required": ["verified", "feedback"]
|
| 799 |
}
|
|
@@ -823,7 +943,7 @@ Return JSON matching the schema."""
|
|
| 823 |
|
| 824 |
|
| 825 |
# ===========================================================================
|
| 826 |
-
# 7. HANDWRITING / OCR
|
| 827 |
# ===========================================================================
|
| 828 |
|
| 829 |
@socketio.on('verify_writing')
|
|
@@ -844,9 +964,9 @@ def handle_writing_verification(data):
|
|
| 844 |
schema = {
|
| 845 |
"type": "OBJECT",
|
| 846 |
"properties": {
|
| 847 |
-
"correct":
|
| 848 |
"detected_text": {"type": "STRING"},
|
| 849 |
-
"feedback":
|
| 850 |
},
|
| 851 |
"required": ["correct", "detected_text"]
|
| 852 |
}
|
|
@@ -875,29 +995,27 @@ Return JSON with: correct (bool), detected_text (what you read), feedback (brief
|
|
| 875 |
|
| 876 |
|
| 877 |
# ===========================================================================
|
| 878 |
-
# 8. GRAMMAR RULE INFO
|
| 879 |
# ===========================================================================
|
| 880 |
|
| 881 |
@socketio.on('get_grammar_rules')
|
| 882 |
def handle_get_grammar_rules(data):
|
| 883 |
-
"""Return all available grammar rules from the active content pack."""
|
| 884 |
pack = get_active_pack()
|
| 885 |
emit('grammar_rules', {
|
| 886 |
-
"rules":
|
| 887 |
"lesson": pack.get("lesson"),
|
| 888 |
})
|
| 889 |
|
| 890 |
|
| 891 |
@socketio.on('get_content_pack_info')
|
| 892 |
def handle_get_content_pack_info(data):
|
| 893 |
-
"""Return info about the active content pack (no full vocab dump)."""
|
| 894 |
pack = get_active_pack()
|
| 895 |
emit('content_pack_info', {
|
| 896 |
-
"lesson":
|
| 897 |
-
"version":
|
| 898 |
-
"vocab_count":
|
| 899 |
"grammar_rules": list(pack.get("grammar_rules", {}).keys()),
|
| 900 |
-
"metadata":
|
| 901 |
})
|
| 902 |
|
| 903 |
|
|
@@ -906,8 +1024,6 @@ def handle_get_content_pack_info(data):
|
|
| 906 |
# ===========================================================================
|
| 907 |
|
| 908 |
if __name__ == '__main__':
|
| 909 |
-
# Purge stale sessions on startup
|
| 910 |
purge_stale_sessions()
|
| 911 |
logger.info("🚀 KLP AI Service starting on port 7860")
|
| 912 |
-
# Port 7860 required for Hugging Face Spaces
|
| 913 |
socketio.run(app, host='0.0.0.0', port=7860)
|
|
|
|
| 57 |
question_gen = QuestionGenerator(gemini_client=client)
|
| 58 |
|
| 59 |
# --- Session ID → socket SID mapping ---
|
|
|
|
| 60 |
_socket_to_learner: dict[str, str] = {}
|
| 61 |
|
| 62 |
|
|
|
|
| 80 |
def sanitize_audio(input_path):
|
| 81 |
"""Force audio into Azure-compliant format: 16kHz, Mono, 16-bit PCM WAV."""
|
| 82 |
output_path = input_path + "_clean.wav"
|
| 83 |
+
|
| 84 |
+
# --- STEP: Log input file info before conversion ---
|
| 85 |
+
try:
|
| 86 |
+
input_size = os.path.getsize(input_path)
|
| 87 |
+
logger.info(f"🔧 [FFmpeg] Input file: {input_path} | Size: {input_size} bytes")
|
| 88 |
+
if input_size == 0:
|
| 89 |
+
logger.error("❌ [FFmpeg] Input file is EMPTY (0 bytes) — audio was not captured correctly")
|
| 90 |
+
return None
|
| 91 |
+
except Exception as e:
|
| 92 |
+
logger.error(f"❌ [FFmpeg] Could not stat input file: {e}")
|
| 93 |
+
|
| 94 |
command = [
|
| 95 |
+
"ffmpeg", "-y", "-v", "verbose",
|
| 96 |
"-i", input_path,
|
| 97 |
"-ac", "1",
|
| 98 |
"-ar", "16000",
|
| 99 |
"-acodec", "pcm_s16le",
|
| 100 |
output_path
|
| 101 |
]
|
| 102 |
+
|
| 103 |
+
logger.info(f"🔧 [FFmpeg] Running command: {' '.join(command)}")
|
| 104 |
+
|
| 105 |
try:
|
| 106 |
+
result = subprocess.run(
|
| 107 |
+
command,
|
| 108 |
+
check=True,
|
| 109 |
+
capture_output=True,
|
| 110 |
+
text=True
|
| 111 |
+
)
|
| 112 |
+
logger.info(f"✅ [FFmpeg] Conversion successful → {output_path}")
|
| 113 |
+
if result.stderr:
|
| 114 |
+
# ffmpeg writes progress/info to stderr even on success
|
| 115 |
+
logger.info(f"🔧 [FFmpeg] stderr output:\n{result.stderr[:2000]}")
|
| 116 |
+
|
| 117 |
+
output_size = os.path.getsize(output_path)
|
| 118 |
+
logger.info(f"🔧 [FFmpeg] Output file size: {output_size} bytes")
|
| 119 |
+
if output_size == 0:
|
| 120 |
+
logger.error("❌ [FFmpeg] Output WAV is EMPTY — conversion produced no data")
|
| 121 |
+
return None
|
| 122 |
+
|
| 123 |
return output_path
|
| 124 |
+
|
| 125 |
except subprocess.CalledProcessError as e:
|
| 126 |
+
logger.error(f"❌ [FFmpeg] Process failed with return code {e.returncode}")
|
| 127 |
+
logger.error(f"❌ [FFmpeg] stdout: {e.stdout}")
|
| 128 |
+
logger.error(f"❌ [FFmpeg] stderr: {e.stderr}")
|
| 129 |
+
return None
|
| 130 |
+
except FileNotFoundError:
|
| 131 |
+
logger.error("❌ [FFmpeg] ffmpeg binary not found — is it installed in the Docker image?")
|
| 132 |
return None
|
| 133 |
except Exception as e:
|
| 134 |
+
logger.error(f"❌ [FFmpeg] Unexpected error: {e}")
|
| 135 |
return None
|
| 136 |
|
| 137 |
|
| 138 |
def analyze_audio_volume(file_path):
|
| 139 |
+
"""Inspect a WAV file: log frame rate, channels, duration, and peak amplitude."""
|
| 140 |
try:
|
| 141 |
with wave.open(file_path, 'rb') as wf:
|
| 142 |
+
framerate = wf.getframerate()
|
| 143 |
+
nframes = wf.getnframes()
|
| 144 |
+
channels = wf.getnchannels()
|
| 145 |
+
sampwidth = wf.getsampwidth()
|
| 146 |
+
duration_s = nframes / framerate if framerate else 0
|
| 147 |
+
|
| 148 |
+
logger.info(
|
| 149 |
+
f"🔊 [WAV] framerate={framerate}Hz | channels={channels} | "
|
| 150 |
+
f"sampwidth={sampwidth}B | nframes={nframes} | duration={duration_s:.2f}s"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
if duration_s < 0.2:
|
| 154 |
+
logger.warning(f"⚠️ [WAV] Audio is very short ({duration_s:.2f}s) — may not be enough for recognition")
|
| 155 |
+
|
| 156 |
raw_data = wf.readframes(nframes)
|
| 157 |
+
if len(raw_data) == 0:
|
| 158 |
+
logger.error("❌ [WAV] No PCM data in file")
|
| 159 |
+
return False
|
| 160 |
+
|
| 161 |
fmt = "%dh" % (len(raw_data) // 2)
|
| 162 |
pcm_data = struct.unpack(fmt, raw_data)
|
| 163 |
+
|
| 164 |
if not pcm_data:
|
| 165 |
+
logger.error("❌ [WAV] PCM unpack produced no samples")
|
| 166 |
return False
|
| 167 |
+
|
| 168 |
max_val = max(abs(x) for x in pcm_data)
|
| 169 |
+
avg_val = sum(abs(x) for x in pcm_data) / len(pcm_data)
|
| 170 |
+
logger.info(f"🔊 [WAV] Peak amplitude: {max_val}/32767 | Avg amplitude: {avg_val:.1f}")
|
| 171 |
+
|
| 172 |
if max_val < 100:
|
| 173 |
+
logger.warning("⚠️ [WAV] Audio appears SILENT (peak < 100) — microphone may not be working")
|
| 174 |
return False
|
| 175 |
+
if max_val < 500:
|
| 176 |
+
logger.warning(f"⚠️ [WAV] Audio is very quiet (peak={max_val}) — may affect recognition accuracy")
|
| 177 |
+
|
| 178 |
return True
|
| 179 |
+
|
| 180 |
+
except wave.Error as e:
|
| 181 |
+
logger.error(f"❌ [WAV] wave.Error reading file: {e} — file may not be a valid WAV")
|
| 182 |
+
return False
|
| 183 |
except Exception as e:
|
| 184 |
+
logger.warning(f"⚠️ [WAV] Could not analyze audio volume: {e}")
|
| 185 |
+
return True # Don't block on analysis failure
|
| 186 |
|
| 187 |
|
| 188 |
def get_learner(socket_sid: str):
|
|
|
|
| 226 |
learner_id = _socket_to_learner.pop(sid, None)
|
| 227 |
if learner_id:
|
| 228 |
logger.info(f"Client disconnected: socket={sid} learner={learner_id}")
|
|
|
|
| 229 |
else:
|
| 230 |
logger.info(f"Client disconnected: socket={sid}")
|
| 231 |
|
|
|
|
| 236 |
|
| 237 |
@socketio.on('load_content_pack')
|
| 238 |
def handle_load_content_pack(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
logger.info("📦 Content pack upload received")
|
| 240 |
|
| 241 |
try:
|
| 242 |
+
file_type = data.get("file_type", "json").lower()
|
| 243 |
+
file_b64 = data.get("file_bytes", "")
|
| 244 |
+
lesson = data.get("lesson", "custom")
|
| 245 |
description = data.get("description", "Custom content pack")
|
| 246 |
|
| 247 |
if "," in file_b64:
|
|
|
|
| 264 |
})
|
| 265 |
|
| 266 |
elif file_type in ("docx", "pdf"):
|
|
|
|
| 267 |
if not client:
|
| 268 |
emit('content_pack_loaded', {"success": False, "error": "Gemini not available"})
|
| 269 |
return
|
|
|
|
| 334 |
|
| 335 |
@socketio.on('request_question')
|
| 336 |
def handle_request_question(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
from flask import request as req
|
| 338 |
sid = req.sid
|
| 339 |
learner = get_learner(sid)
|
|
|
|
| 343 |
return
|
| 344 |
|
| 345 |
try:
|
| 346 |
+
forced_rule = data.get("grammar_rule") if data else None
|
|
|
|
| 347 |
override_difficulty = data.get("difficulty") if data else None
|
| 348 |
+
difficulty = override_difficulty or learner.difficulty
|
| 349 |
+
grammar_rule = forced_rule or learner.get_recommended_rule()
|
|
|
|
|
|
|
| 350 |
|
| 351 |
logger.info(f"🎯 Generating question: rule={grammar_rule} difficulty={difficulty} session={learner.session_id}")
|
| 352 |
|
|
|
|
| 370 |
|
| 371 |
@socketio.on('submit_answer')
|
| 372 |
def handle_submit_answer(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
from flask import request as req
|
| 374 |
+
sid = req.sid
|
| 375 |
learner = get_learner(sid)
|
| 376 |
|
| 377 |
+
q_type = data.get("question_type", "")
|
| 378 |
+
grammar_rule = data.get("grammar_rule", q_type)
|
| 379 |
interaction_mode = data.get("interaction_mode", "")
|
| 380 |
+
attempt = data.get("attempt_number", 1)
|
| 381 |
|
| 382 |
try:
|
| 383 |
correct = False
|
| 384 |
|
|
|
|
| 385 |
if interaction_mode == "assemble":
|
| 386 |
submitted = data.get("token_order", [])
|
| 387 |
+
expected = data.get("correct_order", [])
|
| 388 |
+
correct = rule_engine.validate_token_order(submitted, expected)
|
| 389 |
|
|
|
|
| 390 |
elif interaction_mode in ("choose_select", "fill_in"):
|
| 391 |
+
chosen = str(data.get("answer", "")).strip()
|
| 392 |
answer_key = str(data.get("answer_key", "")).strip()
|
| 393 |
+
word_tested = data.get("word_tested")
|
|
|
|
|
|
|
| 394 |
particle_type = data.get("particle_type")
|
| 395 |
|
| 396 |
if word_tested and particle_type:
|
|
|
|
| 398 |
else:
|
| 399 |
correct = (chosen == answer_key)
|
| 400 |
|
|
|
|
| 401 |
if not correct and q_type in ("indirect_quote_dago", "indirect_quote_commands",
|
| 402 |
"indirect_quote_questions", "indirect_quote_suggestions"):
|
|
|
|
| 403 |
if client and interaction_mode == "fill_in" and attempt <= 2:
|
| 404 |
correct = _gemini_recheck(data)
|
| 405 |
|
|
|
|
| 406 |
if learner:
|
| 407 |
learner.record_outcome(grammar_rule, correct, interaction_mode)
|
| 408 |
|
|
|
|
| 409 |
hint = None
|
| 410 |
if not correct:
|
| 411 |
+
word = data.get("word_tested")
|
| 412 |
ptype = data.get("particle_type")
|
| 413 |
if word and ptype:
|
| 414 |
hint = rule_engine.get_hint(word, ptype)
|
| 415 |
else:
|
| 416 |
hint = data.get("hint_text", "Review the grammar rule and try again")
|
| 417 |
|
| 418 |
+
retry_allowed = not correct and attempt < 3
|
| 419 |
speech_stage_unlocked = correct
|
| 420 |
|
| 421 |
response = {
|
| 422 |
+
"question_id": data.get("question_id"),
|
| 423 |
+
"correct": correct,
|
| 424 |
+
"score_delta": 10 if correct else 0,
|
| 425 |
+
"feedback": _build_feedback(correct, q_type, grammar_rule),
|
| 426 |
+
"hint": hint,
|
| 427 |
+
"retry_allowed": retry_allowed,
|
| 428 |
+
"attempt_number": attempt,
|
| 429 |
"speech_stage_unlocked": speech_stage_unlocked,
|
| 430 |
}
|
| 431 |
|
| 432 |
if learner:
|
| 433 |
response["mastery_update"] = dict(learner.mastery)
|
| 434 |
+
response["streak"] = learner.streak
|
| 435 |
|
| 436 |
emit('answer_result', response)
|
| 437 |
|
|
|
|
| 446 |
|
| 447 |
|
| 448 |
def _gemini_recheck(data: dict) -> bool:
|
|
|
|
| 449 |
try:
|
| 450 |
prompt = f"""You are a Korean language grammar validator.
|
| 451 |
+
|
| 452 |
Direct speech: {data.get('direct_speech', '')}
|
| 453 |
Student's indirect speech: {data.get('answer', '')}
|
| 454 |
Expected indirect speech: {data.get('answer_key', '')}
|
|
|
|
| 470 |
|
| 471 |
|
| 472 |
def _build_feedback(correct: bool, q_type: str, grammar_rule: str) -> str:
|
|
|
|
| 473 |
if correct:
|
| 474 |
messages = [
|
| 475 |
"정확해요! Great job! 🎉",
|
|
|
|
| 481 |
return random.choice(messages)
|
| 482 |
else:
|
| 483 |
rule_hints = {
|
| 484 |
+
"topic_marker": "Remember: 은 for consonant endings, 는 for vowel endings",
|
| 485 |
+
"copula": "Remember: 이에요 for consonant endings, 예요 for vowel endings",
|
| 486 |
+
"negative_copula": "Remember: 이 아니에요 for consonant, 가 아니에요 for vowel/ㄹ",
|
| 487 |
+
"indirect_quote_dago": "Review: V+는다고/ㄴ다고, Adj+다고, Past+었다고",
|
| 488 |
+
"indirect_quote_commands": "Review: (으)라고 commands, 지 말라고 negatives",
|
| 489 |
+
"indirect_quote_questions": "Review: V/Adj+냐고 (drop ㄹ from stem)",
|
| 490 |
+
"indirect_quote_suggestions":"Review: V+자고 for suggestions",
|
| 491 |
+
"regret_expression": "Review: (으)ㄹ 걸 그랬다 = should have; 지 말 걸 = shouldn't have",
|
| 492 |
}
|
| 493 |
base = "다시 해 보세요! Let's try again. "
|
| 494 |
return base + rule_hints.get(grammar_rule, "Review the grammar rule.")
|
| 495 |
|
| 496 |
|
| 497 |
# ===========================================================================
|
| 498 |
+
# 4. PRONUNCIATION ASSESSMENT
|
| 499 |
# ===========================================================================
|
| 500 |
|
| 501 |
@socketio.on('assess_pronunciation')
|
| 502 |
def handle_pronunciation(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 503 |
from flask import request as req
|
| 504 |
+
sid = req.sid
|
| 505 |
learner = get_learner(sid)
|
| 506 |
|
| 507 |
+
ref_text = data.get('text')
|
| 508 |
+
lang = data.get('lang', 'ko-KR')
|
| 509 |
grammar_rule = data.get('grammar_rule', '')
|
| 510 |
|
| 511 |
+
# ── STEP 1: Validate incoming data ──────────────────────────────────────
|
| 512 |
+
logger.info("=" * 60)
|
| 513 |
+
logger.info("🎤 [PRON] ── Pronunciation Assessment Start ──")
|
| 514 |
+
logger.info(f"🎤 [PRON] ref_text='{ref_text}' | lang='{lang}' | grammar_rule='{grammar_rule}'")
|
| 515 |
|
| 516 |
+
if not ref_text:
|
| 517 |
+
logger.error("❌ [PRON] STEP 1 FAILED: No reference text provided in payload")
|
| 518 |
+
emit('pronunciation_result', {
|
| 519 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 520 |
+
"recognized_text": "", "word_details": [],
|
| 521 |
+
"feedback": "No reference text provided.",
|
| 522 |
+
})
|
| 523 |
+
return
|
| 524 |
|
| 525 |
+
audio_b64 = data.get('audio', '')
|
| 526 |
+
if not audio_b64:
|
| 527 |
+
logger.error("❌ [PRON] STEP 1 FAILED: No audio data in payload")
|
| 528 |
+
emit('pronunciation_result', {
|
| 529 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 530 |
+
"recognized_text": "", "word_details": [],
|
| 531 |
+
"feedback": "No audio data received.",
|
| 532 |
+
})
|
| 533 |
+
return
|
| 534 |
|
| 535 |
+
logger.info(f"🎤 [PRON] STEP 1 OK: ref_text present, audio_b64 length={len(audio_b64)} chars")
|
|
|
|
|
|
|
| 536 |
|
| 537 |
+
# ── STEP 2: Validate Azure credentials ──────────────────────────────────
|
| 538 |
+
logger.info(f"🎤 [PRON] STEP 2: Checking Azure credentials...")
|
| 539 |
+
logger.info(f"🎤 [PRON] AZURE_SPEECH_KEY present: {bool(AZURE_SPEECH_KEY)} | length: {len(AZURE_SPEECH_KEY) if AZURE_SPEECH_KEY else 0}")
|
| 540 |
+
logger.info(f"🎤 [PRON] AZURE_SPEECH_REGION: '{AZURE_SPEECH_REGION}'")
|
| 541 |
|
| 542 |
+
if not AZURE_SPEECH_KEY:
|
| 543 |
+
logger.error("❌ [PRON] STEP 2 FAILED: AZURE_SPEECH_KEY env var is not set")
|
| 544 |
+
emit('pronunciation_result', {
|
| 545 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 546 |
+
"recognized_text": "", "word_details": [],
|
| 547 |
+
"feedback": "Azure Speech key not configured on server.",
|
| 548 |
+
})
|
| 549 |
+
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 550 |
|
| 551 |
+
if not AZURE_SPEECH_REGION:
|
| 552 |
+
logger.error("❌ [PRON] STEP 2 FAILED: AZURE_SPEECH_REGION env var is not set")
|
| 553 |
+
emit('pronunciation_result', {
|
| 554 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 555 |
+
"recognized_text": "", "word_details": [],
|
| 556 |
+
"feedback": "Azure Speech region not configured on server.",
|
| 557 |
+
})
|
| 558 |
+
return
|
| 559 |
|
| 560 |
+
logger.info(f"🎤 [PRON] STEP 2 OK: Azure credentials present")
|
| 561 |
|
| 562 |
+
raw_path = None
|
| 563 |
+
clean_path = None
|
| 564 |
+
|
| 565 |
+
try:
|
| 566 |
+
# ── STEP 3: Decode base64 audio ──────────────────────────────────────
|
| 567 |
+
logger.info("🎤 [PRON] STEP 3: Decoding base64 audio...")
|
| 568 |
+
try:
|
| 569 |
+
if "," in audio_b64:
|
| 570 |
+
header, audio_b64 = audio_b64.split(",", 1)
|
| 571 |
+
logger.info(f"🎤 [PRON] Stripped data URI header: '{header[:50]}'")
|
| 572 |
+
audio_bytes = base64.b64decode(audio_b64)
|
| 573 |
+
logger.info(f"🎤 [PRON] STEP 3 OK: Decoded {len(audio_bytes)} raw bytes")
|
| 574 |
+
except Exception as e:
|
| 575 |
+
logger.error(f"❌ [PRON] STEP 3 FAILED: base64 decode error: {e}")
|
| 576 |
+
raise
|
| 577 |
+
|
| 578 |
+
if len(audio_bytes) < 100:
|
| 579 |
+
logger.error(f"❌ [PRON] STEP 3 FAILED: Decoded audio is suspiciously small ({len(audio_bytes)} bytes)")
|
| 580 |
+
raise Exception(f"Audio payload too small: {len(audio_bytes)} bytes")
|
| 581 |
+
|
| 582 |
+
# ── STEP 4: Write to temp file ────────────────────────────────────────
|
| 583 |
+
logger.info("🎤 [PRON] STEP 4: Writing audio to temp file...")
|
| 584 |
+
try:
|
| 585 |
+
with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as temp_raw:
|
| 586 |
+
temp_raw.write(audio_bytes)
|
| 587 |
+
raw_path = temp_raw.name
|
| 588 |
+
logger.info(f"🎤 [PRON] STEP 4 OK: Wrote to {raw_path} ({os.path.getsize(raw_path)} bytes)")
|
| 589 |
+
except Exception as e:
|
| 590 |
+
logger.error(f"❌ [PRON] STEP 4 FAILED: Could not write temp file: {e}")
|
| 591 |
+
raise
|
| 592 |
+
|
| 593 |
+
# ── STEP 5: FFmpeg conversion ─────────────────────────────────────────
|
| 594 |
+
logger.info("🎤 [PRON] STEP 5: Running FFmpeg conversion to 16kHz mono PCM WAV...")
|
| 595 |
+
clean_path = sanitize_audio(raw_path)
|
| 596 |
+
if not clean_path:
|
| 597 |
+
logger.error("❌ [PRON] STEP 5 FAILED: sanitize_audio() returned None — see FFmpeg logs above")
|
| 598 |
+
raise Exception("Audio conversion failed (FFmpeg error)")
|
| 599 |
+
logger.info(f"🎤 [PRON] STEP 5 OK: Clean WAV at {clean_path}")
|
| 600 |
+
|
| 601 |
+
# ── STEP 6: Analyze WAV integrity ─────────────────────────────────────
|
| 602 |
+
logger.info("🎤 [PRON] STEP 6: Analyzing WAV file integrity and volume...")
|
| 603 |
+
audio_ok = analyze_audio_volume(clean_path)
|
| 604 |
+
if not audio_ok:
|
| 605 |
+
logger.warning("⚠️ [PRON] STEP 6 WARNING: Audio appears silent — Azure will likely return NoMatch")
|
| 606 |
+
# Don't raise — let Azure try anyway, it gives a cleaner error
|
| 607 |
+
else:
|
| 608 |
+
logger.info("🎤 [PRON] STEP 6 OK: WAV has audible signal")
|
| 609 |
+
|
| 610 |
+
# ── STEP 7: Build Azure SpeechConfig ─────────────────────────────────
|
| 611 |
+
logger.info("🎤 [PRON] STEP 7: Building Azure SpeechConfig...")
|
| 612 |
+
try:
|
| 613 |
+
speech_config = speechsdk.SpeechConfig(
|
| 614 |
+
subscription=AZURE_SPEECH_KEY,
|
| 615 |
+
region=AZURE_SPEECH_REGION
|
| 616 |
+
)
|
| 617 |
+
speech_config.speech_recognition_language = lang
|
| 618 |
+
logger.info(f"🎤 [PRON] STEP 7 OK: SpeechConfig built — region={AZURE_SPEECH_REGION} lang={lang}")
|
| 619 |
+
except Exception as e:
|
| 620 |
+
logger.error(f"❌ [PRON] STEP 7 FAILED: SpeechConfig construction error: {e}")
|
| 621 |
+
raise
|
| 622 |
+
|
| 623 |
+
# ── STEP 8: Build AudioConfig ─────────────────────────────────────────
|
| 624 |
+
logger.info(f"🎤 [PRON] STEP 8: Building AudioConfig from file: {clean_path}")
|
| 625 |
+
try:
|
| 626 |
+
audio_config = speechsdk.audio.AudioConfig(filename=clean_path)
|
| 627 |
+
logger.info("🎤 [PRON] STEP 8 OK: AudioConfig built")
|
| 628 |
+
except Exception as e:
|
| 629 |
+
logger.error(f"❌ [PRON] STEP 8 FAILED: AudioConfig construction error: {e}")
|
| 630 |
+
raise
|
| 631 |
+
|
| 632 |
+
# ── STEP 9: Build PronunciationAssessmentConfig ───────────────────────
|
| 633 |
+
logger.info(f"🎤 [PRON] STEP 9: Building PronunciationAssessmentConfig for text: '{ref_text}'")
|
| 634 |
+
try:
|
| 635 |
+
pronunciation_config = speechsdk.PronunciationAssessmentConfig(
|
| 636 |
+
reference_text=ref_text,
|
| 637 |
+
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
|
| 638 |
+
granularity=speechsdk.PronunciationAssessmentGranularity.Word,
|
| 639 |
+
enable_miscue=True
|
| 640 |
+
)
|
| 641 |
+
logger.info("🎤 [PRON] STEP 9 OK: PronunciationAssessmentConfig built")
|
| 642 |
+
except Exception as e:
|
| 643 |
+
logger.error(f"❌ [PRON] STEP 9 FAILED: PronunciationAssessmentConfig error: {e}")
|
| 644 |
+
raise
|
| 645 |
+
|
| 646 |
+
# ── STEP 10: Build SpeechRecognizer ──────────────────────────────────
|
| 647 |
+
logger.info("🎤 [PRON] STEP 10: Building SpeechRecognizer...")
|
| 648 |
+
try:
|
| 649 |
+
recognizer = speechsdk.SpeechRecognizer(
|
| 650 |
+
speech_config=speech_config,
|
| 651 |
+
audio_config=audio_config
|
| 652 |
+
)
|
| 653 |
+
pronunciation_config.apply_to(recognizer)
|
| 654 |
+
logger.info("🎤 [PRON] STEP 10 OK: SpeechRecognizer built, pronunciation config applied")
|
| 655 |
+
except Exception as e:
|
| 656 |
+
logger.error(f"❌ [PRON] STEP 10 FAILED: SpeechRecognizer construction error: {e}")
|
| 657 |
+
raise
|
| 658 |
+
|
| 659 |
+
# ── STEP 11: Call Azure (the actual network call) ─────────────────────
|
| 660 |
+
logger.info("🎤 [PRON] STEP 11: Calling Azure recognize_once_async()... (network call)")
|
| 661 |
+
try:
|
| 662 |
+
result = recognizer.recognize_once_async().get()
|
| 663 |
+
logger.info(f"🎤 [PRON] STEP 11 OK: Azure returned result")
|
| 664 |
+
logger.info(f"🎤 [PRON] result.reason = {result.reason}")
|
| 665 |
+
logger.info(f"🎤 [PRON] result.text = '{result.text}'")
|
| 666 |
+
except Exception as e:
|
| 667 |
+
logger.error(f"❌ [PRON] STEP 11 FAILED: recognize_once_async() threw: {e}")
|
| 668 |
+
raise
|
| 669 |
+
|
| 670 |
+
# ── STEP 12: Parse result ─────────────────────────────────────────────
|
| 671 |
+
logger.info("🎤 [PRON] STEP 12: Parsing Azure result...")
|
| 672 |
response = {}
|
| 673 |
+
|
| 674 |
if result.reason == speechsdk.ResultReason.RecognizedSpeech:
|
| 675 |
+
logger.info("🎤 [PRON] STEP 12: Result reason = RecognizedSpeech ✅")
|
| 676 |
+
try:
|
| 677 |
+
pron_result = speechsdk.PronunciationAssessmentResult(result)
|
| 678 |
+
accuracy = pron_result.accuracy_score
|
| 679 |
+
fluency = pron_result.fluency_score
|
| 680 |
+
completeness = pron_result.completeness_score
|
| 681 |
+
logger.info(f"🎤 [PRON] Scores → accuracy={accuracy:.1f} fluency={fluency:.1f} completeness={completeness:.1f}")
|
| 682 |
+
except Exception as e:
|
| 683 |
+
logger.error(f"❌ [PRON] STEP 12 FAILED: PronunciationAssessmentResult parsing error: {e}")
|
| 684 |
+
raise
|
| 685 |
|
| 686 |
detailed_words = []
|
| 687 |
for word in pron_result.words:
|
| 688 |
+
w = {"word": word.word, "score": word.accuracy_score, "error": word.error_type}
|
| 689 |
+
detailed_words.append(w)
|
| 690 |
+
logger.info(f"🎤 [PRON] Word: '{word.word}' score={word.accuracy_score:.1f} error='{word.error_type}'")
|
| 691 |
+
|
| 692 |
+
feedback = _build_pronunciation_feedback(accuracy, fluency, completeness, detailed_words, ref_text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 693 |
|
| 694 |
response = {
|
| 695 |
+
"success": True,
|
| 696 |
+
"score": accuracy,
|
| 697 |
+
"fluency": fluency,
|
| 698 |
+
"completeness": completeness,
|
| 699 |
"recognized_text": result.text,
|
| 700 |
+
"word_details": detailed_words,
|
| 701 |
+
"feedback": feedback,
|
| 702 |
+
"question_id": data.get("question_id"),
|
| 703 |
}
|
| 704 |
|
|
|
|
| 705 |
if learner and grammar_rule and accuracy >= 70:
|
| 706 |
learner.record_outcome(grammar_rule, True, "speak")
|
| 707 |
response["mastery_update"] = dict(learner.mastery)
|
| 708 |
|
| 709 |
+
logger.info(f"✅ [PRON] STEP 12 OK: Assessment complete — acc={accuracy:.1f}")
|
| 710 |
|
| 711 |
elif result.reason == speechsdk.ResultReason.NoMatch:
|
| 712 |
+
no_match_detail = result.no_match_details if hasattr(result, 'no_match_details') else 'N/A'
|
| 713 |
+
logger.warning(f"⚠️ [PRON] STEP 12: Result reason = NoMatch — Azure heard nothing useful")
|
| 714 |
+
logger.warning(f"⚠️ [PRON] NoMatch details: {no_match_detail}")
|
| 715 |
response = {
|
| 716 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
|
|
|
|
|
|
|
|
|
| 717 |
"recognized_text": "",
|
| 718 |
"word_details": [],
|
| 719 |
+
"feedback": "I couldn't hear you clearly. Please check your microphone and try speaking again.",
|
| 720 |
}
|
| 721 |
+
|
| 722 |
+
elif result.reason == speechsdk.ResultReason.Canceled:
|
| 723 |
+
try:
|
| 724 |
+
cancellation = speechsdk.CancellationDetails(result)
|
| 725 |
+
logger.error(f"❌ [PRON] STEP 12: Result reason = Canceled")
|
| 726 |
+
logger.error(f"❌ [PRON] Cancellation reason: {cancellation.reason}")
|
| 727 |
+
logger.error(f"❌ [PRON] Cancellation error code: {cancellation.error_code}")
|
| 728 |
+
logger.error(f"❌ [PRON] Cancellation error details: {cancellation.error_details}")
|
| 729 |
+
|
| 730 |
+
# Common cancellation codes
|
| 731 |
+
if cancellation.reason == speechsdk.CancellationReason.Error:
|
| 732 |
+
if "401" in str(cancellation.error_details):
|
| 733 |
+
logger.error("❌ [PRON] HTTP 401 — Azure key is invalid or expired")
|
| 734 |
+
elif "403" in str(cancellation.error_details):
|
| 735 |
+
logger.error("❌ [PRON] HTTP 403 — Azure key doesn't have access to this region or feature")
|
| 736 |
+
elif "connection" in str(cancellation.error_details).lower():
|
| 737 |
+
logger.error("❌ [PRON] Connection error — Hugging Face Space may not have network access to Azure endpoint")
|
| 738 |
+
|
| 739 |
+
response = {
|
| 740 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 741 |
+
"recognized_text": "",
|
| 742 |
+
"word_details": [],
|
| 743 |
+
"feedback": f"Recognition canceled: {cancellation.error_details}",
|
| 744 |
+
}
|
| 745 |
+
except Exception as parse_e:
|
| 746 |
+
logger.error(f"❌ [PRON] Could not parse CancellationDetails: {parse_e}")
|
| 747 |
+
response = {
|
| 748 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 749 |
+
"recognized_text": "", "word_details": [],
|
| 750 |
+
"feedback": "Recognition was canceled by Azure.",
|
| 751 |
+
}
|
| 752 |
+
|
| 753 |
else:
|
| 754 |
+
logger.error(f"❌ [PRON] STEP 12: Unexpected result reason: {result.reason}")
|
| 755 |
response = {
|
| 756 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 757 |
+
"recognized_text": "", "word_details": [],
|
| 758 |
+
"feedback": f"Unexpected recognition result: {result.reason}",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 759 |
}
|
| 760 |
|
| 761 |
+
logger.info("🎤 [PRON] ── Assessment End ──")
|
| 762 |
+
logger.info("=" * 60)
|
| 763 |
emit('pronunciation_result', response)
|
| 764 |
|
| 765 |
except Exception as e:
|
| 766 |
+
logger.error(f"❌ [PRON] UNHANDLED EXCEPTION in handle_pronunciation: {type(e).__name__}: {e}")
|
| 767 |
+
import traceback
|
| 768 |
+
logger.error(f"❌ [PRON] Traceback:\n{traceback.format_exc()}")
|
| 769 |
emit('pronunciation_result', {
|
| 770 |
+
"success": False, "score": 0, "fluency": 0, "completeness": 0,
|
| 771 |
+
"recognized_text": "", "word_details": [],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 772 |
"feedback": "Server error during assessment.",
|
| 773 |
})
|
| 774 |
finally:
|
| 775 |
if raw_path and os.path.exists(raw_path):
|
| 776 |
os.remove(raw_path)
|
| 777 |
+
logger.info(f"🧹 [PRON] Cleaned up raw file: {raw_path}")
|
| 778 |
if clean_path and os.path.exists(clean_path):
|
| 779 |
os.remove(clean_path)
|
| 780 |
+
logger.info(f"🧹 [PRON] Cleaned up clean WAV: {clean_path}")
|
| 781 |
|
| 782 |
|
| 783 |
def _build_pronunciation_feedback(accuracy: float, fluency: float,
|
| 784 |
completeness: float, words: list,
|
| 785 |
ref_text: str) -> str:
|
|
|
|
| 786 |
issues = [w for w in words if w.get("error") not in (None, "None", "") or w.get("score", 100) < 60]
|
| 787 |
|
| 788 |
if accuracy >= 85:
|
|
|
|
| 810 |
|
| 811 |
@socketio.on('get_mastery')
|
| 812 |
def handle_get_mastery(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 813 |
from flask import request as req
|
| 814 |
learner = get_learner(req.sid)
|
| 815 |
|
|
|
|
| 822 |
|
| 823 |
@socketio.on('restore_session')
|
| 824 |
def handle_restore_session(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
from flask import request as req
|
| 826 |
sid = req.sid
|
| 827 |
|
|
|
|
| 836 |
logger.info(f"♻️ Session restored for {learner_id}: difficulty={learner.difficulty}")
|
| 837 |
|
| 838 |
emit('session_restored', {
|
| 839 |
+
"success": True,
|
| 840 |
+
"session_id": learner_id,
|
| 841 |
+
"mastery": learner.mastery,
|
| 842 |
+
"difficulty": learner.difficulty,
|
| 843 |
"question_count": learner.question_count,
|
| 844 |
})
|
| 845 |
|
|
|
|
| 850 |
|
| 851 |
@socketio.on('reset_session')
|
| 852 |
def handle_reset_session(data):
|
|
|
|
| 853 |
from flask import request as req
|
| 854 |
+
sid = req.sid
|
| 855 |
learner = get_learner(sid)
|
| 856 |
|
| 857 |
if learner:
|
| 858 |
learner.reset()
|
| 859 |
logger.info(f"🔄 Session reset: {learner.session_id}")
|
| 860 |
emit('session_reset', {
|
| 861 |
+
"success": True,
|
| 862 |
+
"mastery": learner.mastery,
|
| 863 |
"difficulty": learner.difficulty,
|
| 864 |
})
|
| 865 |
else:
|
|
|
|
| 868 |
|
| 869 |
@socketio.on('update_mastery')
|
| 870 |
def handle_update_mastery(data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 871 |
from flask import request as req
|
| 872 |
learner = get_learner(req.sid)
|
| 873 |
|
|
|
|
| 876 |
return
|
| 877 |
|
| 878 |
grammar_rule = data.get("grammar_rule", "")
|
| 879 |
+
correct = data.get("correct", False)
|
| 880 |
+
mode = data.get("interaction_mode", "")
|
| 881 |
|
| 882 |
if grammar_rule:
|
| 883 |
learner.record_outcome(grammar_rule, correct, mode)
|
| 884 |
|
| 885 |
emit('mastery_updated', {
|
| 886 |
+
"mastery": learner.mastery,
|
| 887 |
"difficulty": learner.difficulty,
|
| 888 |
+
"streak": learner.streak,
|
| 889 |
})
|
| 890 |
|
| 891 |
|
| 892 |
# ===========================================================================
|
| 893 |
+
# 6. VISUAL RECOGNITION
|
| 894 |
# ===========================================================================
|
| 895 |
|
| 896 |
@socketio.on('verify_object')
|
|
|
|
| 911 |
schema = {
|
| 912 |
"type": "OBJECT",
|
| 913 |
"properties": {
|
| 914 |
+
"verified": {"type": "BOOLEAN"},
|
| 915 |
"confidence": {"type": "NUMBER"},
|
| 916 |
+
"feedback": {"type": "STRING"}
|
| 917 |
},
|
| 918 |
"required": ["verified", "feedback"]
|
| 919 |
}
|
|
|
|
| 943 |
|
| 944 |
|
| 945 |
# ===========================================================================
|
| 946 |
+
# 7. HANDWRITING / OCR
|
| 947 |
# ===========================================================================
|
| 948 |
|
| 949 |
@socketio.on('verify_writing')
|
|
|
|
| 964 |
schema = {
|
| 965 |
"type": "OBJECT",
|
| 966 |
"properties": {
|
| 967 |
+
"correct": {"type": "BOOLEAN"},
|
| 968 |
"detected_text": {"type": "STRING"},
|
| 969 |
+
"feedback": {"type": "STRING"}
|
| 970 |
},
|
| 971 |
"required": ["correct", "detected_text"]
|
| 972 |
}
|
|
|
|
| 995 |
|
| 996 |
|
| 997 |
# ===========================================================================
|
| 998 |
+
# 8. GRAMMAR RULE INFO
|
| 999 |
# ===========================================================================
|
| 1000 |
|
| 1001 |
@socketio.on('get_grammar_rules')
|
| 1002 |
def handle_get_grammar_rules(data):
|
|
|
|
| 1003 |
pack = get_active_pack()
|
| 1004 |
emit('grammar_rules', {
|
| 1005 |
+
"rules": pack.get("grammar_rules", {}),
|
| 1006 |
"lesson": pack.get("lesson"),
|
| 1007 |
})
|
| 1008 |
|
| 1009 |
|
| 1010 |
@socketio.on('get_content_pack_info')
|
| 1011 |
def handle_get_content_pack_info(data):
|
|
|
|
| 1012 |
pack = get_active_pack()
|
| 1013 |
emit('content_pack_info', {
|
| 1014 |
+
"lesson": pack.get("lesson"),
|
| 1015 |
+
"version": pack.get("version"),
|
| 1016 |
+
"vocab_count": len(pack.get("vocab", [])),
|
| 1017 |
"grammar_rules": list(pack.get("grammar_rules", {}).keys()),
|
| 1018 |
+
"metadata": pack.get("metadata", {}),
|
| 1019 |
})
|
| 1020 |
|
| 1021 |
|
|
|
|
| 1024 |
# ===========================================================================
|
| 1025 |
|
| 1026 |
if __name__ == '__main__':
|
|
|
|
| 1027 |
purge_stale_sessions()
|
| 1028 |
logger.info("🚀 KLP AI Service starting on port 7860")
|
|
|
|
| 1029 |
socketio.run(app, host='0.0.0.0', port=7860)
|