Yu Chen commited on
Commit
1ce0b71
·
1 Parent(s): ad5d4b6

fix the answer parsing not aligned logic

Browse files
chatkit/backend/app/answer_grid.py CHANGED
@@ -15,7 +15,10 @@ import re
15
  from dataclasses import dataclass
16
  from typing import Optional
17
 
18
- _BLANK_MARKERS = {"", "-", "–", "—", "n/a", "none", "null"}
 
 
 
19
  _LETTER_RE = re.compile(r"^[A-Z]$")
20
 
21
 
@@ -48,12 +51,19 @@ class WrongAnswer:
48
 
49
 
50
  def normalize_letter(raw: Optional[str]) -> Optional[str]:
51
- """Normalize a raw cell to a single uppercase letter or None."""
 
 
 
 
 
52
  if raw is None:
53
  return None
54
  s = str(raw).strip()
55
- if s.casefold() in _BLANK_MARKERS:
56
  return None
 
 
57
  s = s.upper()
58
  # Tolerate "A)" or "A) foo" -> "A"
59
  if len(s) > 1 and s[0].isalpha() and s[1] in (")", ".", "、", " "):
@@ -147,7 +157,7 @@ def seed_from_parsed(
147
  - N is derived from questions_data if present, else from teacher_answers length,
148
  else from the longest student row.
149
  - Student rows are padded/truncated to N.
150
- - Cells are normalized to letters or None.
151
  """
152
  q_list = (questions_data or {}).get("questions") or []
153
  t_list = (teacher_answers_data or {}).get("answers") or []
@@ -175,7 +185,8 @@ def seed_from_parsed(
175
  if isinstance(qn, int) and 1 <= qn <= n:
176
  official[qn - 1] = normalize_letter(a.get("correct_answer"))
177
 
178
- # Build student rows by question_number index
 
179
  students: list[StudentRow] = []
180
  for i, s in enumerate(s_list):
181
  row: list[Optional[str]] = [None] * n
@@ -207,17 +218,20 @@ def seed_from_parsed(
207
  def diff_student(grid: AnswerGrid, student_index: int) -> list[WrongAnswer]:
208
  """Return the student's wrong answers ordered by question number.
209
 
210
- Skips questions where the official key is blank (None).
 
 
211
  """
212
  if student_index < 0 or student_index >= len(grid.students):
213
  raise IndexError(f"student_index {student_index} out of range")
214
  row = grid.students[student_index].answers
215
  wrongs: list[WrongAnswer] = []
216
  for i, correct in enumerate(grid.official_answers):
217
- if correct is None:
218
  continue
219
  student_ans = row[i] if i < len(row) else None
220
- # Blank student answer is always wrong when a key exists
 
221
  if student_ans != correct:
222
  wrongs.append(
223
  WrongAnswer(
 
15
  from dataclasses import dataclass
16
  from typing import Optional
17
 
18
+ _BLANK_MARKERS = {"", "n/a", "none", "null"}
19
+ _CORRECT_MARKER = "="
20
+ # Legacy dash markers are accepted as input and converted to CORRECT_MARKER
21
+ _DASH_MARKERS = {"-", "–", "—"}
22
  _LETTER_RE = re.compile(r"^[A-Z]$")
23
 
24
 
 
51
 
52
 
53
  def normalize_letter(raw: Optional[str]) -> Optional[str]:
54
+ """Normalize a raw cell to a letter (A-Z), the correct-marker "=", or None.
55
+
56
+ - "=" or any legacy dash marker ("-", "–", "—") → "=" (answered correctly)
57
+ - Single letters with optional trailing "A) foo" → uppercase letter
58
+ - Empty/blank/unparseable → None
59
+ """
60
  if raw is None:
61
  return None
62
  s = str(raw).strip()
63
+ if not s or s.casefold() in _BLANK_MARKERS:
64
  return None
65
+ if s == _CORRECT_MARKER or s in _DASH_MARKERS:
66
+ return _CORRECT_MARKER
67
  s = s.upper()
68
  # Tolerate "A)" or "A) foo" -> "A"
69
  if len(s) > 1 and s[0].isalpha() and s[1] in (")", ".", "、", " "):
 
157
  - N is derived from questions_data if present, else from teacher_answers length,
158
  else from the longest student row.
159
  - Student rows are padded/truncated to N.
160
+ - Cells are normalized to letters, the correct-marker "=", or None.
161
  """
162
  q_list = (questions_data or {}).get("questions") or []
163
  t_list = (teacher_answers_data or {}).get("answers") or []
 
185
  if isinstance(qn, int) and 1 <= qn <= n:
186
  official[qn - 1] = normalize_letter(a.get("correct_answer"))
187
 
188
+ # Build student rows by question_number index. Legacy "-" markers are
189
+ # mapped to the canonical "=" correct-marker by normalize_letter.
190
  students: list[StudentRow] = []
191
  for i, s in enumerate(s_list):
192
  row: list[Optional[str]] = [None] * n
 
218
  def diff_student(grid: AnswerGrid, student_index: int) -> list[WrongAnswer]:
219
  """Return the student's wrong answers ordered by question number.
220
 
221
+ - Skips questions where the official key is blank (None) or "=".
222
+ - Skips questions where the student's cell is "=" (answered correctly).
223
+ - Otherwise flags any cell that differs from the official letter.
224
  """
225
  if student_index < 0 or student_index >= len(grid.students):
226
  raise IndexError(f"student_index {student_index} out of range")
227
  row = grid.students[student_index].answers
228
  wrongs: list[WrongAnswer] = []
229
  for i, correct in enumerate(grid.official_answers):
230
+ if correct is None or correct == _CORRECT_MARKER:
231
  continue
232
  student_ans = row[i] if i < len(row) else None
233
+ if student_ans == _CORRECT_MARKER:
234
+ continue
235
  if student_ans != correct:
236
  wrongs.append(
237
  WrongAnswer(
chatkit/backend/app/file_processor.py CHANGED
@@ -52,15 +52,15 @@ IMPORTANT - Images, Diagrams, Charts, and Figures:
52
  EXTRACTION_PROMPTS = {
53
  "questions": """You are an exam data extraction expert. Extract all exam questions from the provided content.
54
 
 
 
55
  Return a JSON object with this structure:
56
  {
57
  "questions": [
58
  {
59
  "number": 1,
60
  "text": "the full question text (including [圖片描述: ...] for any images/diagrams)",
61
- "type": "multiple_choice" or "short_answer" or "essay" or "true_false",
62
- "options": ["A) ...", "B) ...", "C) ...", "D) ..."], // null if not multiple choice
63
- "points": 10 // point value if visible, null otherwise
64
  }
65
  ]
66
  }
@@ -68,12 +68,14 @@ Return a JSON object with this structure:
68
  Rules:
69
  - Extract EVERY question you can see
70
  - Preserve the original language of the questions
71
- - If options are present, include them exactly as written
72
- - If point values are shown, include them
73
  """ + IMAGE_DESCRIPTION_RULE + NO_FABRICATION_RULE + "- Return valid JSON only",
74
 
75
  "student_answers": """You are an exam data extraction expert. Extract all student answers from the provided content.
76
 
 
 
77
  Return a JSON object with this structure:
78
  {
79
  "students": [
@@ -83,7 +85,7 @@ Return a JSON object with this structure:
83
  "answers": [
84
  {
85
  "question_number": 1,
86
- "answer": "the student's answer text"
87
  }
88
  ]
89
  }
@@ -92,20 +94,23 @@ Return a JSON object with this structure:
92
 
93
  Rules:
94
  - Extract answers for EVERY student visible
95
- - Preserve exact answer text
96
- - If a student left a question blank, set answer to null
 
 
97
  - Use the student's name or ID as shown in the document
98
- - If a student's answer includes a drawing or diagram, describe it as text: [圖片描述: ...]
99
- """ + IMAGE_DESCRIPTION_RULE + NO_FABRICATION_RULE + "- Return valid JSON only",
100
 
101
  "teacher_answers": """You are an exam data extraction expert. Extract the correct/model answers (answer key) from the provided content.
102
 
 
 
103
  Return a JSON object with this structure:
104
  {
105
  "answers": [
106
  {
107
  "question_number": 1,
108
- "correct_answer": "the correct answer",
109
  "explanation": "explanation if provided, otherwise null"
110
  }
111
  ]
@@ -113,10 +118,9 @@ Return a JSON object with this structure:
113
 
114
  Rules:
115
  - Extract the correct answer for EVERY question
 
116
  - Include explanations if they are provided
117
- - Preserve exact answer text
118
- - If the answer references or includes an image/diagram, describe it as text: [圖片描述: ...]
119
- """ + IMAGE_DESCRIPTION_RULE + NO_FABRICATION_RULE + "- Return valid JSON only",
120
  }
121
 
122
 
 
52
  EXTRACTION_PROMPTS = {
53
  "questions": """You are an exam data extraction expert. Extract all exam questions from the provided content.
54
 
55
+ This system supports SINGLE-CHOICE questions only (one correct letter: A, B, C, D, ...).
56
+
57
  Return a JSON object with this structure:
58
  {
59
  "questions": [
60
  {
61
  "number": 1,
62
  "text": "the full question text (including [圖片描述: ...] for any images/diagrams)",
63
+ "options": ["A) ...", "B) ...", "C) ...", "D) ..."]
 
 
64
  }
65
  ]
66
  }
 
68
  Rules:
69
  - Extract EVERY question you can see
70
  - Preserve the original language of the questions
71
+ - Always include the options exactly as written; every question must have options
72
+ - Do NOT include fields other than number/text/options
73
  """ + IMAGE_DESCRIPTION_RULE + NO_FABRICATION_RULE + "- Return valid JSON only",
74
 
75
  "student_answers": """You are an exam data extraction expert. Extract all student answers from the provided content.
76
 
77
+ This system supports SINGLE-CHOICE questions only.
78
+
79
  Return a JSON object with this structure:
80
  {
81
  "students": [
 
85
  "answers": [
86
  {
87
  "question_number": 1,
88
+ "answer": "A" | "B" | "C" | ... | "=" | null
89
  }
90
  ]
91
  }
 
94
 
95
  Rules:
96
  - Extract answers for EVERY student visible
97
+ - Answer format (use exactly one of these):
98
+ - A single uppercase letter (A, B, C, D, ...) — the letter the student chose
99
+ - "=" — the student answered correctly (e.g., source uses "-" / "✓" / "correct" / matches the answer key)
100
+ - null — the student left the question blank
101
  - Use the student's name or ID as shown in the document
102
+ """ + NO_FABRICATION_RULE + "- Return valid JSON only",
 
103
 
104
  "teacher_answers": """You are an exam data extraction expert. Extract the correct/model answers (answer key) from the provided content.
105
 
106
+ This system supports SINGLE-CHOICE questions only.
107
+
108
  Return a JSON object with this structure:
109
  {
110
  "answers": [
111
  {
112
  "question_number": 1,
113
+ "correct_answer": "A" | "B" | "C" | ...,
114
  "explanation": "explanation if provided, otherwise null"
115
  }
116
  ]
 
118
 
119
  Rules:
120
  - Extract the correct answer for EVERY question
121
+ - correct_answer MUST be a single uppercase letter (A, B, C, D, ...)
122
  - Include explanations if they are provided
123
+ """ + NO_FABRICATION_RULE + "- Return valid JSON only",
 
 
124
  }
125
 
126
 
chatkit/backend/tests/test_answer_grid.py CHANGED
@@ -30,10 +30,12 @@ from app.answer_grid import (
30
  (" b ", "B"),
31
  ("C)", "C"),
32
  ("D) foo", "D"),
 
 
 
 
 
33
  ("", None),
34
- ("-", None),
35
- ("–", None),
36
- ("—", None),
37
  (None, None),
38
  (" ", None),
39
  ("N/A", None),
@@ -184,17 +186,41 @@ def test_seed_from_parsed_basic():
184
 
185
 
186
  @pytest.mark.unit
187
- def test_seed_from_parsed_handles_dash_as_blank():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  questions = {"questions": [{"number": 1, "text": "Q"}]}
189
  students = {
190
  "students": [
191
- {"name": "Alice", "answers": [{"question_number": 1, "answer": "-"}]}
192
  ]
193
  }
194
  teachers = {"answers": [{"question_number": 1, "correct_answer": "A"}]}
195
  grid = seed_from_parsed(questions, students, teachers)
196
- # Dash normalizes to None — diff treats None != "A" as wrong
197
- assert grid.students[0].answers == (None,)
198
 
199
 
200
  @pytest.mark.unit
@@ -263,6 +289,39 @@ def test_diff_student_blank_student_answer_is_wrong():
263
  assert wrongs[0].student_answer is None
264
 
265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  @pytest.mark.unit
267
  def test_diff_student_skips_blank_official_key():
268
  grid = AnswerGrid(
 
30
  (" b ", "B"),
31
  ("C)", "C"),
32
  ("D) foo", "D"),
33
+ ("=", "="),
34
+ (" = ", "="),
35
+ ("-", "="),
36
+ ("–", "="),
37
+ ("—", "="),
38
  ("", None),
 
 
 
39
  (None, None),
40
  (" ", None),
41
  ("N/A", None),
 
186
 
187
 
188
  @pytest.mark.unit
189
+ def test_seed_from_parsed_legacy_dash_becomes_correct_marker():
190
+ """Legacy '-' in a student answer normalizes to the canonical '=' marker."""
191
+ questions = {"questions": [{"number": 1, "text": "Q"}, {"number": 2, "text": "Q"}]}
192
+ students = {
193
+ "students": [
194
+ {
195
+ "name": "Alice",
196
+ "answers": [
197
+ {"question_number": 1, "answer": "-"},
198
+ {"question_number": 2, "answer": "D"},
199
+ ],
200
+ }
201
+ ]
202
+ }
203
+ teachers = {
204
+ "answers": [
205
+ {"question_number": 1, "correct_answer": "A"},
206
+ {"question_number": 2, "correct_answer": "B"},
207
+ ]
208
+ }
209
+ grid = seed_from_parsed(questions, students, teachers)
210
+ assert grid.students[0].answers == ("=", "D")
211
+
212
+
213
+ @pytest.mark.unit
214
+ def test_seed_from_parsed_preserves_explicit_equals():
215
  questions = {"questions": [{"number": 1, "text": "Q"}]}
216
  students = {
217
  "students": [
218
+ {"name": "Alice", "answers": [{"question_number": 1, "answer": "="}]}
219
  ]
220
  }
221
  teachers = {"answers": [{"question_number": 1, "correct_answer": "A"}]}
222
  grid = seed_from_parsed(questions, students, teachers)
223
+ assert grid.students[0].answers == ("=",)
 
224
 
225
 
226
  @pytest.mark.unit
 
289
  assert wrongs[0].student_answer is None
290
 
291
 
292
+ @pytest.mark.unit
293
+ def test_diff_student_equals_marker_is_correct():
294
+ """A '=' cell means 'student answered correctly' and must never appear in wrongs."""
295
+ grid = AnswerGrid(
296
+ total_questions=3,
297
+ official_answers=("A", "B", "C"),
298
+ students=(StudentRow(name="Alice", answers=("=", "X", "=")),),
299
+ questions=tuple(
300
+ QuestionMeta(number=i + 1, text=f"Q{i+1}", options=()) for i in range(3)
301
+ ),
302
+ )
303
+ wrongs = diff_student(grid, 0)
304
+ assert len(wrongs) == 1
305
+ assert wrongs[0].question.number == 2
306
+ assert wrongs[0].student_answer == "X"
307
+
308
+
309
+ @pytest.mark.unit
310
+ def test_diff_student_skips_equals_in_official():
311
+ grid = AnswerGrid(
312
+ total_questions=2,
313
+ official_answers=("A", "="), # Q2 has no authoritative key
314
+ students=(StudentRow(name="Alice", answers=("X", "Y")),),
315
+ questions=(
316
+ QuestionMeta(number=1, text="Q1", options=()),
317
+ QuestionMeta(number=2, text="Q2", options=()),
318
+ ),
319
+ )
320
+ wrongs = diff_student(grid, 0)
321
+ assert len(wrongs) == 1
322
+ assert wrongs[0].question.number == 1
323
+
324
+
325
  @pytest.mark.unit
326
  def test_diff_student_skips_blank_official_key():
327
  grid = AnswerGrid(
chatkit/frontend/src/components/step1/FileUploadPanel.tsx CHANGED
@@ -274,7 +274,7 @@ function ZoneSummary({ type, data }: { type: string; data: Record<string, unknow
274
 
275
  function QuestionsTable({ data }: { data?: Record<string, unknown> }) {
276
  if (!data) return null;
277
- const questions = (data as { questions?: Array<{ number: number; text: string; type?: string; options?: string[] | null; points?: number | null }> }).questions;
278
  if (!questions?.length) return null;
279
 
280
  return (
@@ -288,8 +288,6 @@ function QuestionsTable({ data }: { data?: Record<string, unknown> }) {
288
  <tr className="border-b border-[var(--color-border)]">
289
  <th className="text-left py-2 px-3 text-[var(--color-text-muted)] w-12">#</th>
290
  <th className="text-left py-2 px-3 text-[var(--color-text-muted)]">題目</th>
291
- <th className="text-left py-2 px-3 text-[var(--color-text-muted)] w-28">類型</th>
292
- <th className="text-left py-2 px-3 text-[var(--color-text-muted)] w-16">分數</th>
293
  </tr>
294
  </thead>
295
  <tbody>
@@ -304,10 +302,6 @@ function QuestionsTable({ data }: { data?: Record<string, unknown> }) {
304
  </div>
305
  )}
306
  </td>
307
- <td className="py-2 px-3">
308
- {q.type && <span className="badge badge-success text-xs">{q.type}</span>}
309
- </td>
310
- <td className="py-2 px-3 text-[var(--color-text-muted)]">{q.points ?? "-"}</td>
311
  </tr>
312
  ))}
313
  </tbody>
 
274
 
275
  function QuestionsTable({ data }: { data?: Record<string, unknown> }) {
276
  if (!data) return null;
277
+ const questions = (data as { questions?: Array<{ number: number; text: string; options?: string[] | null }> }).questions;
278
  if (!questions?.length) return null;
279
 
280
  return (
 
288
  <tr className="border-b border-[var(--color-border)]">
289
  <th className="text-left py-2 px-3 text-[var(--color-text-muted)] w-12">#</th>
290
  <th className="text-left py-2 px-3 text-[var(--color-text-muted)]">題目</th>
 
 
291
  </tr>
292
  </thead>
293
  <tbody>
 
302
  </div>
303
  )}
304
  </td>
 
 
 
 
305
  </tr>
306
  ))}
307
  </tbody>
chatkit/frontend/src/components/step2/AnswerGridEditor.tsx CHANGED
@@ -29,11 +29,16 @@ interface AnswerGridEditorProps {
29
  onConfirmedChange: (grid: AnswerGrid | null) => void;
30
  }
31
 
 
 
 
32
  function normalizeCell(raw: string): string | null {
33
- const trimmed = raw.trim().toUpperCase();
34
- if (!trimmed || trimmed === "-") return null;
35
- // Take first letter only, allow A-Z
36
- const first = trimmed.charAt(0);
 
 
37
  return /^[A-Z]$/.test(first) ? first : null;
38
  }
39
 
@@ -179,6 +184,7 @@ export function AnswerGridEditor({
179
  </h3>
180
  <p className="text-xs text-[var(--color-text-muted)] mt-1">
181
  請核對學生作答與標準答案,確認後才能生成報告。支援單選題(A-Z)。
 
182
  </p>
183
  </div>
184
  <div className="flex items-center gap-2">
@@ -217,20 +223,28 @@ export function AnswerGridEditor({
217
  <td className="py-1 px-3 border-b border-[var(--color-border)]/50 sticky left-0 bg-[var(--color-background)] z-10 font-medium text-[var(--color-text)]">
218
  {s.name}
219
  </td>
220
- {qNums.map((n, qi) => (
221
- <td
222
- key={n}
223
- className="border-b border-[var(--color-border)]/50 p-0.5"
224
- >
225
- <input
226
- type="text"
227
- maxLength={2}
228
- defaultValue={s.answers[qi] ?? ""}
229
- onBlur={(e) => updateStudentCell(si, qi, e.target.value)}
230
- className="w-full text-center py-1 bg-transparent text-[var(--color-text)] border border-transparent hover:border-[var(--color-border)] focus:border-[var(--color-primary)] focus:outline-none rounded"
231
- />
232
- </td>
233
- ))}
 
 
 
 
 
 
 
 
234
  </tr>
235
  ))}
236
 
 
29
  onConfirmedChange: (grid: AnswerGrid | null) => void;
30
  }
31
 
32
+ const CORRECT_MARKER = "=";
33
+ const DASH_MARKERS = new Set(["-", "–", "—"]);
34
+
35
  function normalizeCell(raw: string): string | null {
36
+ const trimmed = raw.trim();
37
+ if (!trimmed) return null;
38
+ if (trimmed === CORRECT_MARKER || DASH_MARKERS.has(trimmed)) {
39
+ return CORRECT_MARKER;
40
+ }
41
+ const first = trimmed.charAt(0).toUpperCase();
42
  return /^[A-Z]$/.test(first) ? first : null;
43
  }
44
 
 
184
  </h3>
185
  <p className="text-xs text-[var(--color-text-muted)] mt-1">
186
  請核對學生作答與標準答案,確認後才能生成報告。支援單選題(A-Z)。
187
+ 學生欄輸入 <code className="px-1 bg-[var(--color-background)] rounded">=</code> 代表該題答對。
188
  </p>
189
  </div>
190
  <div className="flex items-center gap-2">
 
223
  <td className="py-1 px-3 border-b border-[var(--color-border)]/50 sticky left-0 bg-[var(--color-background)] z-10 font-medium text-[var(--color-text)]">
224
  {s.name}
225
  </td>
226
+ {qNums.map((n, qi) => {
227
+ const cell = s.answers[qi];
228
+ const isCorrect = cell === CORRECT_MARKER;
229
+ return (
230
+ <td
231
+ key={n}
232
+ className="border-b border-[var(--color-border)]/50 p-0.5"
233
+ >
234
+ <input
235
+ type="text"
236
+ maxLength={2}
237
+ defaultValue={cell ?? ""}
238
+ onBlur={(e) => updateStudentCell(si, qi, e.target.value)}
239
+ className={`w-full text-center py-1 bg-transparent border border-transparent hover:border-[var(--color-border)] focus:border-[var(--color-primary)] focus:outline-none rounded ${
240
+ isCorrect
241
+ ? "text-[var(--color-success)] font-semibold"
242
+ : "text-[var(--color-text)]"
243
+ }`}
244
+ />
245
+ </td>
246
+ );
247
+ })}
248
  </tr>
249
  ))}
250