Kentlo commited on
Commit
db55e77
·
verified ·
1 Parent(s): 5f8be73

Sync from GitHub 567c1a39900defc0bb5a8dce17616ee3bb817643

Browse files
.gitattributes CHANGED
@@ -245,3 +245,7 @@ static/question_assets/AZ-104_dump_56cadaca/page_95.jpg filter=lfs diff=lfs merg
245
  static/question_assets/AZ-104_dump_56cadaca/page_97.jpg filter=lfs diff=lfs merge=lfs -text
246
  static/question_assets/AZ-104_dump_56cadaca/page_98.jpg filter=lfs diff=lfs merge=lfs -text
247
  static/question_assets/AZ-104_dump_56cadaca/page_99.jpg filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
245
  static/question_assets/AZ-104_dump_56cadaca/page_97.jpg filter=lfs diff=lfs merge=lfs -text
246
  static/question_assets/AZ-104_dump_56cadaca/page_98.jpg filter=lfs diff=lfs merge=lfs -text
247
  static/question_assets/AZ-104_dump_56cadaca/page_99.jpg filter=lfs diff=lfs merge=lfs -text
248
+ static/question_assets/AZ-104_dump_56cadaca/page_12.jpg filter=lfs diff=lfs merge=lfs -text
249
+ static/question_assets/AZ-104_dump_56cadaca/page_21.jpg filter=lfs diff=lfs merge=lfs -text
250
+ static/question_assets/AZ-104_dump_56cadaca/page_30.jpg filter=lfs diff=lfs merge=lfs -text
251
+ static/question_assets/AZ-104_dump_56cadaca/page_32.jpg filter=lfs diff=lfs merge=lfs -text
cert_study_app/demo_data/questions_seed.json CHANGED
The diff for this file is too large to render. See raw diff
 
cert_study_app/services/demo_seed_service.py CHANGED
@@ -30,48 +30,78 @@ def _json_text(value, fallback):
30
  return fallback
31
 
32
 
33
- def seed_demo_questions_if_empty(db) -> int:
34
- if os.getenv("CERT_STUDY_SEED_DEMO", "1") == "0":
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  return 0
36
 
37
- existing = db.query(func.count(Question.id)).scalar() or 0
38
- if existing:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  return 0
40
 
41
  with _seed_data_path().open("r", encoding="utf-8") as file:
42
  rows = json.load(file)
43
 
 
 
 
 
44
  inserted = 0
45
  for row in rows:
46
- question = Question(
47
- stem=row["stem"],
48
- answer=_json_text(row.get("answer"), None),
49
- explanation=row.get("explanation"),
50
- question_type=row.get("question_type", "MCQ"),
51
- question_number=row.get("question_number"),
52
- group_id=row.get("group_id"),
53
- parent_stem=row.get("parent_stem"),
54
- parent_image_paths=_json_text(row.get("parent_image_paths"), "[]"),
55
- page=row.get("page"),
56
- category=row.get("category"),
57
- subcategory=row.get("subcategory"),
58
- source=row.get("source") or DEMO_SOURCE,
59
- image_path=row.get("image_path"),
60
- raw_text=row.get("raw_text") or row.get("stem"),
61
- structured_data_json=json.dumps(row, ensure_ascii=False),
62
- parse_status=row.get("parse_status") or "approved",
63
- quality_score=row.get("quality_score") or 100,
64
- quality_status=row.get("quality_status") or "seed",
65
- quality_issues=_json_text(row.get("quality_issues"), "[]"),
66
- chunk_key=row.get("chunk_key"),
67
- chunk_index=row.get("chunk_index"),
68
- parser_version=row.get("parser_version") or "seed-v1",
69
- code=row.get("code"),
70
- pairs=_json_text(row.get("pairs"), None),
71
- sequence=_json_text(row.get("sequence"), None),
72
- )
73
- question.set_options(row.get("options") or [])
74
- question.set_concept_tags(row.get("concept_tags") or [])
75
  db.add(question)
76
  inserted += 1
77
 
 
30
  return fallback
31
 
32
 
33
+ def _apply_seed_row(question: Question, row: dict) -> None:
34
+ question.stem = row["stem"]
35
+ question.answer = _json_text(row.get("answer"), None)
36
+ question.explanation = row.get("explanation")
37
+ question.question_type = row.get("question_type", "MCQ")
38
+ question.question_number = row.get("question_number")
39
+ question.group_id = row.get("group_id")
40
+ question.parent_stem = row.get("parent_stem")
41
+ question.parent_image_paths = _json_text(row.get("parent_image_paths"), "[]")
42
+ question.page = row.get("page")
43
+ question.category = row.get("category")
44
+ question.subcategory = row.get("subcategory")
45
+ question.source = row.get("source") or DEMO_SOURCE
46
+ question.image_path = row.get("image_path")
47
+ question.raw_text = row.get("raw_text") or row.get("stem")
48
+ question.structured_data_json = json.dumps(row, ensure_ascii=False)
49
+ question.parse_status = row.get("parse_status") or "approved"
50
+ question.quality_score = row.get("quality_score") or 100
51
+ question.quality_status = row.get("quality_status") or "seed"
52
+ question.quality_issues = _json_text(row.get("quality_issues"), "[]")
53
+ question.chunk_key = row.get("chunk_key")
54
+ question.chunk_index = row.get("chunk_index")
55
+ question.parser_version = row.get("parser_version") or "seed-v1"
56
+ question.code = row.get("code")
57
+ question.pairs = _json_text(row.get("pairs"), None)
58
+ question.sequence = _json_text(row.get("sequence"), None)
59
+ question.set_options(row.get("options") or [])
60
+ question.set_concept_tags(row.get("concept_tags") or [])
61
+
62
+
63
+ def _refresh_seed_questions(db, rows: list[dict]) -> int:
64
+ if os.getenv("CERT_STUDY_REFRESH_SEED", "1") == "0":
65
  return 0
66
 
67
+ updated = 0
68
+ for row in rows:
69
+ question_number = row.get("question_number")
70
+ if not question_number:
71
+ continue
72
+
73
+ question = (
74
+ db.query(Question)
75
+ .filter(Question.question_number == question_number)
76
+ .order_by(Question.id.asc())
77
+ .first()
78
+ )
79
+ if not question:
80
+ continue
81
+
82
+ _apply_seed_row(question, row)
83
+ updated += 1
84
+
85
+ if updated:
86
+ db.commit()
87
+ return updated
88
+
89
+
90
+ def seed_demo_questions_if_empty(db) -> int:
91
+ if os.getenv("CERT_STUDY_SEED_DEMO", "1") == "0":
92
  return 0
93
 
94
  with _seed_data_path().open("r", encoding="utf-8") as file:
95
  rows = json.load(file)
96
 
97
+ existing = db.query(func.count(Question.id)).scalar() or 0
98
+ if existing:
99
+ return _refresh_seed_questions(db, rows)
100
+
101
  inserted = 0
102
  for row in rows:
103
+ question = Question()
104
+ _apply_seed_row(question, row)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  db.add(question)
106
  inserted += 1
107
 
scripts/export_hf_seed.py CHANGED
@@ -9,6 +9,7 @@ from typing import Any
9
 
10
  ROOT = Path(__file__).resolve().parent.parent
11
  DEFAULT_SOURCE_JSON = ROOT / "data" / "Json" / "questions.json"
 
12
  DEFAULT_IMAGE_ROOT = ROOT / "data" / "images"
13
  DEFAULT_SEED_PATH = ROOT / "cert_study_app" / "demo_data" / "questions_seed.json"
14
  DEFAULT_ASSET_ROOT = ROOT / "static" / "question_assets"
@@ -63,19 +64,84 @@ def _copy_image(source_path: Path, asset_root: Path) -> str:
63
  return target_path.relative_to(ROOT).as_posix()
64
 
65
 
66
- def _export_row(row: dict[str, Any], image_root: Path, asset_root: Path) -> dict[str, Any]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  source_pages = row.get("source_pages") or []
68
  if not isinstance(source_pages, list):
69
  source_pages = [source_pages]
70
 
71
- image_path = row.get("image_path")
72
  if not image_path:
73
  source_image = _first_existing_page_image(source_pages, image_root)
74
  if source_image:
75
  image_path = _copy_image(source_image, asset_root)
76
 
77
- question_number = row.get("question_number") or row.get("question_id") or row.get("number")
78
  page = source_pages[0] if source_pages else row.get("page")
 
 
 
 
 
 
 
 
 
 
 
 
79
 
80
  return {
81
  "question_number": question_number,
@@ -90,8 +156,9 @@ def _export_row(row: dict[str, Any], image_root: Path, asset_root: Path) -> dict
90
  "page": page,
91
  "image_path": image_path,
92
  "raw_text": row.get("raw_text") or row.get("stem") or row.get("question") or "",
93
- "parent_stem": row.get("parent_stem") or row.get("scenario"),
94
- "parent_image_paths": row.get("parent_image_paths") or [],
 
95
  "concept_tags": row.get("concept_tags") or [],
96
  "parse_status": row.get("parse_status") or "approved",
97
  "quality_score": row.get("quality_score") or 100,
@@ -101,15 +168,27 @@ def _export_row(row: dict[str, Any], image_root: Path, asset_root: Path) -> dict
101
  "chunk_index": row.get("chunk_index"),
102
  "parser_version": row.get("parser_version") or "hf-seed-export-v1",
103
  "source_pages": source_pages,
104
- }
105
 
106
 
107
- def export_seed(source_json: Path, image_root: Path, seed_path: Path, asset_root: Path) -> int:
 
 
 
 
 
 
108
  rows = _load_rows(source_json)
 
 
 
109
  asset_root.mkdir(parents=True, exist_ok=True)
110
  seed_path.parent.mkdir(parents=True, exist_ok=True)
111
 
112
- exported = [_export_row(row, image_root, asset_root) for row in rows]
 
 
 
113
  with seed_path.open("w", encoding="utf-8") as file:
114
  json.dump(exported, file, ensure_ascii=False, indent=2)
115
  file.write("\n")
@@ -119,6 +198,7 @@ def export_seed(source_json: Path, image_root: Path, seed_path: Path, asset_root
119
  def main() -> None:
120
  parser = argparse.ArgumentParser(description="Export deployable question seed data for Hugging Face Space.")
121
  parser.add_argument("--source-json", type=Path, default=DEFAULT_SOURCE_JSON)
 
122
  parser.add_argument("--image-root", type=Path, default=DEFAULT_IMAGE_ROOT)
123
  parser.add_argument("--seed-path", type=Path, default=DEFAULT_SEED_PATH)
124
  parser.add_argument("--asset-root", type=Path, default=DEFAULT_ASSET_ROOT)
@@ -126,6 +206,7 @@ def main() -> None:
126
 
127
  count = export_seed(
128
  source_json=args.source_json,
 
129
  image_root=args.image_root,
130
  seed_path=args.seed_path,
131
  asset_root=args.asset_root,
 
9
 
10
  ROOT = Path(__file__).resolve().parent.parent
11
  DEFAULT_SOURCE_JSON = ROOT / "data" / "Json" / "questions.json"
12
+ DEFAULT_PARENT_SOURCE_JSON = ROOT / "data" / "parsed_json" / "reparsed_multipage_images_20260601_205501.json"
13
  DEFAULT_IMAGE_ROOT = ROOT / "data" / "images"
14
  DEFAULT_SEED_PATH = ROOT / "cert_study_app" / "demo_data" / "questions_seed.json"
15
  DEFAULT_ASSET_ROOT = ROOT / "static" / "question_assets"
 
64
  return target_path.relative_to(ROOT).as_posix()
65
 
66
 
67
+ def _copy_image_reference(image_path: Any, asset_root: Path) -> str | None:
68
+ if not image_path:
69
+ return None
70
+ source_path = Path(str(image_path))
71
+ if not source_path.is_absolute():
72
+ source_path = ROOT / source_path
73
+ if not source_path.exists():
74
+ return str(image_path)
75
+ return _copy_image(source_path, asset_root)
76
+
77
+
78
+ def _question_number(row: dict[str, Any]) -> Any:
79
+ return row.get("question_number") or row.get("question_id") or row.get("number")
80
+
81
+
82
+ def _parent_context_by_number(parent_rows: list[dict[str, Any]]) -> dict[int, dict[str, Any]]:
83
+ contexts = {}
84
+ for row in parent_rows:
85
+ number = _question_number(row)
86
+ try:
87
+ number = int(number)
88
+ except Exception:
89
+ continue
90
+ if not row.get("parent_stem"):
91
+ continue
92
+ contexts[number] = row
93
+ return contexts
94
+
95
+
96
+ def _parent_context_by_scenario(
97
+ rows: list[dict[str, Any]], parent_contexts: dict[int, dict[str, Any]]
98
+ ) -> dict[str, dict[str, Any]]:
99
+ contexts = {}
100
+ for row in rows:
101
+ scenario = row.get("scenario")
102
+ if not scenario or scenario == "단일문제" or scenario in contexts:
103
+ continue
104
+ try:
105
+ number = int(_question_number(row))
106
+ except Exception:
107
+ continue
108
+ parent_context = parent_contexts.get(number)
109
+ if parent_context:
110
+ contexts[str(scenario)] = parent_context
111
+ return contexts
112
+
113
+
114
+ def _export_row(
115
+ row: dict[str, Any],
116
+ image_root: Path,
117
+ asset_root: Path,
118
+ parent_contexts: dict[int, dict[str, Any]],
119
+ scenario_contexts: dict[str, dict[str, Any]],
120
+ ) -> dict[str, Any]:
121
  source_pages = row.get("source_pages") or []
122
  if not isinstance(source_pages, list):
123
  source_pages = [source_pages]
124
 
125
+ image_path = _copy_image_reference(row.get("image_path"), asset_root)
126
  if not image_path:
127
  source_image = _first_existing_page_image(source_pages, image_root)
128
  if source_image:
129
  image_path = _copy_image(source_image, asset_root)
130
 
131
+ question_number = _question_number(row)
132
  page = source_pages[0] if source_pages else row.get("page")
133
+ try:
134
+ parent_context = parent_contexts.get(int(question_number)) or {}
135
+ except Exception:
136
+ parent_context = {}
137
+ if not parent_context and row.get("scenario"):
138
+ parent_context = scenario_contexts.get(str(row.get("scenario"))) or {}
139
+
140
+ parent_image_paths = []
141
+ for parent_image_path in parent_context.get("parent_image_paths") or row.get("parent_image_paths") or []:
142
+ copied_path = _copy_image_reference(parent_image_path, asset_root)
143
+ if copied_path:
144
+ parent_image_paths.append(copied_path)
145
 
146
  return {
147
  "question_number": question_number,
 
156
  "page": page,
157
  "image_path": image_path,
158
  "raw_text": row.get("raw_text") or row.get("stem") or row.get("question") or "",
159
+ "group_id": parent_context.get("group_id") or row.get("group_id"),
160
+ "parent_stem": parent_context.get("parent_stem") or row.get("parent_stem") or row.get("scenario"),
161
+ "parent_image_paths": parent_image_paths,
162
  "concept_tags": row.get("concept_tags") or [],
163
  "parse_status": row.get("parse_status") or "approved",
164
  "quality_score": row.get("quality_score") or 100,
 
168
  "chunk_index": row.get("chunk_index"),
169
  "parser_version": row.get("parser_version") or "hf-seed-export-v1",
170
  "source_pages": source_pages,
171
+ }
172
 
173
 
174
+ def export_seed(
175
+ source_json: Path,
176
+ parent_source_json: Path,
177
+ image_root: Path,
178
+ seed_path: Path,
179
+ asset_root: Path,
180
+ ) -> int:
181
  rows = _load_rows(source_json)
182
+ parent_rows = _load_rows(parent_source_json) if parent_source_json.exists() else []
183
+ parent_contexts = _parent_context_by_number(parent_rows)
184
+ scenario_contexts = _parent_context_by_scenario(rows, parent_contexts)
185
  asset_root.mkdir(parents=True, exist_ok=True)
186
  seed_path.parent.mkdir(parents=True, exist_ok=True)
187
 
188
+ exported = [
189
+ _export_row(row, image_root, asset_root, parent_contexts, scenario_contexts)
190
+ for row in rows
191
+ ]
192
  with seed_path.open("w", encoding="utf-8") as file:
193
  json.dump(exported, file, ensure_ascii=False, indent=2)
194
  file.write("\n")
 
198
  def main() -> None:
199
  parser = argparse.ArgumentParser(description="Export deployable question seed data for Hugging Face Space.")
200
  parser.add_argument("--source-json", type=Path, default=DEFAULT_SOURCE_JSON)
201
+ parser.add_argument("--parent-source-json", type=Path, default=DEFAULT_PARENT_SOURCE_JSON)
202
  parser.add_argument("--image-root", type=Path, default=DEFAULT_IMAGE_ROOT)
203
  parser.add_argument("--seed-path", type=Path, default=DEFAULT_SEED_PATH)
204
  parser.add_argument("--asset-root", type=Path, default=DEFAULT_ASSET_ROOT)
 
206
 
207
  count = export_seed(
208
  source_json=args.source_json,
209
+ parent_source_json=args.parent_source_json,
210
  image_root=args.image_root,
211
  seed_path=args.seed_path,
212
  asset_root=args.asset_root,
static/question_assets/AZ-104_dump_56cadaca/page_12.jpg ADDED

Git LFS Details

  • SHA256: 7416abeec52a3d1f3dde8621fac76fcd0b4970fce48b580d02263b6282a298e2
  • Pointer size: 131 Bytes
  • Size of remote file: 427 kB
static/question_assets/AZ-104_dump_56cadaca/page_21.jpg ADDED

Git LFS Details

  • SHA256: 0bd4e0dd3bf0d1dd2f1b505dca6d592fea4d04b353c722304410a40a2567f8a3
  • Pointer size: 131 Bytes
  • Size of remote file: 493 kB
static/question_assets/AZ-104_dump_56cadaca/page_30.jpg ADDED

Git LFS Details

  • SHA256: 8b5ae6eb0a60219307064f5180987f57325fe9d6f1ce788f7d3490ce51e220ab
  • Pointer size: 131 Bytes
  • Size of remote file: 467 kB
static/question_assets/AZ-104_dump_56cadaca/page_32.jpg ADDED

Git LFS Details

  • SHA256: b127f827faf8afee4e80e73521c94a2a1f5cbfd291f0ae31b1d936cb835e2f4a
  • Pointer size: 131 Bytes
  • Size of remote file: 439 kB